bjooo commited on
Commit
d140a6f
·
verified ·
1 Parent(s): 4fee53b

Delete dolphin_vision_director.py

Browse files
Files changed (1) hide show
  1. dolphin_vision_director.py +0 -788
dolphin_vision_director.py DELETED
@@ -1,788 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- import os
3
- import re
4
- import io
5
- import json
6
- import base64
7
- import hashlib
8
- import sqlite3
9
- import urllib.request
10
- import urllib.error
11
-
12
- import numpy as np
13
- import folder_paths
14
- import comfy.sd
15
- import comfy.lora
16
- import comfy.utils
17
- from safetensors import safe_open
18
- from comfy_execution.graph_utils import ExecutionBlocker
19
-
20
- try:
21
- from PIL import Image
22
- _HAS_PIL = True
23
- except Exception:
24
- _HAS_PIL = False
25
-
26
- CACHE_FILE = os.path.join(os.path.dirname(__file__), "dolphin_tags_cache.json")
27
-
28
-
29
- # ======================================================================
30
- # 모델 프리셋 (텍스트 LLM = 4샷 분배 / 비전 VLM = 이미지 인식)
31
- # ======================================================================
32
- LLM_PRESETS = {
33
- "🔞 DeepSeek V3.2 (무삭제·JSON안정·추천)": "deepseek/deepseek-v3.2",
34
- "🔞 Magnum v4 72B (표현 자연스러움)": "anthracite-org/magnum-v4-72b",
35
- "🔞 Cydonia 24B v4.1 (경량·빠름)": "thedrummer/cydonia-24b-v4.1",
36
- "🔞 Dolphin Mistral Venice (무료)": "cognitivecomputations/dolphin-mistral-24b-venice-edition:free",
37
- "⚙️ custom (아래 텍스트 사용)": "__custom__",
38
- }
39
- LLM_PRESET_LABELS = list(LLM_PRESETS.keys())
40
-
41
- # 비전 모델: 특수 상황 이해 + 다국어. Qwen2.5-VL 계열이 포즈/맥락 파악에 강함.
42
- VLM_PRESETS = {
43
- "👁 Qwen2.5-VL 72B (특수상황·추천)": "qwen/qwen2.5-vl-72b-instruct",
44
- "👁 Qwen2.5-VL 32B (균형)": "qwen/qwen2.5-vl-32b-instruct",
45
- "👁 Qwen2.5-VL 32B (무료)": "qwen/qwen2.5-vl-32b-instruct:free",
46
- "👁 Mistral Small 3.1 24B (Pixtral)": "mistralai/mistral-small-3.1-24b-instruct",
47
- "👁 Gemma 3 27B (경량)": "google/gemma-3-27b-it",
48
- "⚙️ custom (아래 텍스트 사용)": "__custom__",
49
- }
50
- VLM_PRESET_LABELS = list(VLM_PRESETS.keys())
51
-
52
-
53
- def resolve_slug(table, label, custom_text, default):
54
- slug = table.get(label, "__custom__")
55
- if slug == "__custom__":
56
- return custom_text.strip() or default
57
- return slug
58
-
59
-
60
- def resolve_api_key(widget_value):
61
- """위젯에 키가 있으면 그대로 사용. 비어있으면 OPENROUTER_API_KEY 환경변수로 폴백.
62
- -> 워크플로우 JSON(공유/백업 시 유출 위험)에 실키를 박아둘 필요가 없어짐."""
63
- key = (widget_value or "").strip()
64
- if key:
65
- return key
66
- return os.environ.get("OPENROUTER_API_KEY", "").strip()
67
-
68
-
69
- # ======================================================================
70
- # 캐시 & 트리거 (검증된 로직 재사용)
71
- # ======================================================================
72
- def _load_cache():
73
- if os.path.exists(CACHE_FILE):
74
- try:
75
- with open(CACHE_FILE, "r", encoding="utf-8") as f:
76
- return json.load(f)
77
- except (json.JSONDecodeError, OSError):
78
- return {}
79
- return {}
80
-
81
-
82
- def _save_cache(cache):
83
- try:
84
- with open(CACHE_FILE, "w", encoding="utf-8") as f:
85
- json.dump(cache, f, indent=4, ensure_ascii=False)
86
- except OSError:
87
- pass
88
-
89
-
90
- def _file_sig(path):
91
- st = os.stat(path)
92
- return f"{os.path.basename(path)}::{st.st_size}::{int(st.st_mtime)}"
93
-
94
-
95
- def _sha256(path):
96
- h = hashlib.sha256()
97
- with open(path, "rb") as f:
98
- for block in iter(lambda: f.read(4096 * 1024), b""):
99
- h.update(block)
100
- return h.hexdigest()
101
-
102
-
103
- # CivitAI 및 미러(civitaired) 해시 조회 엔드포인트. 앞에서부터 순서대로 시도.
104
- TRIGGER_HASH_APIS = [
105
- "https://civitai.com/api/v1/model-versions/by-hash/{h}",
106
- "https://civitaired.com/api/v1/model-versions/by-hash/{h}",
107
- ]
108
-
109
- # LoRA Manager(comfyui-lora-manager)가 스캔 시 만들어두는 로컬 SQLite 캐시.
110
- # sha256으로 trained_words를 즉시 조회 가능 -> 네트워크 호출 없이 트리거워드 확보.
111
- LORA_MANAGER_DB = os.path.join(
112
- os.environ.get("LOCALAPPDATA", ""), "ComfyUI-LoRA-Manager", "cache", "model", "comfyui.sqlite")
113
-
114
-
115
- def _query_lora_manager_db(digest):
116
- """LoRA Manager SQLite 캐시에서 sha256으로 trained_words 조회. 실패/미발견 시 None."""
117
- if not digest or not os.path.exists(LORA_MANAGER_DB):
118
- return None
119
- try:
120
- uri = f"file:{LORA_MANAGER_DB}?mode=ro"
121
- con = sqlite3.connect(uri, uri=True, timeout=3)
122
- try:
123
- cur = con.cursor()
124
- cur.execute(
125
- "SELECT trained_words FROM models WHERE model_type='lora' AND sha256=? LIMIT 1",
126
- (digest,))
127
- row = cur.fetchone()
128
- finally:
129
- con.close()
130
- if not row or not row[0]:
131
- return None
132
- tw = json.loads(row[0])
133
- return tw if isinstance(tw, list) else None
134
- except (sqlite3.Error, OSError, ValueError, json.JSONDecodeError) as e:
135
- print(f"⚠️ [Director] LoRA Manager DB 조회 실패: {type(e).__name__}: {e}")
136
- return None
137
-
138
-
139
- def _query_trigger_api(url):
140
- """단일 해시 조회 엔드포인트에서 trainedWords 리스트를 반환. 실패 시 ([], 사유)."""
141
- try:
142
- req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
143
- with urllib.request.urlopen(req, timeout=8) as r:
144
- data = json.loads(r.read().decode('utf-8'))
145
- return (data.get("trainedWords", []) or []), None
146
- except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError,
147
- OSError, ValueError, json.JSONDecodeError) as e:
148
- return [], f"{type(e).__name__}: {e}"
149
-
150
-
151
- def get_triggers(lora_name, cache):
152
- if lora_name == "None":
153
- return []
154
- path = folder_paths.get_full_path("loras", lora_name)
155
- if not path or not os.path.exists(path):
156
- return []
157
- try:
158
- with safe_open(path, framework="pt", device="cpu") as f:
159
- m = f.metadata()
160
- if m and "modelspec.trigger_words" in m:
161
- tw = json.loads(m["modelspec.trigger_words"])
162
- if isinstance(tw, list):
163
- return tw
164
- if isinstance(tw, str):
165
- return [tw]
166
- except (OSError, ValueError, json.JSONDecodeError):
167
- pass
168
- try:
169
- sig = _file_sig(path)
170
- except OSError:
171
- sig = lora_name
172
- # 캐시에 '비어있지 않은' 결과가 있을 때만 신뢰. 과거에 실패로 저장된 빈 값은
173
- # 무시하고 재조회한다 (빈 값 영구화 회귀 방지).
174
- cached = cache.get(sig)
175
- if cached:
176
- return cached
177
-
178
- # 해시는 1회만 계산해서 LoRA Manager DB -> CivitAI -> civitaired 순으로 조회.
179
- result, base = [], os.path.basename(path)
180
- try:
181
- digest = _sha256(path)
182
- except OSError as e:
183
- print(f"⚠️ [Director] 해시 계산 실패({base}): {e}")
184
- digest = None
185
- if digest:
186
- found = _query_lora_manager_db(digest)
187
- if found:
188
- print(f"✅ [Director] 트리거 조회 성공({base}) @ LoRA Manager DB: {found}")
189
- result = found
190
- if digest and not result:
191
- for tmpl in TRIGGER_HASH_APIS:
192
- host = tmpl.split('/')[2]
193
- found, err = _query_trigger_api(tmpl.format(h=digest))
194
- if found:
195
- print(f"✅ [Director] 트리거 조회 성공({base}) @ {host}: {found}")
196
- result = found
197
- break
198
- print(f"⚠️ [Director] 트리거 조회 실패({base}) @ {host}: {err or 'no trainedWords'}")
199
-
200
- # 성공(비어있지 않음)일 때만 캐싱. 빈 결과는 저장하지 않아 다음 실행에서 재시도됨.
201
- if result:
202
- cache[sig] = result
203
- _save_cache(cache)
204
- return result
205
-
206
-
207
- def apply_model_lora(model, lora_data, weight):
208
- try:
209
- return comfy.sd.load_lora_for_models(model, None, lora_data, weight, 0)[0]
210
- except (AttributeError, TypeError):
211
- new_model = model.clone()
212
- key_map = comfy.lora.model_lora_keys_unet(new_model.model)
213
- loaded = comfy.lora.load_lora(lora_data, key_map)
214
- new_model.add_patches(loaded, weight)
215
- return new_model
216
-
217
-
218
- def tensor_to_base64(image_tensor, max_side=1024):
219
- """ComfyUI IMAGE 텐서(B,H,W,C, 0~1 float) 첫 프레임을 JPEG base64 data URI로."""
220
- if not _HAS_PIL:
221
- print("❌ [Director] PIL(Pillow) 없음 - 비전 불가. `pip install Pillow` 필요.")
222
- return None
223
- if image_tensor is None:
224
- print("❌ [Director] image 입력이 None - IMAGE 연결 확인.")
225
- return None
226
- try:
227
- img = image_tensor
228
- if hasattr(img, "cpu"):
229
- img = img.cpu().numpy()
230
- img = np.asarray(img)
231
- if img.ndim == 4:
232
- img = img[0]
233
- arr = np.clip(img * 255.0, 0, 255).astype(np.uint8)
234
- pil = Image.fromarray(arr)
235
- w, h = pil.size
236
- if max(w, h) > max_side:
237
- scale = max_side / float(max(w, h))
238
- pil = pil.resize((int(w * scale), int(h * scale)), Image.LANCZOS)
239
- buf = io.BytesIO()
240
- pil.convert("RGB").save(buf, format="JPEG", quality=90)
241
- b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
242
- print(f"🖼 [Director] image encoded ok: {w}x{h} -> {pil.size}, {len(b64)//1024}KB base64")
243
- return f"data:image/jpeg;base64,{b64}"
244
- except Exception as e:
245
- print(f"❌ [Director] image encode failed: {type(e).__name__}: {e}")
246
- return None
247
-
248
-
249
- # ======================================================================
250
- # Vision Director: 이미지 인식 + 필수동작 실행 + 클립당 4로라
251
- # ======================================================================
252
- class DolphinVisionDirector:
253
- """
254
- 이미지 하나와 '필수 동작'과 로라만 넣으면 4개 연속 클립이 완성되는 올인원 노드.
255
-
256
- 1) IMAGE -> 비전 VLM 이 상황(캐릭터 상태/자세/환경)만 앵커로 파악
257
- 2) 그 맥락 위에서 '내가 준 필수 동작(essential_actions)'만 4샷으로 분배
258
- 3) 클립마다 로라 4개씩 개별 적용(웨이트도 개별) + 트리거 자동 주입
259
- """
260
-
261
- @classmethod
262
- def INPUT_TYPES(s):
263
- loras = ["None"] + (folder_paths.get_filename_list("loras") or [])
264
- req = {
265
- "image": ("IMAGE",),
266
- "model_base": ("MODEL",),
267
- "model_high": ("MODEL",),
268
- "model_low": ("MODEL",),
269
- "essential_actions": ("STRING", {"multiline": True,
270
- "default": ("돌진한다, 검을 휘둘러 적을 쓰러뜨린다, 총알을 튕겨낸다, 숨통을 끊는다")}),
271
- "num_clips": (["4 (20s)", "3 (15s)", "2 (10s)"], {"default": "4 (20s)",
272
- "tooltip": "실제로 렌더링할 클립 수. Extend range(Fast Groups Bypasser) 토글과 맞춰서 "
273
- "설정해야 스토리가 그 길이에 맞게 배분됨. 나머지 클립 슬롯은 마지막 클립 내용을 반복."}),
274
- "character_name": ("STRING", {"default": "AUTO"}),
275
- "artistic_vibe": ("STRING", {"multiline": True,
276
- "default": "cinematic lighting, dynamic motion"}),
277
- "pacing": (["auto", "slow-burn (1 beat)", "balanced (multi beat)", "rapid (4 distinct)"],
278
- {"default": "auto"}),
279
- "detail_level": (["짧게(간결)", "필수+앵커만", "상세"], {"default": "필수+앵커만"}),
280
- "inject_triggers": ("BOOLEAN", {"default": True,
281
- "label_on": "TRIGGERS ON", "label_off": "TRIGGERS OFF"}),
282
- "use_vision": ("BOOLEAN", {"default": True,
283
- "label_on": "VISION ON", "label_off": "VISION OFF (text only)"}),
284
- "openrouter_api_key": ("STRING", {"default": ""}),
285
- "vlm_preset": (VLM_PRESET_LABELS, {"default": "👁 Qwen2.5-VL 72B (특수상황·추천)"}),
286
- "vlm_model": ("STRING", {"default": "qwen/qwen2.5-vl-72b-instruct"}),
287
- "llm_preset": (LLM_PRESET_LABELS, {"default": "🔞 DeepSeek V3.2 (무삭제·JSON안정·추천)"}),
288
- "llm_model": ("STRING", {"default": "deepseek/deepseek-v3.2"}),
289
- "creativity": ("FLOAT", {"default": 0.7, "min": 0.1, "max": 1.5, "step": 0.05}),
290
- "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
291
- }
292
- opt = {}
293
- for c in range(1, 5):
294
- for n in range(1, 5):
295
- opt[f"c{c}_lora_{n}"] = (loras, {"default": "None"})
296
- opt[f"c{c}_w_{n}"] = ("STRING", {"default": "1.0, 1.0, 1.0", "multiline": False})
297
- opt["manual_triggers"] = ("STRING", {"default": "", "multiline": True})
298
- opt["vision_caption"] = ("STRING", {"multiline": True, "default": "",
299
- "forceInput": True})
300
- return {"required": req, "optional": opt}
301
-
302
- RETURN_TYPES = (
303
- "MODEL", "MODEL", "MODEL", "STRING",
304
- "MODEL", "MODEL", "MODEL", "STRING",
305
- "MODEL", "MODEL", "MODEL", "STRING",
306
- "MODEL", "MODEL", "MODEL", "STRING",
307
- "STRING", "STRING", "STRING",
308
- "BOOLEAN", "BOOLEAN", "BOOLEAN", "BOOLEAN",
309
- )
310
- RETURN_NAMES = (
311
- "c1_base", "c1_high", "c1_low", "c1_prompt",
312
- "c2_base", "c2_high", "c2_low", "c2_prompt",
313
- "c3_base", "c3_high", "c3_low", "c3_prompt",
314
- "c4_base", "c4_high", "c4_low", "c4_prompt",
315
- "triggers", "scene_analysis", "debug_info",
316
- "c1_is_final", "c2_is_final", "c3_is_final", "c4_is_final",
317
- )
318
- FUNCTION = "direct"
319
- CATEGORY = "Dolphin"
320
-
321
- def _build_final(self, action, master_scene, name, trigger_str):
322
- head = ", ".join([p for p in [name, trigger_str] if p])
323
- body = " ".join([p for p in [master_scene.strip().strip(",. "), action.strip()] if p])
324
- if head and body:
325
- return f"{head}\n{body}"
326
- return head or body
327
-
328
- def _pacing_hint(self, p):
329
- return {
330
- "auto": "Decide pacing yourself based on how many distinct beats the actions contain.",
331
- "slow-burn (1 beat)": "One climactic beat: clips 1-3 build tension, clip 4 detonates.",
332
- "balanced (multi beat)": "Early clips build, later clips deliver; keep motion continuous.",
333
- "rapid (4 distinct)": "Four distinct consecutive actions, one per clip, each kinetic.",
334
- }.get(p, "Decide pacing yourself.")
335
-
336
- def _detail_hint(self, d):
337
- return {
338
- "짧게(간결)": "Keep each clip to a short punchy motion phrase. Pure movement verbs only.",
339
- "필수+앵커만": "Each clip is one concise continuous motion. No description, only movement.",
340
- "상세": "Motion may be described in more detail, but ONLY body movement — no scenery.",
341
- }.get(d, "Motion only, no description.")
342
-
343
- def _extract_json(self, text):
344
- if not text:
345
- return None
346
- text = re.sub(r'```(?:json)?', '', text).strip()
347
- a, b = text.find('{'), text.rfind('}')
348
- if a == -1 or b <= a:
349
- return None
350
- blob = text[a:b + 1]
351
- try:
352
- return json.loads(blob)
353
- except json.JSONDecodeError:
354
- try:
355
- return json.loads(re.sub(r',\s*([}\]])', r'\1', blob))
356
- except json.JSONDecodeError:
357
- return None
358
-
359
- def _call_or(self, messages, model, key, temp, timeout=120, force_json=False):
360
- payload = {"model": model.strip(), "messages": messages, "temperature": temp}
361
- if force_json:
362
- payload["response_format"] = {"type": "json_object"}
363
- req = urllib.request.Request(
364
- "https://openrouter.ai/api/v1/chat/completions",
365
- data=json.dumps(payload).encode('utf-8'),
366
- headers={'Authorization': f'Bearer {key}', 'Content-Type': 'application/json'})
367
- raw = json.loads(urllib.request.urlopen(req, timeout=timeout).read().decode('utf-8'))
368
- if "choices" not in raw:
369
- err = raw.get("error") or raw
370
- msg = err.get("message") if isinstance(err, dict) else str(err)
371
- code = err.get("code") if isinstance(err, dict) else "?"
372
- raise ValueError(f"API error (code={code}): {msg}")
373
- return raw['choices'][0]['message']['content'].strip()
374
-
375
- def _vision_analyze(self, data_uri, vlm_model, key, temp):
376
- sys = ("You are a visual scene analyst for a video pipeline. "
377
- "Look at the image and extract ONLY concrete anchors, no storytelling. "
378
- "Report exactly these lines:\n"
379
- "(1) character appearance & clothing.\n"
380
- "(2) current pose/posture (sitting, crouched, standing, balance, limb positions).\n"
381
- "(3) SUPPORT & CONTACT: what the character is on top of, sitting on, holding, "
382
- "leaning against, or touching (e.g. 'seated balanced on a yoga ball', "
383
- "'gripping a sword', 'kneeling on the floor'). This is critical for physics — "
384
- "state it explicitly even if obvious.\n"
385
- "(4) environment & lighting (brief).\n"
386
- "Be terse, one short line each. English only. "
387
- "Do not invent actions or narrative. If content is mature, describe it plainly and factually.")
388
- messages = [
389
- {"role": "system", "content": sys},
390
- {"role": "user", "content": [
391
- {"type": "text", "text": "Extract the anchors from this image."},
392
- {"type": "image_url", "image_url": {"url": data_uri}},
393
- ]},
394
- ]
395
- import time
396
- last_err = "vision error"
397
- for attempt in range(1, 4):
398
- try:
399
- resp = self._call_or(messages, vlm_model, key, temp, timeout=90).strip()
400
- if not resp:
401
- print(f"⚠️ [Director] VLM({vlm_model}) 응답이 비었음 (거부 or 빈 반환)")
402
- return "", "vision empty response"
403
- print(f"👁 [Director] VLM({vlm_model}) anchors (attempt {attempt}):\n{resp[:300]}")
404
- return resp, None
405
- except urllib.error.HTTPError as e:
406
- body = ""
407
- try:
408
- body = e.read().decode('utf-8')[:200]
409
- except Exception:
410
- pass
411
- print(f"❌ [Director] VLM HTTP {e.code} (attempt {attempt}): {body}")
412
- last_err = f"vision HTTP {e.code}"
413
- if e.code in (429, 503) and attempt < 3:
414
- wait = 2 ** attempt
415
- print(f"⏳ [Director] rate limited, {wait}s 후 재시도...")
416
- time.sleep(wait)
417
- continue
418
- return "", last_err
419
- except (urllib.error.URLError, TimeoutError, KeyError, ValueError) as e:
420
- print(f"❌ [Director] VLM error (attempt {attempt}): {type(e).__name__}: {e}")
421
- last_err = f"vision error: {e}"
422
- if attempt < 3:
423
- time.sleep(2 ** attempt)
424
- continue
425
- return "", last_err
426
- return "", last_err
427
-
428
- def _fallback_name(self, anchors):
429
- import random
430
- a = (anchors or "").lower()
431
- east = ["asian", "east asian", "japanese", "korean", "chinese", "anime",
432
- "kimono", "hanbok", "qipao", "oriental"]
433
- western = ["western", "european", "caucasian", "american", "blonde", "redhead"]
434
- east_names = ["Yuna", "Kaede", "Jin", "Mei", "Haruka", "Rin", "Sora", "Aoi"]
435
- west_names = ["Elena", "Ryan", "Ava", "Lucas", "Mila", "Ethan", "Nora", "Leo"]
436
- if any(k in a for k in east):
437
- pool = east_names
438
- elif any(k in a for k in western):
439
- pool = west_names
440
- else:
441
- pool = east_names + west_names
442
- return random.choice(pool)
443
-
444
- def _split_actions_to_clips(self, actions, num_clips=4):
445
- parts = re.split(r'(?<=[.!?。])\s+|[,、]|\n+', actions)
446
- parts = [p.strip() for p in parts if len(p.strip()) > 1]
447
- if not parts:
448
- parts = [actions.strip()]
449
- n = len(parts)
450
- last = num_clips
451
- assign = {c: [] for c in range(1, num_clips + 1)}
452
- if num_clips == 1:
453
- assign[1] = parts
454
- return assign
455
- build_labels = ["(build-up toward)", "(begin)", "(escalate)", "(continue escalating)"]
456
- if n == 1:
457
- for c in range(1, last):
458
- label = build_labels[min(c - 1, len(build_labels) - 1)]
459
- assign[c] = [f"{label} {parts[0]}"]
460
- assign[last] = [parts[0]]
461
- elif n <= num_clips:
462
- assign[last] = [parts[-1]]
463
- head = parts[:-1]
464
- for i, ph in enumerate(head):
465
- assign[min(i + 1, last - 1)].append(ph)
466
- for c in range(1, last):
467
- if not assign[c]:
468
- nxt = parts[min(c, n - 1)]
469
- assign[c] = [f"(move toward) {nxt}"]
470
- else:
471
- assign[last] = [parts[-1]]
472
- rest = parts[:-1]
473
- k, m = divmod(len(rest), last - 1)
474
- idx = 0
475
- for c in range(1, last):
476
- end = idx + k + (1 if (c - 1) < m else 0)
477
- assign[c] = rest[idx:end]
478
- idx = end
479
- return assign
480
-
481
- def _plan_shots(self, anchors, actions, name, vibe, pacing, detail, llm_model, key, temp, seed, num_clips=4):
482
- assign = self._split_actions_to_clips(actions, num_clips)
483
- assign_block = "\n".join(
484
- f" clip_{c}: {', '.join(assign[c]) if assign[c] else '(continue previous motion)'}"
485
- for c in range(1, num_clips + 1))
486
-
487
- last_clip = num_clips
488
- total_seconds = num_clips * 5
489
- if num_clips >= 3:
490
- chain = " and ".join(f"clip_{c}<-clip_{c-1}" for c in range(3, num_clips + 1))
491
- continuity_middle = (f". The same applies {chain}: each clip picks up exactly where the "
492
- f"previous clip's body position left off")
493
- else:
494
- continuity_middle = ""
495
-
496
- continuity_rule = ""
497
- if num_clips >= 2:
498
- continuity_rule = (
499
- "3) INTER-CLIP CONTINUITY. clip_2 MUST begin from the exact physical end-state of "
500
- "clip_1's action (same position, body orientation, and momentum) — it continues the "
501
- "motion, it does not restart from a neutral stance" + continuity_middle +
502
- f", as one unbroken {total_seconds}-second performance rather than {num_clips} separate poses. Do "
503
- "NOT explicitly narrate the hand-off (no phrases like 'continuing from', 'still', 'as "
504
- "before', 'picking up where') — simply choreograph the new action so it is kinematically "
505
- "consistent with how the previous action would have left the body positioned. This is a "
506
- "physical-continuity constraint, not a scripted transition line.\n"
507
- )
508
-
509
- json_clip_fields = ",".join(
510
- f'"clip_{i}":"<only clip_{i} action{", the finisher" if i == last_clip else ""}, '
511
- f'continuous & fast>"' for i in range(1, num_clips + 1))
512
-
513
- sys = (
514
- f"You are an elite fight/action choreographer for a {total_seconds}-second video made of "
515
- f"{num_clips} consecutive 5-second clip{'s' if num_clips != 1 else ''}. You are given SCENE ANCHORS "
516
- "(the character's starting look/pose/place from an image) and a STRICT per-clip action "
517
- "assignment.\n"
518
- "\n"
519
- "ABSOLUTE RULES:\n"
520
- "1) FOLLOW THE ASSIGNMENT EXACTLY. Each clip animates ONLY the action(s) assigned "
521
- "to it below. You MUST NOT move a later clip's action into an earlier clip. "
522
- f"clip_1 must NOT contain clip_{last_clip}'s action. The final action happens ONLY in "
523
- f"clip_{last_clip}. "
524
- "This is the most important rule - violating the order is a failure. "
525
- "(ONE exception: clip_1 may prepend a brief transition OUT of the starting physics from "
526
- "rule 2 — e.g. leaving the yoga ball — before its assigned action. This is not borrowing "
527
- "a later action; it is grounding the first one.)\n"
528
- "2) STARTING PHYSICS (use the anchors here). Read the character's CURRENT pose AND "
529
- "what they are supported by / in contact with from the SCENE ANCHORS (e.g. seated on a "
530
- "yoga ball, kneeling, gripping a weapon, leaning on a wall). clip_1 MUST begin the first "
531
- "action FROM that exact physical situation and honor its constraints. If the support is "
532
- "unstable or unusual (a yoga ball, a ledge, a moving surface), the motion MUST account "
533
- "for it — e.g. anchor 'seated balanced on a yoga ball' -> clip_1 = 'pushing off the "
534
- "wobbling ball and rising to their feet as it rolls away' BEFORE any further action. "
535
- "Never ignore or contradict the starting support (do not stand if seated on the ball "
536
- "without first leaving it; do not assume a weapon is drawn if the anchor shows it "
537
- "sheathed). This makes the motion physically continuous with the input image. Reference "
538
- "the support ONLY to ground how the motion STARTS — do NOT describe its appearance, "
539
- "color, or the wider environment.\n"
540
- + continuity_rule +
541
- "4) ACTION ONLY. Write physical body movement. NO scenery, NO lighting, NO clothing, "
542
- "NO mood, NO camera talk. If a word is not a movement or body part, delete it. "
543
- "(Referring to the starting pose in clip_1 per rule 2, or to the prior clip's ending "
544
- "position per rule 3, is allowed since it describes how the body moves.)\n"
545
- "5) FILL 5 SECONDS. Expand the assigned action into a single continuous flow of motion "
546
- "that occupies the whole clip (use 'then', 'immediately', 'without pausing'). "
547
- "Do not borrow the next action to fill time - elaborate the CURRENT action instead.\n"
548
- "6) ANTI-SLOW-MOTION: real-time speed, kinetic adverbs (swiftly, rapidly, explosively, "
549
- "in a split second). NO slow motion, NO freeze, NO holding a pose.\n"
550
- "7) CHARACTER NAME: study the SCENE ANCHORS. Judge the character's apparent ethnicity/"
551
- "setting (East Asian, Western, etc.) and INVENT a specific fitting first name that "
552
- "matches it (e.g. East Asian -> 'Yuna','Kaede','Jin'; Western -> 'Elena','Ryan'). "
553
- "NEVER output 'AUTO' or an empty name. If a NAME is given below (not 'AUTO'), use it.\n"
554
- f"8) {detail}\n"
555
- "9) MOTION ENERGY: a MOTION STYLE cue is given below (e.g. 'cinematic lighting, dynamic "
556
- "motion'). Use it ONLY to calibrate how forceful/graceful/frantic the movement FEELS "
557
- "(word choice, verb intensity). Do NOT quote it and do NOT let any scenery/lighting/mood "
558
- "words from it leak into the output — rule 4 still applies.\n"
559
- "\n"
560
- "master_scene: leave it EMPTY.\n"
561
- "\n"
562
- 'OUTPUT ONLY JSON: {"character":"<a real name, never AUTO>","master_scene":"",'
563
- + json_clip_fields + "}"
564
- )
565
- usr = (f"NAME: {name}\nPACING: {self._pacing_hint(pacing)}\n\n"
566
- f"MOTION STYLE (kinetic energy cue only, per rule 9 — do not describe scenery/"
567
- f"lighting/mood from this): {vibe.strip() or '(none)'}\n\n"
568
- f"SCENE ANCHORS (use the POSE to start clip_1's motion per rule 2; "
569
- f"use overall look only for naming — do NOT copy look into the clips):\n"
570
- f"{anchors or '(none — assume a neutral ready stance)'}\n\n"
571
- f"STRICT PER-CLIP ACTION ASSIGNMENT (animate ONLY what is listed per clip, "
572
- f"in this order, finisher in clip_{last_clip}):\n{assign_block}\n")
573
- messages = [{"role": "system", "content": sys}, {"role": "user", "content": usr}]
574
- t = temp
575
- last = "⚠️ plan failed (no attempts ran)"
576
- for attempt in (1, 2):
577
- try:
578
- content = self._call_or(messages, llm_model, key, t, force_json=True)
579
- parsed = self._extract_json(content)
580
- if parsed and all(parsed.get(f"clip_{i}") for i in range(1, num_clips + 1)):
581
- return parsed, f"✅ plan ok (attempt {attempt})"
582
- last = f"⚠️ incomplete JSON (attempt {attempt}): {content[:150]!r}"
583
- print(f"⚠️ [Director] plan attempt {attempt} returned incomplete JSON: {content[:300]}")
584
- except urllib.error.HTTPError as e:
585
- body = ""
586
- try:
587
- body = e.read().decode('utf-8')[:200]
588
- except Exception:
589
- pass
590
- last = f"❌ plan HTTP {e.code} (attempt {attempt}): {body}"
591
- print(f"❌ [Director] plan {last}")
592
- except (urllib.error.URLError, TimeoutError, KeyError, ValueError) as e:
593
- last = f"❌ plan error (attempt {attempt}): {type(e).__name__}: {e}"
594
- print(f"❌ [Director] {last}")
595
- t = min(temp, 0.4)
596
- return None, last
597
-
598
- def _local_split(self, actions, num_clips=4):
599
- parts = re.split(r'(?<=[.!?。])\s+|[,、]|\n+', actions)
600
- parts = [p.strip() for p in parts if len(p.strip()) > 1]
601
- if not parts:
602
- parts = [actions.strip()]
603
- n = len(parts)
604
- last = num_clips - 1
605
-
606
- def fast(phrase, lead="swiftly"):
607
- p = phrase.strip()
608
- return f"{lead} {p}, continuous real-time motion, no slow motion" if p else ""
609
-
610
- if num_clips == 1:
611
- return [fast(", ".join(parts), "explosively")]
612
-
613
- if n == 1:
614
- a = parts[0]
615
- templates = [
616
- f"rapidly moves into position toward {a}, no pause",
617
- f"immediately begins {a}, brisk continuous motion, no slow motion",
618
- f"presses {a} without stopping, fast decisive movement",
619
- ]
620
- out = [templates[min(i, len(templates) - 1)] for i in range(num_clips - 1)]
621
- out.append(f"explosively completes {a} at real-time speed, no slow motion, no freeze")
622
- elif n <= num_clips:
623
- out = [""] * num_clips
624
- out[last] = fast(parts[-1], "explosively")
625
- head = parts[:-1]
626
- for i, ph in enumerate(head):
627
- s = min(i, num_clips - 2)
628
- out[s] = (out[s] + ", then " + fast(ph)).strip(", ") if out[s] else fast(ph)
629
- for i in range(num_clips - 1):
630
- if not out[i]:
631
- key = parts[min(i, n - 1)]
632
- out[i] = f"rapidly moves toward {key}, brisk continuous motion, no slow motion"
633
- else:
634
- k, m = divmod(n - 1, num_clips - 1)
635
- idx, chunks = 0, []
636
- for i in range(num_clips - 1):
637
- end = idx + k + (1 if i < m else 0)
638
- seg = parts[idx:end]
639
- idx = end
640
- chunks.append(", then ".join(fast(p) for p in seg) if seg else "")
641
- out = chunks + [fast(parts[-1], "explosively")]
642
- return out[:num_clips]
643
-
644
- def _load(self, name):
645
- return comfy.utils.load_torch_file(folder_paths.get_full_path("loras", name))
646
-
647
- @staticmethod
648
- def _parse_weights(text):
649
- try:
650
- nums = [float(x) for x in re.split(r'[,\s]+', text.strip()) if x != ""]
651
- except (ValueError, AttributeError):
652
- nums = []
653
- if not nums:
654
- return 1.0, 1.0, 1.0
655
- if len(nums) == 1:
656
- return nums[0], nums[0], nums[0]
657
- if len(nums) == 2:
658
- return nums[0], nums[1], nums[1]
659
- return nums[0], nums[1], nums[2]
660
-
661
- def _patch_clip(self, mb, mh, ml, slots, cache, inject):
662
- cb, ch, cl = mb, mh, ml
663
- used = []
664
- for (name, w_str) in slots:
665
- if name == "None":
666
- continue
667
- w_b, w_h, w_l = self._parse_weights(w_str)
668
- if w_b == 0.0 and w_h == 0.0 and w_l == 0.0:
669
- continue
670
- data = self._load(name)
671
- if w_b != 0.0:
672
- cb = apply_model_lora(cb, data, w_b)
673
- if w_h != 0.0:
674
- ch = apply_model_lora(ch, data, w_h)
675
- if w_l != 0.0:
676
- cl = apply_model_lora(cl, data, w_l)
677
- used.append(name)
678
- trig = []
679
- trig_dbg = []
680
- for n in dict.fromkeys(used):
681
- found = get_triggers(n, cache)
682
- trig.extend(found)
683
- trig_dbg.append(f"{os.path.basename(n)}:{len(found)}")
684
- self._last_trig_dbg = trig_dbg
685
- trig_str = ", ".join(dict.fromkeys(t for t in trig if t))
686
- prompt_trig = trig_str if inject else ""
687
- return cb, ch, cl, prompt_trig, trig_str
688
-
689
- def direct(self, image, model_base, model_high, model_low, essential_actions, num_clips,
690
- character_name, artistic_vibe, pacing, detail_level, inject_triggers,
691
- use_vision, openrouter_api_key, vlm_preset, vlm_model, llm_preset, llm_model,
692
- creativity, seed, **kw):
693
-
694
- n_clips = int(str(num_clips).split()[0])
695
- vlm = resolve_slug(VLM_PRESETS, vlm_preset, vlm_model, "qwen/qwen2.5-vl-72b-instruct")
696
- llm = resolve_slug(LLM_PRESETS, llm_preset, llm_model, "deepseek/deepseek-v3.2")
697
- user_name = "" if character_name.strip().upper() in ["AUTO", ""] else character_name.strip()
698
- key = resolve_api_key(openrouter_api_key)
699
-
700
- vision_caption = str(kw.get("vision_caption", "") or "").strip()
701
- anchors, dbg_v = "", ""
702
- if vision_caption:
703
- anchors = vision_caption
704
- dbg_v = "vision ok (local caption node)"
705
- print(f"👁 [Director] 로컬 캡션 입력 사용 (JoyCaption 등):\n{anchors[:300]}")
706
- elif not use_vision:
707
- dbg_v = "vision OFF (토글 확인)"
708
- print("ℹ️ [Director] use_vision=OFF - 비전 건너뜀")
709
- elif not key:
710
- dbg_v = "no api key"
711
- print("❌ [Director] openrouter_api_key 비어있음 (위젯도, OPENROUTER_API_KEY 환경변수도 없음) - 비전/LLM 불가")
712
- elif image is None:
713
- dbg_v = "no image"
714
- print("❌ [Director] image 입력 없음 - IMAGE 연결 확인")
715
- else:
716
- data_uri = tensor_to_base64(image)
717
- if data_uri:
718
- anchors, err = self._vision_analyze(data_uri, vlm, key, min(creativity, 0.5))
719
- dbg_v = err or "vision ok"
720
- else:
721
- dbg_v = "image encode failed / PIL missing"
722
- if not vision_caption and use_vision and key and image is not None and not anchors:
723
- print(f"⚠️ [Director] 비전 앵커가 비었음 -> 이름/씬 근거 없음. 원인: {dbg_v}")
724
-
725
- if key:
726
- parsed, dbg_p = self._plan_shots(anchors, essential_actions, character_name,
727
- artistic_vibe, pacing, self._detail_hint(detail_level),
728
- llm, key, creativity, seed, num_clips=n_clips)
729
- if parsed:
730
- clips = [str(parsed[f"clip_{i}"]).strip() for i in range(1, n_clips + 1)]
731
- master = str(parsed.get("master_scene", "")).strip()
732
- cand = str(parsed.get("character", "")).strip()
733
- if not cand or cand.upper() == "AUTO":
734
- cand = self._fallback_name(anchors)
735
- final_name = user_name or cand
736
- else:
737
- clips = self._local_split(essential_actions, num_clips=n_clips)
738
- master = anchors.replace("\n", " ")[:160]
739
- final_name = user_name or self._fallback_name(anchors)
740
- else:
741
- parsed, dbg_p = None, "no api key"
742
- clips = self._local_split(essential_actions, num_clips=n_clips)
743
- master = anchors.replace("\n", " ")[:160]
744
- final_name = user_name
745
-
746
- if len(clips) < 4:
747
- clips = clips + [clips[-1]] * (4 - len(clips))
748
-
749
- cache = _load_cache()
750
- manual_list = [t.strip() for t in kw.get("manual_triggers", "").split(',') if t.strip()]
751
- outs = []
752
- all_trigs_collected = []
753
- for c in range(1, 5):
754
- if c > n_clips:
755
- cb = ch = cl = ExecutionBlocker(None)
756
- prompt = self._build_final(clips[c - 1], master, final_name, "")
757
- outs.extend([cb, ch, cl, prompt])
758
- print(f" [clip{c}] SKIPPED (num_clips={n_clips}) - ExecutionBlocker 반환")
759
- continue
760
- slots = [(kw.get(f"c{c}_lora_{n}", "None"),
761
- kw.get(f"c{c}_w_{n}", "1.0, 1.0, 1.0")) for n in range(1, 5)]
762
- cb, ch, cl, prompt_trig, always_trig = self._patch_clip(
763
- model_base, model_high, model_low, slots, cache, inject_triggers)
764
- all_trig = ", ".join(dict.fromkeys(
765
- ([prompt_trig] if prompt_trig else []) + manual_list)) \
766
- if (prompt_trig or manual_list) else ""
767
- if always_trig:
768
- all_trigs_collected.extend(t.strip() for t in always_trig.split(',') if t.strip())
769
- prompt = self._build_final(clips[c - 1], master, final_name, all_trig)
770
- outs.extend([cb, ch, cl, prompt])
771
- print(f" [clip{c}] loras/triggers: {getattr(self, '_last_trig_dbg', [])} "
772
- f"inject={inject_triggers}")
773
-
774
- triggers_out = ", ".join(dict.fromkeys(
775
- [t for t in all_trigs_collected if t] + manual_list))
776
-
777
- debug = f"vlm={dbg_v} | plan={dbg_p} | char={final_name} | clips={n_clips} ({n_clips*5}s)"
778
- print(f"\n🎬 [Dolphin Vision Director] {debug}\nANCHORS: {anchors[:120]}")
779
- for i in range(4):
780
- print(f" CLIP {i+1}: {outs[i*4+3][:90]}")
781
-
782
- is_final = tuple(c == n_clips for c in range(1, 5))
783
-
784
- return tuple(outs) + (triggers_out, anchors, debug) + is_final
785
-
786
-
787
- NODE_CLASS_MAPPINGS = {"DolphinVisionDirector": DolphinVisionDirector}
788
- NODE_DISPLAY_NAME_MAPPINGS = {"DolphinVisionDirector": "🎬👁 Dolphin Vision Director (Image→4Clips)"}