MogensR commited on
Commit
ba76682
·
verified ·
1 Parent(s): 2ada5b3

Update processing/two_stage/two_stage_processor.py

Browse files
processing/two_stage/two_stage_processor.py CHANGED
@@ -1,18 +1,10 @@
1
  #!/usr/bin/env python3
2
  """
3
- Two-Stage Green-Screen Processing System ✅ 2025-08-29
4
  FIXED: Now uses enhanced SAM2Handler with full-body detection strategies
5
  Stage 1: Original → keyed background (auto-selected colour)
6
  Stage 2: Keyed video → final composite (hybrid chroma + segmentation rescue)
7
- UPDATED: Enhanced quality profiles, improved frame handling, better status reporting
8
- - New: Integrate QualityManager for separated logic
9
- - New: Return green screen path for monitoring
10
- - Fix: Force green key color
11
- - Fix: Use RobustVideoWriter to prevent frame loss
12
- - Fix: Add logging for background preparation issue
13
- - COMPOSITING FIX: Normalize frame and background scales to prevent dark backgrounds
14
- - MAJOR FIX: Use enhanced SAM2Handler instead of basic segment_person_hq
15
- - TESTING FIX: Disabled double mask cleaning to eliminate artifacts
16
  """
17
  from __future__ import annotations
18
  import cv2, numpy as np, os, gc, pickle, logging, tempfile, traceback, threading
@@ -30,797 +22,493 @@
30
  # ---------------------------------------------------------------------------
31
  # Local video-writer helper with frame count guarantee
32
  # ---------------------------------------------------------------------------
33
- def create_video_writer(output_path: str, fps: float, width: int, height: int, prefer_mp4: bool = True):
34
- """Creates video writer with fallback options"""
35
- try:
36
- ext = ".mp4" if prefer_mp4 else ".avi"
37
- if not output_path:
38
- output_path = tempfile.mktemp(suffix=ext)
39
- else:
40
- base, curr_ext = os.path.splitext(output_path)
41
- if curr_ext.lower() not in [".mp4", ".avi", ".mov", ".mkv"]:
42
- output_path = base + ext
43
- fourcc = cv2.VideoWriter_fourcc(*("mp4v" if prefer_mp4 else "XVID"))
44
- writer = cv2.VideoWriter(output_path, fourcc, float(fps), (int(width), int(height)))
45
- if writer is None or not writer.isOpened():
46
- alt_ext = ".avi" if prefer_mp4 else ".mp4"
47
- alt_fourcc = cv2.VideoWriter_fourcc(*("XVID" if prefer_mp4 else "mp4v"))
48
- alt_path = os.path.splitext(output_path)[0] + alt_ext
49
- writer = cv2.VideoWriter(alt_path, alt_fourcc, float(fps), (int(width), int(height)))
50
- if writer is None or not writer.isOpened():
51
- return None, output_path
52
- return writer, alt_path
53
- return writer, output_path
54
- except Exception as e:
55
- logger.error(f"create_video_writer failed: {e}")
56
- return None, output_path
57
-
58
- # ---------------------------------------------------------------------------
59
- # Robust video writer wrapper to prevent frame loss
60
- # ---------------------------------------------------------------------------
61
- class RobustVideoWriter:
62
- """Wrapper that ensures all frames are written"""
63
-
64
- def __init__(self, writer, output_path: str):
65
- self.writer = writer
66
- self.output_path = output_path
67
- self.frame_buffer = []
68
- self.frames_written = 0
69
- self.frames_attempted = 0
70
-
71
- def write(self, frame):
72
- """Buffer and write frame"""
73
- if frame is None:
74
- return False
75
-
76
- self.frames_attempted += 1
77
- self.frame_buffer.append(frame.copy())
78
-
79
- # Try to write buffered frames
80
- while self.frame_buffer and self.writer:
81
- try:
82
- self.writer.write(self.frame_buffer[0])
83
- self.frame_buffer.pop(0)
84
- self.frames_written += 1
85
- except Exception as e:
86
- logger.warning(f"Frame write failed: {e}")
87
- return False
88
- return True
89
-
90
- def release(self):
91
- """Flush remaining frames and close"""
92
- # Write any remaining buffered frames
93
- while self.frame_buffer and self.writer:
94
- try:
95
- self.writer.write(self.frame_buffer[0])
96
- self.frame_buffer.pop(0)
97
- self.frames_written += 1
98
- except Exception:
99
- break
100
-
101
- # Duplicate last frame 3 times to force flush
102
- if self.frames_written > 0 and self.writer:
103
- last_frame = self.frame_buffer[-1] if self.frame_buffer else None
104
- if last_frame is None:
105
- # Read last written frame if needed
106
- try:
107
- cap = cv2.VideoCapture(self.output_path)
108
- cap.set(cv2.CAP_PROP_POS_FRAMES, self.frames_written - 1)
109
- _, last_frame = cap.read()
110
- cap.release()
111
- except Exception:
112
- pass
113
- if last_frame is not None:
114
- for _ in range(3):
115
- self.writer.write(last_frame)
116
- self.frames_written += 1
117
-
118
- # Close writer
119
- if self.writer:
120
- self.writer.release()
121
-
122
- # Log statistics
123
- logger.info(f"Video writer closed: {self.frames_written}/{self.frames_attempted} frames written")
124
-
125
- # Verify output exists
126
- if os.path.exists(self.output_path):
127
- size = os.path.getsize(self.output_path)
128
- if size == 0:
129
- logger.error("WARNING: Output file is empty!")
130
- else:
131
- logger.info(f"Output file size: {size:,} bytes")
132
-
133
- # ---------------------------------------------------------------------------
134
- # Key-colour helpers (fast, no external deps)
135
- # ---------------------------------------------------------------------------
136
- def _bgr_to_hsv_hue_deg(bgr: np.ndarray) -> np.ndarray:
137
- hsv = cv2.cvtColor(bgr, cv2.COLOR_BGR2HSV)
138
- # OpenCV H is 0-180; scale to degrees 0-360
139
- return hsv[..., 0].astype(np.float32) * 2.0
140
-
141
- def _hue_distance(a_deg: float, b_deg: float) -> float:
142
- """Circular distance on the hue wheel (degrees)."""
143
- d = abs(a_deg - b_deg) % 360.0
144
- return min(d, 360.0 - d)
145
-
146
- def _key_candidates_bgr() -> dict:
147
- return {
148
- "green": {"bgr": np.array([ 0,255, 0], dtype=np.uint8), "hue": 120.0},
149
- "blue": {"bgr": np.array([255, 0, 0], dtype=np.uint8), "hue": 240.0},
150
- "cyan": {"bgr": np.array([255,255, 0], dtype=np.uint8), "hue": 180.0},
151
- "magenta": {"bgr": np.array([255, 0,255], dtype=np.uint8), "hue": 300.0},
152
- }
153
-
154
- def _choose_best_key_color(frame_bgr: np.ndarray, mask_uint8: np.ndarray) -> dict:
155
- """Pick the candidate colour farthest from the actor's dominant hues."""
156
- try:
157
- fg = frame_bgr[mask_uint8 > 127]
158
- if fg.size < 1_000:
159
- return _key_candidates_bgr()["green"]
160
- fg_hue = _bgr_to_hsv_hue_deg(fg.reshape(-1, 1, 3)).reshape(-1)
161
- hist, edges = np.histogram(fg_hue, bins=36, range=(0.0, 360.0))
162
- top_idx = np.argsort(hist)[-3:]
163
- top_hues = [(edges[i] + edges[i+1]) * 0.5 for i in top_idx]
164
- best_name, best_score = None, -1.0
165
- for name, info in _key_candidates_bgr().items():
166
- cand_hue = info["hue"]
167
- score = min(abs((cand_hue - th + 180) % 360 - 180) for th in top_hues)
168
- if score > best_score:
169
- best_name, best_score = name, score
170
- return _key_candidates_bgr().get(best_name, _key_candidates_bgr()["green"])
171
- except Exception:
172
- return _key_candidates_bgr()["green"]
173
-
174
- # ---------------------------------------------------------------------------
175
- # Chroma presets
176
- # ---------------------------------------------------------------------------
177
- CHROMA_PRESETS: Dict[str, Dict[str, Any]] = {
178
- 'standard': {'key_color': [0,255,0], 'tolerance': 38, 'edge_softness': 2, 'spill_suppression': 0.35},
179
- 'studio': {'key_color': [0,255,0], 'tolerance': 30, 'edge_softness': 1, 'spill_suppression': 0.45},
180
- 'outdoor': {'key_color': [0,255,0], 'tolerance': 50, 'edge_softness': 3, 'spill_suppression': 0.25},
181
- }
182
 
183
  # ---------------------------------------------------------------------------
184
- # Two-Stage Processor - TESTING: NO DOUBLE CLEANING
185
  # ---------------------------------------------------------------------------
186
  class TwoStageProcessor:
187
- def __init__(self, sam2_predictor=None, matanyone_model=None):
188
- # CRITICAL FIX: Store the ENHANCED SAM2Handler, not just the predictor
189
- self.sam2_handler = sam2_predictor # This should be your enhanced SAM2Handler instance
190
- self.sam2 = self._unwrap_sam2(sam2_predictor) # Keep original logic for compatibility
191
- self.matanyone = matanyone_model
192
- self.mask_cache_dir = Path("/tmp/mask_cache")
193
- self.mask_cache_dir.mkdir(parents=True, exist_ok=True)
194
- # Internal flags/state
195
- self._mat_bootstrapped = False
196
- self._alpha_prev: Optional[np.ndarray] = None # temporal smoothing
197
-
198
- # Frame tracking
199
- self.total_frames_processed = 0
200
- self.frames_refined = 0
201
-
202
- # Load quality profile
203
- self.q = quality_manager.get_params()
204
- logger.info(f"TwoStageProcessor init – SAM2: {self.sam2 is not None} | MatAnyOne: {self.matanyone is not None}")
205
-
206
- # --------------------------- internal utils ---------------------------
207
- def _unwrap_sam2(self, predictor):
208
- """Unwrap the SAM2 predictor if needed."""
209
- if predictor is None:
210
- return None
211
- if hasattr(predictor, 'predictor'): # SAM2Handler has .predictor attribute
212
- return predictor.predictor
213
- if hasattr(predictor, 'sam_predictor'):
214
- return predictor.sam_predictor
215
- return predictor
216
-
217
- def _get_mask(self, frame: np.ndarray) -> np.ndarray:
218
- """TESTING: Disable double cleaning to isolate mask quality issues."""
219
- logger.info("=== TwoStageProcessor _get_mask called (TESTING - NO DOUBLE CLEANING) ===")
220
 
221
- if self.sam2_handler is None:
222
- logger.warning("No SAM2Handler available - using fallback threshold")
223
- gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
224
- _, mask = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY)
225
- return mask
 
 
 
226
 
227
  try:
 
228
  if hasattr(self.sam2_handler, 'create_mask'):
229
- logger.info("Using SAM2Handler.create_mask() WITHOUT additional cleaning")
230
- frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
 
231
  mask = self.sam2_handler.create_mask(frame_rgb)
232
 
233
- if mask is not None:
234
- # DISABLED: No additional cleaning - let SAM2Handler's cleaning stand
235
- # mask = self._maximum_quality_mask_cleaning(mask) # DISABLED FOR TESTING
236
- logger.info(f"SAM2 mask WITHOUT additional cleaning - coverage: {np.mean(mask/255.0):.3f}")
237
- return mask
238
- else:
239
- logger.warning("SAM2Handler returned None mask")
240
  else:
241
- logger.warning("SAM2Handler doesn't have create_mask method")
242
-
243
- # Fallback to basic SAM2 if enhanced handler fails
244
- if self.sam2 is not None:
245
- logger.info("Fallback: Using basic SAM2 predictor")
246
- try:
247
- from utils.cv_processing import segment_person_hq
248
- mask = segment_person_hq(frame, self.sam2)
249
- # DISABLED: No additional cleaning in fallback either
250
- # mask = self._maximum_quality_mask_cleaning(mask) # DISABLED FOR TESTING
251
- return mask
252
- except ImportError:
253
- logger.warning("Could not import segment_person_hq")
254
- except Exception as e:
255
- logger.warning(f"Basic SAM2 segmentation failed: {e}")
256
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
257
  except Exception as e:
258
- logger.error(f"SAM2 segmentation failed: {e}")
259
-
260
- # Final fallback
261
- logger.warning("All SAM2 methods failed - using simple threshold")
262
- gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
263
- _, mask = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY)
264
- return mask
265
-
266
- def _maximum_quality_mask_cleaning(self, mask: np.ndarray) -> np.ndarray:
267
- """Maximum quality mask cleaning and refinement - CURRENTLY DISABLED."""
268
- # THIS METHOD IS DISABLED TO PREVENT DOUBLE CLEANING
269
- # The SAM2Handler already applies its own cleaning
270
- logger.info("Maximum quality mask cleaning DISABLED to prevent double processing")
271
- return mask
272
-
273
- # ORIGINAL CODE COMMENTED OUT:
274
- # try:
275
- # # Ensure uint8 format
276
- # if mask.max() <= 1.0:
277
- # mask_uint8 = (mask * 255).astype(np.uint8)
278
- # else:
279
- # mask_uint8 = mask.astype(np.uint8)
280
- #
281
- # # Step 1: Fill small holes aggressively
282
- # kernel_fill = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9, 9))
283
- # mask_filled = cv2.morphologyEx(mask_uint8, cv2.MORPH_CLOSE, kernel_fill)
284
- #
285
- # # Step 2: Connect nearby regions
286
- # kernel_connect = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7, 7))
287
- # mask_connected = cv2.morphologyEx(mask_filled, cv2.MORPH_CLOSE, kernel_connect)
288
- #
289
- # # Step 3: Smooth boundaries heavily
290
- # mask_smooth1 = cv2.GaussianBlur(mask_connected, (7, 7), 2.0)
291
- #
292
- # # Step 4: Re-threshold to crisp edges
293
- # _, mask_thresh = cv2.threshold(mask_smooth1, 127, 255, cv2.THRESH_BINARY)
294
- #
295
- # # Step 5: Final edge smoothing
296
- # mask_final = cv2.GaussianBlur(mask_thresh, (5, 5), 1.0)
297
- #
298
- # # Step 6: Dilate slightly to ensure full coverage
299
- # kernel_dilate = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
300
- # mask_dilated = cv2.dilate(mask_final, kernel_dilate, iterations=1)
301
- #
302
- # logger.info("Maximum quality mask cleaning applied successfully")
303
- # return mask_dilated
304
- #
305
- # except Exception as e:
306
- # logger.warning(f"Maximum quality mask cleaning failed: {e}")
307
- # return mask
308
-
309
- @staticmethod
310
- def _to_binary_mask(mask: np.ndarray) -> Optional[np.ndarray]:
311
- """Convert mask to uint8(0..255)."""
312
- if mask is None:
313
  return None
314
- if mask.dtype == bool:
315
- return mask.astype(np.uint8) * 255
316
- if np.issubdtype(mask.dtype, np.floating):
317
- m = np.clip(mask, 0.0, 1.0)
318
- return (m * 255.0 + 0.5).astype(np.uint8)
319
- return mask
320
 
321
- @staticmethod
322
- def _to_float01(mask: np.ndarray, h: int = None, w: int = None) -> Optional[np.ndarray]:
323
- """Float [0,1] mask, optionally resized to (h,w)."""
324
- if mask is None:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
325
  return None
326
- m = mask.astype(np.float32)
327
- if m.max() > 1.0:
328
- m = m / 255.0
329
- if h is not None and w is not None and (m.shape[0] != h or m.shape[1] != w):
330
- m = cv2.resize(m, (w, h), interpolation=cv2.INTER_LINEAR)
331
- return np.clip(m, 0.0, 1.0)
332
-
333
- @staticmethod
334
- def _apply_greenscreen_hard(frame: np.ndarray, mask: np.ndarray, bg: np.ndarray) -> np.ndarray:
335
- """Apply hard greenscreen compositing with FIXED scale normalization."""
336
- mask_3ch = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR) if mask.ndim == 2 else mask
337
- mask_norm = mask_3ch.astype(np.float32) / 255.0
338
- # COMPOSITING FIX: Normalize both frame and bg to same [0,1] scale
339
- frame_norm = frame.astype(np.float32) / 255.0
340
- bg_norm = bg.astype(np.float32) / 255.0
341
- result = frame_norm * mask_norm + bg_norm * (1 - mask_norm)
342
- return (result * 255.0).astype(np.uint8)
343
-
344
- # -------- improved spill suppression (preserves luminance & skin) --------
345
- def _suppress_green_spill(self, frame: np.ndarray, amount: float = 0.35) -> np.ndarray:
346
- """
347
- Desaturate green dominance while preserving luminance and red skin hues.
348
- amount: 0..1
349
- """
350
- b, g, r = cv2.split(frame.astype(np.float32))
351
- green_dom = (g > r) & (g > b)
352
- avg_rb = (r + b) * 0.5
353
- g2 = np.where(green_dom, g*(1.0-amount) + avg_rb*amount, g)
354
- skin = (r > g + 12) # protect skin tones
355
- g2 = np.where(skin, g, g2)
356
- out = cv2.merge([np.clip(b,0,255), np.clip(g2,0,255), np.clip(r,0,255)]).astype(np.uint8)
357
- return out
358
-
359
- # -------- edge-aware alpha refinement (guided-like) --------
360
- def _refine_alpha_edges(self, frame_bgr: np.ndarray, alpha_u8: np.ndarray, radius: int = 3, iters: int = 1) -> np.ndarray:
361
- """
362
- Fast, dependency-free, guided-like refinement on the alpha border.
363
- Returns: uint8 alpha
364
- """
365
- a = alpha_u8.astype(np.uint8)
366
- if radius <= 0:
367
- return a
368
- band = cv2.Canny(a, 32, 64)
369
- if band.max() == 0:
370
- return a
371
- for _ in range(max(1, iters)):
372
- a_blur = cv2.GaussianBlur(a, (radius*2+1, radius*2+1), 0)
373
- b,g,r = cv2.split(frame_bgr.astype(np.float32))
374
- green_dom = (g > r) & (g > b)
375
- spill_mask = (green_dom & (a > 96) & (a < 224)).astype(np.uint8)*255
376
- u = cv2.bitwise_or(band, spill_mask)
377
- a = np.where(u>0, a_blur, a).astype(np.uint8)
378
- return a
379
 
380
- # -------- soft key based on chosen color (robust to blue/cyan/magenta) --------
381
- def _soft_key_mask(self, frame_bgr: np.ndarray, key_bgr: np.ndarray, tol: int = 40) -> np.ndarray:
382
  """
383
- Soft chroma mask (uint8 0..255, 255=keep subject) using CbCr distance.
 
384
  """
385
- if key_bgr is None:
386
- return np.full(frame_bgr.shape[:2], 255, np.uint8)
387
- ycbcr = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2YCrCb).astype(np.float32)
388
- kycbcr = cv2.cvtColor(key_bgr.reshape(1,1,3).astype(np.uint8), cv2.COLOR_BGR2YCrCb).astype(np.float32)[0,0]
389
- d = np.linalg.norm((ycbcr[...,1:] - kycbcr[1:]), axis=-1)
390
- d = cv2.GaussianBlur(d, (5,5), 0)
391
- alpha = 255.0 * np.clip((d - tol) / (tol*1.7), 0.0, 1.0) # far from key = keep (255)
392
- return alpha.astype(np.uint8)
393
 
394
- # --------------------- MatAnyone bootstrap ----------------------
395
- def _bootstrap_matanyone_if_needed(self, frame_bgr: np.ndarray, coarse_mask: np.ndarray):
 
 
 
 
396
  """
397
- Call the MatAnyone session ONCE with the first coarse mask to initialize
398
- its memory. This guarantees downstream calls never hit "first frame without a mask".
399
  """
400
- if self.matanyone is None or self._mat_bootstrapped:
401
- return
402
  try:
403
- h, w = frame_bgr.shape[:2]
404
- mask_f = self._to_float01(coarse_mask, h, w)
405
- rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
406
- _ = self.matanyone(rgb, mask_f) # boot only; ignore returned returned alpha
407
- self._mat_bootstrapped = True
408
- logger.info("MatAnyone session bootstrapped with first-frame mask.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
409
  except Exception as e:
410
- logger.warning(f"MatAnyone bootstrap failed (continuing without): {e}")
411
-
412
- def _should_refine_frame(self, frame_idx: int) -> bool:
413
- """Check if current frame should be refined based on quality profile"""
414
- if not self.matanyone:
415
- return False
416
-
417
- # Always refine first frame for bootstrap
418
- if frame_idx == 0:
419
- return True
420
-
421
- return quality_manager.should_refine_frame(frame_idx)
422
 
423
- # ---------------------------------------------------------------------
424
- # Stage 1 – Original → keyed (green/blue/…) -- chooses colour on 1st frame
425
- # ---------------------------------------------------------------------
426
- def stage1_extract_to_greenscreen(
427
- self,
428
- video_path: str,
429
- output_path: str,
430
- *,
431
- key_color_mode: str = "auto", # "auto" | "green" | "blue" | "cyan" | "magenta"
432
- progress_callback: Optional[Callable[[float, str], None]] = None,
433
- stop_event: Optional[threading.Event] = None,
434
- ) -> Tuple[Optional[dict], str]:
435
- def _prog(p, d):
436
- if progress_callback:
437
- try:
438
- progress_callback(float(p), str(d))
439
- except Exception:
440
- pass
441
  try:
442
- # pick up any new quality selection
443
- quality_manager.load_profile() # Refresh
444
- self.q = quality_manager.get_params()
445
- _prog(0.0, "Stage 1: opening video…")
446
- cap = cv2.VideoCapture(video_path)
447
  if not cap.isOpened():
448
- return None, "Could not open input video"
449
- fps = cap.get(cv2.CAP_PROP_FPS) or 25.0
450
- total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) or 0
451
- w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
452
- h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
453
- base_writer, out_path = create_video_writer(output_path, fps, w, h)
454
- if base_writer is None:
455
- cap.release()
456
- return None, "Could not create output writer"
457
-
458
- # Use robust wrapper
459
- writer = RobustVideoWriter(base_writer, out_path)
460
- key_info: dict | None = None
461
- chosen_bgr = np.array([0, 255, 0], np.uint8) # Force green
462
- probe_done = False
463
- masks: List[np.ndarray] = []
464
- frame_idx = 0
465
- self.frames_refined = 0
466
- solid_bg = np.zeros((h, w, 3), np.uint8) # overwritten per-frame
467
-
468
- while True:
469
- if stop_event and stop_event.is_set():
470
- _prog(1.0, "Stage 1: cancelled")
471
- break
472
- ok, frame = cap.read()
473
- if not ok:
 
 
 
474
  break
475
-
476
- # --- SAM2 segmentation WITHOUT double cleaning ---
477
- logger.info(f"Processing frame {frame_idx} with SAM2 (no double cleaning)")
478
- mask = self._get_mask(frame)
479
-
480
- # --- MatAnyone bootstrap exactly once (first frame) ---
481
- if frame_idx == 0 and self.matanyone is not None:
482
- try:
483
- self._bootstrap_matanyone_if_needed(frame, mask)
484
- except Exception as e:
485
- logger.warning(f"Bootstrap error (non-fatal): {e}")
486
-
487
- # --- Decide key colour once ---
488
- if not probe_done:
489
- if key_color_mode.lower() == "auto":
490
- key_info = _choose_best_key_color(frame, self._to_binary_mask(mask))
491
- chosen_bgr = key_info["bgr"]
 
 
 
492
  else:
493
- cand = _key_candidates_bgr().get(key_color_mode.lower())
494
- if cand is not None:
495
- chosen_bgr = cand["bgr"]
496
- probe_done = True
497
- logger.info(f"[TwoStage] Using key colour: {key_color_mode} → {chosen_bgr.tolist()}")
498
-
499
- # --- Optional refinement via MatAnyone (profile cadence) ---
500
- if self._should_refine_frame(frame_idx):
501
- try:
502
- # Use proper refine function if available
503
- try:
504
- from utils.cv_processing import refine_mask_hq
505
- mask = refine_mask_hq(frame, mask, self.matanyone, fallback_enabled=True)
506
- except ImportError:
507
- logger.warning("refine_mask_hq not available - skipping refinement")
508
- self.frames_refined += 1
509
- logger.debug(f"Frame {frame_idx}: Refined (quality={quality_manager.profile_name})")
510
- except Exception as e:
511
- logger.warning(f"MatAnyOne refine fail f={frame_idx}: {e}")
512
  else:
513
- logger.debug(f"Frame {frame_idx}: Skipped refinement (cadence={self.q['refine_cadence']})")
514
-
515
- # --- Composite onto solid key colour ---
516
- solid_bg[:] = chosen_bgr
517
- mask_u8 = self._to_binary_mask(mask)
518
- gs = self._apply_greenscreen_hard(frame, mask_u8, solid_bg)
519
- writer.write(gs)
520
- masks.append(mask_u8)
521
- frame_idx += 1
522
- pct = 0.05 + 0.9 * (frame_idx / total) if total else min(0.95, 0.05 + frame_idx * 0.002)
523
- _prog(pct, f"Stage 1: {frame_idx}/{total or '?'} (refined: {self.frames_refined})")
524
-
 
525
  cap.release()
526
  writer.release()
527
- self.total_frames_processed = frame_idx
528
-
529
- # save mask cache
530
- try:
531
- cache_file = self.mask_cache_dir / (Path(out_path).stem + "_masks.pkl")
532
- with open(cache_file, "wb") as f:
533
- pickle.dump(masks, f)
534
- logger.info(f"Cached {len(masks)} masks to {cache_file}")
535
- except Exception as e:
536
- logger.warning(f"mask cache save fail: {e}")
537
-
538
- _prog(1.0, "Stage 1: complete")
539
-
540
- # Log quality impact
541
- logger.info(f"Stage 1 complete: {frame_idx} frames, {self.frames_refined} refined "
542
- f"({100*self.frames_refined/max(1,frame_idx):.1f}%)")
543
-
544
- return (
545
- {"path": out_path, "frames": frame_idx, "key_bgr": chosen_bgr.tolist()},
546
- f"Green-screen video created ({frame_idx} frames, {self.frames_refined} refined)"
547
- )
548
  except Exception as e:
549
- logger.error(f"Stage 1 error: {e}\n{traceback.format_exc()}")
550
- return None, f"Stage 1 failed: {e}"
 
551
 
552
- # ---------------------------------------------------------------------
553
- # Stage 2 – keyed video → final composite (hybrid matte)
554
- # ---------------------------------------------------------------------
555
- def stage2_greenscreen_to_final(
556
- self,
557
- gs_path: str,
558
- background: np.ndarray | str,
559
- output_path: str,
560
- *,
561
- chroma_settings: Optional[Dict[str, Any]] = None,
562
- progress_callback: Optional[Callable[[float, str], None]] = None,
563
- stop_event: Optional[threading.Event] = None,
564
- key_bgr: Optional[np.ndarray] = None, # pass chosen key color
565
- ) -> Tuple[Optional[str], str]:
566
- def _prog(p, d):
567
- if progress_callback:
568
- try:
569
- progress_callback(float(p), str(d))
570
- except Exception:
571
- pass
572
  try:
573
- # pick up any new quality selection
574
- quality_manager.load_profile()
575
- self.q = quality_manager.get_params()
576
- _prog(0.0, "Stage 2: opening keyed video…")
577
- cap = cv2.VideoCapture(gs_path)
 
 
578
  if not cap.isOpened():
579
- return None, "Could not open keyed video"
580
- fps = cap.get(cv2.CAP_PROP_FPS) or 25.0
581
- total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) or 0
582
- w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
583
- h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
584
-
585
- # Load or prepare background
586
- if isinstance(background, str):
587
- bg = cv2.imread(background)
588
- if bg is None:
589
- logger.error(f"Failed to load background image from {background}")
590
- cap.release()
591
- return None, "Could not load background image - check file path and permissions"
592
- bg = cv2.resize(bg, (w, h))
593
- else:
594
- bg = cv2.resize(background, (w, h))
595
-
596
- # Optional tiny BG blur per profile to hide seams on flat BGs
597
- sigma = float(self.q['bg_blur_sigma'])
598
- if sigma > 0:
599
- bg = cv2.GaussianBlur(bg, (0, 0), sigmaX=sigma, sigmaY=sigma)
600
- logger.debug(f"Applied background blur: sigma={sigma:.1f}")
601
-
602
- base_writer, out_path = create_video_writer(output_path, fps, w, h)
603
- if base_writer is None:
604
- cap.release()
605
- return None, "Could not create output writer"
606
-
607
- # Use robust wrapper
608
- writer = RobustVideoWriter(base_writer, out_path)
609
-
610
- # Load cached masks if available
611
- masks = None
612
- try:
613
- cache_file = self.mask_cache_dir / (Path(gs_path).stem + "_masks.pkl")
614
- if cache_file.exists():
615
- with open(cache_file, "rb") as f:
616
- masks = pickle.load(f)
617
- logger.info(f"Loaded {len(masks)} cached masks")
618
- except Exception as e:
619
- logger.warning(f"Could not load mask cache: {e}")
620
-
621
- # Get chroma settings and override with profile
622
- settings = chroma_settings or CHROMA_PRESETS.get('standard', {})
623
- tolerance = int(self.q['chroma_tolerance'])
624
- edge_softness = int(self.q['chroma_softness'])
625
- spill_suppression = float(self.q['spill_suppression'])
626
-
627
- # If caller didn't pass key_bgr, try preset or default green
628
- if key_bgr is None:
629
- key_bgr = np.array(settings.get('key_color', [0,255,0]), dtype=np.uint8)
630
-
631
- self._alpha_prev = None # reset temporal smoothing per render
632
- frame_idx = 0
633
-
634
- while True:
635
- if stop_event and stop_event.is_set():
636
- _prog(1.0, "Stage 2: cancelled")
637
- break
638
- ok, frame = cap.read()
639
- if not ok:
640
  break
641
-
642
- # Apply chroma keying with optional mask assistance
643
- if masks and frame_idx < len(masks):
644
- mask = masks[frame_idx]
645
- final_frame = self._hybrid_composite(
646
- frame, bg, mask,
647
- tolerance=tolerance,
648
- edge_softness=edge_softness,
649
- spill_suppression=spill_suppression,
650
- key_bgr=key_bgr
651
- )
652
- else:
653
- # Pure chroma key
654
- final_frame = self._chroma_key_composite(
655
- frame, bg,
656
- tolerance=tolerance,
657
- edge_softness=edge_softness,
658
- spill_suppression=spill_suppression,
659
- key_bgr=key_bgr
660
- )
661
-
662
- writer.write(final_frame)
663
- frame_idx += 1
664
- pct = 0.05 + 0.9 * (frame_idx / total) if total else min(0.95, 0.05 + frame_idx * 0.002)
665
- _prog(pct, f"Stage 2: {frame_idx}/{total or '?'}")
666
-
667
  cap.release()
668
  writer.release()
669
- _prog(1.0, "Stage 2: complete")
670
-
671
- # Verify frame counts match
672
- if total > 0 and frame_idx != total:
673
- logger.warning(f"Frame count mismatch: processed {frame_idx}, expected {total}")
674
-
675
- return out_path, f"Final composite created ({frame_idx} frames)"
 
676
  except Exception as e:
677
- logger.error(f"Stage 2 error: {e}\n{traceback.format_exc()}")
678
- return None, f"Stage 2 failed: {e}"
679
-
680
- # ---------------- chroma + hybrid compositors (polished) ----------------
681
- def _chroma_key_composite(self, frame, bg, *, tolerance=38, edge_softness=2, spill_suppression=0.35, key_bgr: Optional[np.ndarray] = None):
682
- """Apply chroma key compositing with soft color distance + edge refinement."""
683
- # 1) spill first
684
- if spill_suppression > 0:
685
- frame = self._suppress_green_spill(frame, spill_suppression)
686
-
687
- # 2) build alpha
688
- if key_bgr is not None:
689
- alpha = self._soft_key_mask(frame, key_bgr, tol=int(tolerance))
690
- else:
691
- # Fallback: HSV green range
692
- hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
693
- lower_green = np.array([40, 40, 40])
694
- upper_green = np.array([80, 255, 255])
695
- alpha = cv2.bitwise_not(cv2.inRange(hsv, lower_green, upper_green))
696
-
697
- # 3) soft edges + refinement
698
- if edge_softness > 0:
699
- k = edge_softness * 2 + 1
700
- alpha = cv2.GaussianBlur(alpha, (k, k), 0)
701
- alpha = self._refine_alpha_edges(frame, alpha, radius=max(1, edge_softness), iters=1)
702
-
703
- # 4) temporal smoothing
704
- if self._alpha_prev is not None and self._alpha_prev.shape == alpha.shape:
705
- alpha = cv2.addWeighted(alpha, 0.75, self._alpha_prev, 0.25, 0)
706
- self._alpha_prev = alpha
707
-
708
- # 5) composite - COMPOSITING FIX: Normalize scales
709
- mask_3ch = cv2.cvtColor(alpha, cv2.COLOR_GRAY2BGR).astype(np.float32) / 255.0
710
- frame_norm = frame.astype(np.float32) / 255.0
711
- bg_norm = bg.astype(np.float32) / 255.0
712
- out = frame_norm * mask_3ch + bg_norm * (1.0 - mask_3ch)
713
- return np.clip(out * 255.0, 0, 255).astype(np.uint8)
714
-
715
- def _hybrid_composite(self, frame, bg, mask, *, tolerance=38, edge_softness=2, spill_suppression=0.35, key_bgr: Optional[np.ndarray] = None):
716
- """Apply hybrid compositing using both chroma key and cached mask, with profile controls."""
717
- chroma_result = self._chroma_key_composite(
718
- frame, bg,
719
- tolerance=tolerance,
720
- edge_softness=edge_softness,
721
- spill_suppression=spill_suppression,
722
- key_bgr=key_bgr
723
- )
724
-
725
- if mask is None:
726
- return chroma_result
727
-
728
- # profile-driven dilate/feather on cached mask to close pinholes + soften edges
729
- m = mask
730
- d = int(self.q['mask_dilate'])
731
- if d > 0:
732
- k = 2*d + 1
733
- se = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (k, k))
734
- m = cv2.dilate(m, se, iterations=1)
735
- b = int(self.q['mask_blur'])
736
- if b > 0:
737
- m = cv2.GaussianBlur(m, (2*b+1, 2*b+1), 0)
738
 
739
- m3 = cv2.cvtColor(m, cv2.COLOR_GRAY2BGR) if m.ndim == 2 else m
740
- m3f = (m3.astype(np.float32) / 255.0)
 
 
741
 
742
- # COMPOSITING FIX: Normalize scales
743
- frame_norm = frame.astype(np.float32) / 255.0
744
- bg_norm = bg.astype(np.float32) / 255.0
745
- seg_comp = frame_norm * m3f + bg_norm * (1.0 - m3f)
746
 
747
- mix = float(self.q['hybrid_mix']) # weight towards segmentation on "max"
748
- out = chroma_result.astype(np.float32) * (1.0 - mix) + (seg_comp * 255.0) * mix
749
- return np.clip(out, 0, 255).astype(np.uint8)
750
-
751
- # ---------------------------------------------------------------------
752
- # Combined pipeline
753
- # ---------------------------------------------------------------------
754
- def process_full_pipeline(
755
- self,
756
- video_path: str,
757
- background: np.ndarray | str,
758
- output_path: str,
759
- *,
760
- key_color_mode: str = "auto",
761
- chroma_settings: Optional[Dict[str, Any]] = None,
762
- progress_callback: Optional[Callable[[float, str], None]] = None,
763
- stop_event: Optional[threading.Event] = None,
764
- ) -> Tuple[Optional[str], Optional[str], str]:
765
- """Run both stages in sequence, return final, green, msg."""
766
- def _combined_progress(pct, desc):
767
- # Scale progress: Stage 1 is 0-50%, Stage 2 is 50-100%
768
- if "Stage 1" in desc:
769
- actual_pct = pct * 0.5
770
- else: # Stage 2
771
- actual_pct = 0.5 + pct * 0.5
772
- if progress_callback:
773
- try:
774
- progress_callback(actual_pct, desc)
775
- except Exception:
776
- pass
777
 
 
 
778
  try:
779
- # pick up any new new quality selection once per run
780
- quality_manager.load_profile()
781
- self.q = quality_manager.get_params()
782
-
783
- # Reset per-video state
784
- self._mat_bootstrapped = False
785
- self._alpha_prev = None
786
- self.total_frames_processed = 0
787
- self.frames_refined = 0
788
-
789
- if self.matanyone is not None and hasattr(self.matanyone, "reset"):
790
- try:
791
- self.matanyone.reset()
792
- except Exception:
793
- pass
794
-
795
- # Stage 1 - use persistent path for green screen
796
- green_path = os.path.splitext(output_path)[0] + '_green.mp4'
797
- stage1_result, stage1_msg = self.stage1_extract_to_greenscreen(
798
- video_path, green_path,
799
- key_color_mode=key_color_mode,
800
- progress_callback=_combined_progress,
801
- stop_event=stop_event
802
- )
803
 
804
- if stage1_result is None:
805
- return None, None, stage1_msg
 
 
 
 
 
 
 
 
 
 
 
 
 
 
806
 
807
- # Stage 2 (pass through chosen key color)
808
- key_bgr = np.array(stage1_result.get("key_bgr", [0,255,0]), dtype=np.uint8)
809
- final_path, stage2_msg = self.stage2_greenscreen_to_final(
810
- green_path, background, output_path,
811
- chroma_settings=chroma_settings,
812
- progress_callback=_combined_progress,
813
- stop_event=stop_event,
814
- key_bgr=key_bgr,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
815
  )
816
-
817
- # Report quality impact
818
- logger.info(f"Pipeline complete with quality='{quality_manager.profile_name}': "
819
- f"{self.total_frames_processed} frames, "
820
- f"{self.frames_refined} refined ({100*self.frames_refined/max(1,self.total_frames_processed):.1f}%)")
821
-
822
- return final_path, green_path, stage2_msg
823
-
 
 
 
 
 
 
 
 
 
 
 
 
824
  except Exception as e:
825
- logger.error(f"Full pipeline error: {e}\n{traceback.format_exc()}")
826
- return None, None, f"Pipeline failed: {e}"
 
 
1
  #!/usr/bin/env python3
2
  """
3
+ Two-Stage Green-Screen Processing System with Comprehensive Debug Outputs ✅ 2025-09-09
4
  FIXED: Now uses enhanced SAM2Handler with full-body detection strategies
5
  Stage 1: Original → keyed background (auto-selected colour)
6
  Stage 2: Keyed video → final composite (hybrid chroma + segmentation rescue)
7
+ DEBUG: Added extensive debug outputs to isolate quality degradation at every step
 
 
 
 
 
 
 
 
8
  """
9
  from __future__ import annotations
10
  import cv2, numpy as np, os, gc, pickle, logging, tempfile, traceback, threading
 
22
  # ---------------------------------------------------------------------------
23
  # Local video-writer helper with frame count guarantee
24
  # ---------------------------------------------------------------------------
25
+ def create_video_writer(output_path: str, fps: float, width: int, height: int, prefer_mp4: bool = True) -> cv2.VideoWriter:
26
+ """Create robust video writer with fallback codec support."""
27
+ codecs = [
28
+ ('mp4v', '.mp4') if prefer_mp4 else ('XVID', '.avi'),
29
+ ('XVID', '.avi') if prefer_mp4 else ('mp4v', '.mp4'),
30
+ ('MJPG', '.avi')
31
+ ]
32
+
33
+ for codec, ext in codecs:
34
+ try:
35
+ if not output_path.endswith(ext):
36
+ output_path = os.path.splitext(output_path)[0] + ext
37
+ fourcc = cv2.VideoWriter_fourcc(*codec)
38
+ writer = cv2.VideoWriter(output_path, fourcc, fps, (width, height))
39
+ if writer.isOpened():
40
+ logger.info(f"Created video writer: {codec} → {output_path}")
41
+ return writer
42
+ writer.release()
43
+ except Exception as e:
44
+ logger.warning(f"Failed to create writer with {codec}: {e}")
45
+
46
+ raise RuntimeError(f"Could not create video writer for {output_path}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
 
48
  # ---------------------------------------------------------------------------
49
+ # TwoStageProcessor: Main processing pipeline with debug outputs
50
  # ---------------------------------------------------------------------------
51
  class TwoStageProcessor:
52
+ """
53
+ Two-stage video background replacement with comprehensive debugging:
54
+ Stage 1: Person segmentation → Green screen video
55
+ Stage 2: Green screen video → Final composite
56
+ """
57
+
58
+ def __init__(self, sam2_handler, matanyone_processor=None):
59
+ """Initialize with enhanced SAM2Handler and optional MatAnyone."""
60
+ # CRITICAL: Ensure we receive the full SAM2Handler, not just predictor
61
+ if hasattr(sam2_handler, 'create_mask'):
62
+ self.sam2_handler = sam2_handler
63
+ logger.info("✅ Using ENHANCED SAM2Handler with create_mask() method")
64
+ elif hasattr(sam2_handler, 'predictor'):
65
+ # Fallback: wrap basic predictor
66
+ self.sam2_handler = BasicSAM2Wrapper(sam2_handler.predictor)
67
+ logger.warning("⚠️ Wrapping basic SAM2 predictor - enhanced features unavailable")
68
+ else:
69
+ # Last resort: assume it's the predictor directly
70
+ self.sam2_handler = BasicSAM2Wrapper(sam2_handler)
71
+ logger.warning("⚠️ Assuming SAM2 predictor - enhanced features unavailable")
72
+
73
+ self.matanyone = matanyone_processor
74
+ self.temp_dir = None
75
+ self._cleanup_list = []
 
 
 
 
 
 
 
 
 
76
 
77
+ # Quality settings
78
+ self.quality_profile = quality_manager.get_current_profile()
79
+ logger.info(f"Initialized TwoStageProcessor with quality profile: {self.quality_profile['name']}")
80
+
81
+ def _get_mask(self, frame_rgb, frame_index):
82
+ """Get person mask with extensive debug outputs at every step."""
83
+ logger.info(f"=== DEBUG: TwoStageProcessor._get_mask called for frame {frame_index} ===")
84
+ logger.info(f"DEBUG: Input frame shape: {frame_rgb.shape}, dtype: {frame_rgb.dtype}")
85
 
86
  try:
87
+ # Check what type of SAM2 handler we have
88
  if hasattr(self.sam2_handler, 'create_mask'):
89
+ logger.info("DEBUG: Using ENHANCED SAM2Handler.create_mask() with full-body detection")
90
+
91
+ # Call SAM2Handler's create_mask method (which has its own debug outputs)
92
  mask = self.sam2_handler.create_mask(frame_rgb)
93
 
 
 
 
 
 
 
 
94
  else:
95
+ logger.info("DEBUG: Using basic SAM2 segmentation fallback")
96
+ mask = self._basic_sam2_fallback(frame_rgb)
 
 
 
 
 
 
 
 
 
 
 
 
 
97
 
98
+ # DEBUG: Check if TwoStageProcessor is getting the mask correctly
99
+ if mask is not None:
100
+ logger.info(f"DEBUG: TwoStageProcessor received mask shape: {mask.shape}, dtype: {mask.dtype}")
101
+ logger.info(f"DEBUG: TwoStageProcessor received mask range: {mask.min():.3f} to {mask.max():.3f}")
102
+ logger.info(f"DEBUG: TwoStageProcessor received mask non-zero pixels: {np.count_nonzero(mask)}")
103
+
104
+ # Convert to uint8 for saving if needed
105
+ if mask.dtype != np.uint8:
106
+ debug_mask = (mask * 255).astype(np.uint8)
107
+ else:
108
+ debug_mask = mask.copy()
109
+
110
+ cv2.imwrite(f"/tmp/debug_twostage_received_mask_frame_{frame_index}.png", debug_mask)
111
+ logger.info(f"DEBUG: Saved TwoStageProcessor received mask to /tmp/debug_twostage_received_mask_frame_{frame_index}.png")
112
+
113
+ # DISABLED: No additional cleaning - let SAM2Handler's cleaning stand
114
+ # mask = self._maximum_quality_mask_cleaning(mask) # DISABLED FOR TESTING
115
+ logger.info("DEBUG: Double cleaning DISABLED - using mask as received from SAM2Handler")
116
+
117
+ return mask
118
+ else:
119
+ logger.warning(f"DEBUG: No mask received from SAM2Handler for frame {frame_index}")
120
+ return None
121
+
122
  except Exception as e:
123
+ logger.error(f"DEBUG: Exception in TwoStageProcessor._get_mask for frame {frame_index}: {e}")
124
+ logger.error(f"DEBUG: Traceback: {traceback.format_exc()}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
125
  return None
 
 
 
 
 
 
126
 
127
+ def _basic_sam2_fallback(self, frame_rgb):
128
+ """Fallback for basic SAM2 predictor without enhanced features."""
129
+ try:
130
+ logger.info("DEBUG: Running basic SAM2 fallback")
131
+ self.sam2_handler.set_image(frame_rgb)
132
+ # Simple center point prompt
133
+ h, w = frame_rgb.shape[:2]
134
+ center_point = np.array([[w//2, h//2]])
135
+ center_label = np.array([1])
136
+
137
+ masks, scores, _ = self.sam2_handler.predict(
138
+ point_coords=center_point,
139
+ point_labels=center_label,
140
+ multimask_output=True
141
+ )
142
+
143
+ if len(masks) > 0:
144
+ # Select best mask by score
145
+ best_idx = np.argmax(scores)
146
+ mask = masks[best_idx].astype(np.float32)
147
+ logger.info(f"DEBUG: Basic SAM2 fallback - selected mask {best_idx} with score {scores[best_idx]:.3f}")
148
+ return mask
149
+ else:
150
+ logger.warning("DEBUG: Basic SAM2 fallback produced no masks")
151
+ return None
152
+ except Exception as e:
153
+ logger.error(f"DEBUG: Basic SAM2 fallback failed: {e}")
154
  return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
155
 
156
+ def _maximum_quality_mask_cleaning(self, mask):
 
157
  """
158
+ DISABLED FOR TESTING - Maximum quality mask post-processing.
159
+ Returns mask unchanged to prevent double cleaning artifacts.
160
  """
161
+ logger.info("DEBUG: _maximum_quality_mask_cleaning called but DISABLED")
162
+ logger.info("DEBUG: Returning mask unchanged to prevent double cleaning artifacts")
163
+ return mask
 
 
 
 
 
164
 
165
+ def process_video(self,
166
+ input_path: str,
167
+ background_image_path: Optional[str] = None,
168
+ background_preset: str = "office",
169
+ callback: Optional[Callable] = None,
170
+ trim_seconds: float = 5.0) -> Tuple[Optional[str], Optional[str], str]:
171
  """
172
+ Process video with two-stage background replacement.
173
+ Returns: (final_output_path, green_screen_path, status_message)
174
  """
 
 
175
  try:
176
+ self.temp_dir = tempfile.mkdtemp(prefix="twostage_debug_")
177
+ logger.info(f"DEBUG: Processing video: {input_path}")
178
+ logger.info(f"DEBUG: Temp directory: {self.temp_dir}")
179
+
180
+ # Stage 1: Create green screen video
181
+ green_screen_path = self._stage1_create_greenscreen(
182
+ input_path, trim_seconds, callback
183
+ )
184
+
185
+ if not green_screen_path:
186
+ return None, None, "❌ Stage 1 failed: Could not create green screen video"
187
+
188
+ if callback:
189
+ callback(50, "Stage 1 complete, starting Stage 2...")
190
+
191
+ # Stage 2: Apply background to green screen video
192
+ final_output_path = self._stage2_apply_background(
193
+ green_screen_path, background_image_path, background_preset, callback
194
+ )
195
+
196
+ if not final_output_path:
197
+ return None, green_screen_path, "❌ Stage 2 failed: Could not apply background"
198
+
199
+ logger.info(f"✅ Two-stage processing complete: {final_output_path}")
200
+ return final_output_path, green_screen_path, "✅ Video processed successfully!"
201
+
202
  except Exception as e:
203
+ logger.error(f"Two-stage processing failed: {e}")
204
+ logger.error(traceback.format_exc())
205
+ return None, None, f"❌ Processing failed: {str(e)}"
206
+
207
+ finally:
208
+ self._cleanup()
 
 
 
 
 
 
209
 
210
+ def _stage1_create_greenscreen(self, input_path: str, trim_seconds: float, callback: Optional[Callable]) -> Optional[str]:
211
+ """Stage 1: Create green screen video using person segmentation with debug."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
212
  try:
213
+ cap = cv2.VideoCapture(input_path)
 
 
 
 
214
  if not cap.isOpened():
215
+ logger.error(f"Cannot open video: {input_path}")
216
+ return None
217
+
218
+ # Video properties
219
+ fps = cap.get(cv2.CAP_PROP_FPS)
220
+ width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
221
+ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
222
+ total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
223
+
224
+ if trim_seconds > 0:
225
+ max_frames = int(fps * trim_seconds)
226
+ total_frames = min(total_frames, max_frames)
227
+
228
+ logger.info(f"DEBUG: Stage 1 - {total_frames} frames, {fps:.2f} FPS, {width}x{height}")
229
+
230
+ # Output path
231
+ green_screen_path = os.path.join(self.temp_dir, "green_screen_debug.mp4")
232
+ writer = create_video_writer(green_screen_path, fps, width, height)
233
+
234
+ # Define green screen color (force consistent green)
235
+ green_color = (0, 255, 0) # Pure green in BGR
236
+
237
+ frame_count = 0
238
+ successful_masks = 0
239
+ failed_masks = 0
240
+
241
+ while frame_count < total_frames:
242
+ ret, frame = cap.read()
243
+ if not ret:
244
  break
245
+
246
+ # Convert to RGB for SAM2
247
+ frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
248
+
249
+ # Get person mask with debug outputs
250
+ mask = self._get_mask(frame_rgb, frame_count)
251
+
252
+ if mask is not None:
253
+ successful_masks += 1
254
+
255
+ # Ensure mask is proper format
256
+ if mask.dtype != np.uint8:
257
+ mask = (mask * 255).astype(np.uint8)
258
+
259
+ # Create green screen frame
260
+ green_frame = np.full_like(frame, green_color, dtype=np.uint8)
261
+
262
+ # Apply mask to original frame
263
+ if len(mask.shape) == 2:
264
+ mask_3ch = cv2.merge([mask, mask, mask])
265
  else:
266
+ mask_3ch = mask
267
+
268
+ # Normalize mask to 0-1 range
269
+ mask_norm = mask_3ch.astype(np.float32) / 255.0
270
+
271
+ # Composite: person on green background
272
+ result_frame = (frame.astype(np.float32) * mask_norm +
273
+ green_frame.astype(np.float32) * (1 - mask_norm)).astype(np.uint8)
274
+
275
+ writer.write(result_frame)
276
+
277
+ # Save debug frame occasionally
278
+ if frame_count % 30 == 0: # Every 30th frame
279
+ debug_frame_path = f"/tmp/debug_greenscreen_frame_{frame_count}.png"
280
+ cv2.imwrite(debug_frame_path, result_frame)
281
+ logger.info(f"DEBUG: Saved green screen frame to {debug_frame_path}")
282
+
 
 
283
  else:
284
+ failed_masks += 1
285
+ # No mask detected - use pure green frame
286
+ logger.warning(f"DEBUG: No mask detected for frame {frame_count}, using pure green")
287
+ green_frame = np.full_like(frame, green_color, dtype=np.uint8)
288
+ writer.write(green_frame)
289
+
290
+ frame_count += 1
291
+
292
+ # Progress callback
293
+ if callback and frame_count % 10 == 0:
294
+ progress = int(25 * frame_count / total_frames) # Stage 1 is 0-25%
295
+ callback(progress, f"Creating green screen... {frame_count}/{total_frames}")
296
+
297
  cap.release()
298
  writer.release()
299
+
300
+ logger.info(f"DEBUG: Stage 1 complete - Successful masks: {successful_masks}, Failed masks: {failed_masks}")
301
+
302
+ if os.path.exists(green_screen_path) and os.path.getsize(green_screen_path) > 0:
303
+ logger.info(f"✅ Stage 1 complete: {green_screen_path}")
304
+ self._cleanup_list.append(green_screen_path)
305
+ return green_screen_path
306
+ else:
307
+ logger.error("Stage 1 failed: Empty output file")
308
+ return None
309
+
 
 
 
 
 
 
 
 
 
 
310
  except Exception as e:
311
+ logger.error(f"Stage 1 failed: {e}")
312
+ logger.error(traceback.format_exc())
313
+ return None
314
 
315
+ def _stage2_apply_background(self, green_screen_path: str, background_image_path: Optional[str],
316
+ background_preset: str, callback: Optional[Callable]) -> Optional[str]:
317
+ """Stage 2: Apply background to green screen video using professional chroma key."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
318
  try:
319
+ # Load background
320
+ background = self._prepare_background(background_image_path, background_preset)
321
+ if background is None:
322
+ logger.error("Failed to prepare background")
323
+ return None
324
+
325
+ cap = cv2.VideoCapture(green_screen_path)
326
  if not cap.isOpened():
327
+ logger.error(f"Cannot open green screen video: {green_screen_path}")
328
+ return None
329
+
330
+ # Video properties
331
+ fps = cap.get(cv2.CAP_PROP_FPS)
332
+ width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
333
+ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
334
+ total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
335
+
336
+ logger.info(f"DEBUG: Stage 2 - {total_frames} frames, {fps:.2f} FPS, {width}x{height}")
337
+
338
+ # Resize background to match video
339
+ background = cv2.resize(background, (width, height))
340
+ logger.info(f"DEBUG: Background resized to {width}x{height}")
341
+
342
+ # Output path
343
+ final_output_path = os.path.join(self.temp_dir, "final_output_debug.mp4")
344
+ writer = create_video_writer(final_output_path, fps, width, height)
345
+
346
+ frame_count = 0
347
+ while frame_count < total_frames:
348
+ ret, frame = cap.read()
349
+ if not ret:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
350
  break
351
+
352
+ # Professional chroma key for green screen
353
+ result_frame = self._professional_chroma_key(frame, background)
354
+ writer.write(result_frame)
355
+
356
+ # Save debug frame occasionally
357
+ if frame_count % 30 == 0: # Every 30th frame
358
+ debug_final_path = f"/tmp/debug_final_frame_{frame_count}.png"
359
+ cv2.imwrite(debug_final_path, result_frame)
360
+ logger.info(f"DEBUG: Saved final frame to {debug_final_path}")
361
+
362
+ frame_count += 1
363
+
364
+ # Progress callback
365
+ if callback and frame_count % 10 == 0:
366
+ progress = int(25 + 75 * frame_count / total_frames) # Stage 2 is 25-100%
367
+ callback(progress, f"Applying background... {frame_count}/{total_frames}")
368
+
 
 
 
 
 
 
 
 
369
  cap.release()
370
  writer.release()
371
+
372
+ if os.path.exists(final_output_path) and os.path.getsize(final_output_path) > 0:
373
+ logger.info(f"✅ Stage 2 complete: {final_output_path}")
374
+ return final_output_path
375
+ else:
376
+ logger.error("Stage 2 failed: Empty output file")
377
+ return None
378
+
379
  except Exception as e:
380
+ logger.error(f"Stage 2 failed: {e}")
381
+ logger.error(traceback.format_exc())
382
+ return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
383
 
384
+ def _professional_chroma_key(self, frame, background):
385
+ """Professional chroma key for green screen removal."""
386
+ # Convert to HSV for better green detection
387
+ hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
388
 
389
+ # Define green color range in HSV
390
+ # Green hue is around 60 in OpenCV HSV (0-179 range)
391
+ lower_green = np.array([40, 40, 40]) # Lower bound
392
+ upper_green = np.array([80, 255, 255]) # Upper bound
393
 
394
+ # Create mask for green pixels
395
+ green_mask = cv2.inRange(hsv, lower_green, upper_green)
396
+
397
+ # Smooth the mask
398
+ kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
399
+ green_mask = cv2.morphologyEx(green_mask, cv2.MORPH_OPEN, kernel)
400
+ green_mask = cv2.morphologyEx(green_mask, cv2.MORPH_CLOSE, kernel)
401
+
402
+ # Blur for soft edges
403
+ green_mask = cv2.GaussianBlur(green_mask, (5, 5), 0)
404
+
405
+ # Convert mask to 3-channel and normalize
406
+ green_mask_3ch = cv2.merge([green_mask, green_mask, green_mask])
407
+ alpha = green_mask_3ch.astype(np.float32) / 255.0
408
+
409
+ # Composite: background where green, original where not green
410
+ result = (background.astype(np.float32) * alpha +
411
+ frame.astype(np.float32) * (1 - alpha)).astype(np.uint8)
412
+
413
+ return result
 
 
 
 
 
 
 
 
 
 
414
 
415
+ def _prepare_background(self, background_image_path: Optional[str], background_preset: str) -> Optional[np.ndarray]:
416
+ """Prepare background image from file or preset."""
417
  try:
418
+ if background_image_path and os.path.exists(background_image_path):
419
+ logger.info(f"Loading custom background: {background_image_path}")
420
+ background = cv2.imread(background_image_path)
421
+ if background is not None:
422
+ return background
423
+ logger.warning(f"Failed to load custom background, falling back to preset")
424
+
425
+ # Generate preset background
426
+ logger.info(f"Using preset background: {background_preset}")
427
+ return self._generate_preset_background(background_preset)
428
+
429
+ except Exception as e:
430
+ logger.error(f"Background preparation failed: {e}")
431
+ return None
 
 
 
 
 
 
 
 
 
 
432
 
433
+ def _generate_preset_background(self, preset: str) -> np.ndarray:
434
+ """Generate preset background (fallback implementation)."""
435
+ # Simple solid color backgrounds
436
+ color_map = {
437
+ "office": (240, 240, 240), # Light gray
438
+ "blue": (255, 200, 100), # Light blue
439
+ "studio": (200, 200, 200), # Medium gray
440
+ "white": (255, 255, 255), # White
441
+ "black": (0, 0, 0) # Black
442
+ }
443
+
444
+ color = color_map.get(preset, (240, 240, 240)) # Default to office
445
+ background = np.full((720, 1280, 3), color, dtype=np.uint8) # Default size
446
+
447
+ logger.info(f"Generated {preset} background with color {color}")
448
+ return background
449
 
450
+ def _cleanup(self):
451
+ """Clean up temporary files."""
452
+ try:
453
+ for file_path in self._cleanup_list:
454
+ if os.path.exists(file_path):
455
+ os.remove(file_path)
456
+ logger.debug(f"Cleaned up: {file_path}")
457
+
458
+ if self.temp_dir and os.path.exists(self.temp_dir):
459
+ import shutil
460
+ shutil.rmtree(self.temp_dir)
461
+ logger.debug(f"Cleaned up temp directory: {self.temp_dir}")
462
+
463
+ except Exception as e:
464
+ logger.warning(f"Cleanup failed: {e}")
465
+
466
+ class BasicSAM2Wrapper:
467
+ """Wrapper for basic SAM2 predictor to provide create_mask interface with debug."""
468
+
469
+ def __init__(self, predictor):
470
+ self.predictor = predictor
471
+ logger.info("DEBUG: Created BasicSAM2Wrapper for compatibility")
472
+
473
+ def create_mask(self, frame_rgb):
474
+ """Basic mask creation using center point prompt with debug outputs."""
475
+ try:
476
+ logger.info("DEBUG: BasicSAM2Wrapper.create_mask() called")
477
+ logger.info(f"DEBUG: BasicSAM2Wrapper input frame shape: {frame_rgb.shape}")
478
+
479
+ self.predictor.set_image(frame_rgb)
480
+
481
+ # Simple center point prompt
482
+ h, w = frame_rgb.shape[:2]
483
+ center_point = np.array([[w//2, h//2]])
484
+ center_label = np.array([1])
485
+
486
+ masks, scores, _ = self.predictor.predict(
487
+ point_coords=center_point,
488
+ point_labels=center_label,
489
+ multimask_output=True
490
  )
491
+
492
+ if len(masks) > 0:
493
+ # Select best mask by score
494
+ best_idx = np.argmax(scores)
495
+ mask = masks[best_idx].astype(np.float32)
496
+
497
+ logger.info(f"DEBUG: BasicSAM2Wrapper generated {len(masks)} masks")
498
+ logger.info(f"DEBUG: BasicSAM2Wrapper selected mask {best_idx} with score {scores[best_idx]:.3f}")
499
+ logger.info(f"DEBUG: BasicSAM2Wrapper mask has {np.count_nonzero(mask)} non-zero pixels")
500
+
501
+ # Save debug mask
502
+ debug_mask = (mask * 255).astype(np.uint8)
503
+ cv2.imwrite("/tmp/debug_basic_sam2_wrapper_mask.png", debug_mask)
504
+ logger.info("DEBUG: Saved BasicSAM2Wrapper mask to /tmp/debug_basic_sam2_wrapper_mask.png")
505
+
506
+ return mask
507
+
508
+ logger.warning("DEBUG: BasicSAM2Wrapper - No masks generated")
509
+ return None
510
+
511
  except Exception as e:
512
+ logger.error(f"DEBUG: BasicSAM2Wrapper failed: {e}")
513
+ logger.error(f"DEBUG: BasicSAM2Wrapper traceback: {traceback.format_exc()}")
514
+ return None