Update app.py
Browse files
app.py
CHANGED
|
@@ -1531,6 +1531,65 @@ def process_image_sync(image_data, model_name=None, transparent=True,
|
|
| 1531 |
gc.collect(); trim_memory()
|
| 1532 |
|
| 1533 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1534 |
def _process_frames_sync(raw_frames, resolved_model, transparent,
|
| 1535 |
session_id: Optional[str] = None,
|
| 1536 |
smooth: bool = True,
|
|
@@ -1544,6 +1603,7 @@ def _process_frames_sync(raw_frames, resolved_model, transparent,
|
|
| 1544 |
fallback_chain = ['silueta'] # Keep video RAM predictable; no heavyweight fallback.
|
| 1545 |
active_session = None # Disable frame-to-frame alpha ghosting on moving subjects.
|
| 1546 |
results = []
|
|
|
|
| 1547 |
previous_alpha = None
|
| 1548 |
for idx, content in enumerate(raw_frames):
|
| 1549 |
try:
|
|
@@ -1615,28 +1675,65 @@ def _process_frames_sync(raw_frames, resolved_model, transparent,
|
|
| 1615 |
if blended:
|
| 1616 |
arr[:, :, 3] = smoothed
|
| 1617 |
result = Image.fromarray(arr, "RGBA")
|
| 1618 |
-
|
| 1619 |
-
|
| 1620 |
-
|
| 1621 |
-
|
| 1622 |
-
|
| 1623 |
-
|
| 1624 |
-
|
| 1625 |
-
else:
|
| 1626 |
-
result = result.convert("RGB")
|
| 1627 |
-
result.save(buf, format="JPEG", quality=85, optimize=True); fmt = "jpg"
|
| 1628 |
-
results.append({"data":base64.b64encode(buf.getvalue()).decode("ascii"),
|
| 1629 |
-
"format":fmt,"size":buf.tell(),
|
| 1630 |
-
"model":used_model,"smoothed":blended,
|
| 1631 |
-
"matting":used_matting,"dropout_recovered":did_recover,
|
| 1632 |
-
"raw_foreground_fraction":raw_fraction,
|
| 1633 |
-
"final_foreground_fraction":final_fraction,
|
| 1634 |
-
"crop_retry":retry_status})
|
| 1635 |
except Exception as e:
|
| 1636 |
logger.error(f"[frames-batch] frame {idx} failed: {e}")
|
| 1637 |
results.append({"data":"","format":"png","error":str(e)[:120]})
|
| 1638 |
finally:
|
| 1639 |
gc.collect(); trim_memory()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1640 |
return results
|
| 1641 |
|
| 1642 |
|
|
|
|
| 1531 |
gc.collect(); trim_memory()
|
| 1532 |
|
| 1533 |
|
| 1534 |
+
def _recover_bidirectional_video_mask(rgb, alpha, prev_rgb, prev_alpha, next_rgb, next_alpha):
|
| 1535 |
+
"""Conservative CPU-only, two-sided optical-flow recovery within one 8-frame batch.
|
| 1536 |
+
|
| 1537 |
+
Only fill pixels where BOTH motion-aligned neighbors agree; preserve the
|
| 1538 |
+
current Silueta mask and never replace it with a full previous silhouette.
|
| 1539 |
+
"""
|
| 1540 |
+
if not (alpha.shape == prev_alpha.shape == next_alpha.shape):
|
| 1541 |
+
return alpha, 'shape_mismatch'
|
| 1542 |
+
h, w = alpha.shape
|
| 1543 |
+
area = np.count_nonzero(alpha >= 128)
|
| 1544 |
+
pa = np.count_nonzero(prev_alpha >= 128)
|
| 1545 |
+
na = np.count_nonzero(next_alpha >= 128)
|
| 1546 |
+
if area < h*w*.005 or min(pa, na) < h*w*.035:
|
| 1547 |
+
return alpha, 'insufficient_subject'
|
| 1548 |
+
if area > min(pa, na)*.65:
|
| 1549 |
+
return alpha, 'normal_coverage'
|
| 1550 |
+
if max(pa, na) > min(pa, na)*1.65:
|
| 1551 |
+
return alpha, 'neighbors_disagree_area'
|
| 1552 |
+
try:
|
| 1553 |
+
# Low-resolution flow bounds CPU and memory consumption on free Spaces.
|
| 1554 |
+
scale = min(1.0, 256.0 / max(h, w))
|
| 1555 |
+
sw, sh = max(32, int(w*scale)), max(32, int(h*scale))
|
| 1556 |
+
def gray(im):
|
| 1557 |
+
return cv2.cvtColor(cv2.resize(im, (sw, sh), interpolation=cv2.INTER_AREA), cv2.COLOR_RGB2GRAY)
|
| 1558 |
+
target = gray(rgb)
|
| 1559 |
+
gridx, gridy = np.meshgrid(np.arange(sw, dtype=np.float32), np.arange(sh, dtype=np.float32))
|
| 1560 |
+
def aligned(neighbor_rgb, neighbor_mask):
|
| 1561 |
+
# Flow target -> neighbor gives sampling coordinates for remap.
|
| 1562 |
+
flow = cv2.calcOpticalFlowFarneback(target, gray(neighbor_rgb), None,
|
| 1563 |
+
.5, 3, 15, 3, 5, 1.2, 0)
|
| 1564 |
+
coords_x = gridx + flow[:,:,0]
|
| 1565 |
+
coords_y = gridy + flow[:,:,1]
|
| 1566 |
+
small_mask = cv2.resize(neighbor_mask, (sw, sh), interpolation=cv2.INTER_LINEAR)
|
| 1567 |
+
warped = cv2.remap(small_mask, coords_x, coords_y,
|
| 1568 |
+
cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0)
|
| 1569 |
+
return cv2.resize(warped, (w,h), interpolation=cv2.INTER_LINEAR)
|
| 1570 |
+
prev_warp = aligned(prev_rgb, prev_alpha)
|
| 1571 |
+
next_warp = aligned(next_rgb, next_alpha)
|
| 1572 |
+
agreement = (prev_warp >= 150) & (next_warp >= 150)
|
| 1573 |
+
union = (prev_warp >= 150) | (next_warp >= 150)
|
| 1574 |
+
agreement_ratio = np.count_nonzero(agreement) / max(1, np.count_nonzero(union))
|
| 1575 |
+
if agreement_ratio < .60:
|
| 1576 |
+
return alpha, 'neighbors_disagree_motion'
|
| 1577 |
+
current_core = alpha >= 160
|
| 1578 |
+
# Must match a substantial part of the existing subject, not a chair.
|
| 1579 |
+
if np.count_nonzero(current_core & agreement) / max(1, np.count_nonzero(current_core)) < .72:
|
| 1580 |
+
return alpha, 'subject_mismatch'
|
| 1581 |
+
proposed = np.maximum(alpha, np.where(agreement, np.minimum(prev_warp, next_warp), 0).astype(np.uint8))
|
| 1582 |
+
new_area = np.count_nonzero(proposed >= 128)
|
| 1583 |
+
if new_area < area*1.22 or new_area > min(h*w*.65, min(pa, na)*1.20):
|
| 1584 |
+
return alpha, 'recovery_rejected_size'
|
| 1585 |
+
logger.info('[video-mask] two-sided recovery %.3f -> %.3f agreement=%.2f',
|
| 1586 |
+
area/(h*w), new_area/(h*w), agreement_ratio)
|
| 1587 |
+
return proposed, 'recovered'
|
| 1588 |
+
except Exception as exc:
|
| 1589 |
+
logger.warning('[video-mask] two-sided recovery failed: %s', str(exc)[:150])
|
| 1590 |
+
return alpha, 'flow_error'
|
| 1591 |
+
|
| 1592 |
+
|
| 1593 |
def _process_frames_sync(raw_frames, resolved_model, transparent,
|
| 1594 |
session_id: Optional[str] = None,
|
| 1595 |
smooth: bool = True,
|
|
|
|
| 1603 |
fallback_chain = ['silueta'] # Keep video RAM predictable; no heavyweight fallback.
|
| 1604 |
active_session = None # Disable frame-to-frame alpha ghosting on moving subjects.
|
| 1605 |
results = []
|
| 1606 |
+
pending = []
|
| 1607 |
previous_alpha = None
|
| 1608 |
for idx, content in enumerate(raw_frames):
|
| 1609 |
try:
|
|
|
|
| 1675 |
if blended:
|
| 1676 |
arr[:, :, 3] = smoothed
|
| 1677 |
result = Image.fromarray(arr, "RGBA")
|
| 1678 |
+
pending.append({"image": result, "rgb": np.asarray(img.convert("RGB")),
|
| 1679 |
+
"model": used_model, "smoothed": blended,
|
| 1680 |
+
"matting": used_matting, "dropout_recovered": did_recover,
|
| 1681 |
+
"raw_foreground_fraction": raw_fraction,
|
| 1682 |
+
"final_foreground_fraction": final_fraction,
|
| 1683 |
+
"crop_retry": retry_status})
|
| 1684 |
+
results.append(None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1685 |
except Exception as e:
|
| 1686 |
logger.error(f"[frames-batch] frame {idx} failed: {e}")
|
| 1687 |
results.append({"data":"","format":"png","error":str(e)[:120]})
|
| 1688 |
finally:
|
| 1689 |
gc.collect(); trim_memory()
|
| 1690 |
+
# Two-sided recovery happens AFTER all eight masks have been inferred.
|
| 1691 |
+
# Pending entries correspond only to successful frames; failed entries remain untouched.
|
| 1692 |
+
recovered_count = 0
|
| 1693 |
+
pending_index = 0
|
| 1694 |
+
for frame_index, entry in enumerate(results):
|
| 1695 |
+
if entry is not None:
|
| 1696 |
+
continue
|
| 1697 |
+
record = pending[pending_index]
|
| 1698 |
+
pending_index += 1
|
| 1699 |
+
before = pending[pending_index - 2] if pending_index > 1 and results[frame_index - 1] is None else None
|
| 1700 |
+
after = pending[pending_index] if (pending_index < len(pending) and frame_index + 1 < len(results) and results[frame_index + 1] is None) else None
|
| 1701 |
+
img_out = record['image']
|
| 1702 |
+
temporal_status = 'not_applicable'
|
| 1703 |
+
if before is not None and after is not None:
|
| 1704 |
+
mask = np.asarray(img_out.convert('RGBA'))[:, :, 3]
|
| 1705 |
+
prev_mask = np.asarray(before['image'].convert('RGBA'))[:, :, 3]
|
| 1706 |
+
next_mask = np.asarray(after['image'].convert('RGBA'))[:, :, 3]
|
| 1707 |
+
recovered_mask, temporal_status = _recover_bidirectional_video_mask(
|
| 1708 |
+
record['rgb'], mask, before['rgb'], prev_mask,
|
| 1709 |
+
after['rgb'], next_mask)
|
| 1710 |
+
if temporal_status == 'recovered':
|
| 1711 |
+
pixels = np.array(img_out.convert('RGBA'))
|
| 1712 |
+
pixels[:, :, 3] = recovered_mask
|
| 1713 |
+
img_out = Image.fromarray(pixels, 'RGBA')
|
| 1714 |
+
recovered_count += 1
|
| 1715 |
+
buf = io.BytesIO()
|
| 1716 |
+
if transparent:
|
| 1717 |
+
img_out.save(buf, format='PNG', compress_level=3)
|
| 1718 |
+
fmt = 'png'
|
| 1719 |
+
else:
|
| 1720 |
+
bg = Image.new('RGBA', img_out.size, (0, 0, 0, 255))
|
| 1721 |
+
Image.alpha_composite(bg, img_out.convert('RGBA')).convert('RGB').save(
|
| 1722 |
+
buf, format='JPEG', quality=85, optimize=True)
|
| 1723 |
+
fmt = 'jpg'
|
| 1724 |
+
results[frame_index] = {
|
| 1725 |
+
'data': base64.b64encode(buf.getvalue()).decode('ascii'),
|
| 1726 |
+
'format': fmt, 'size': buf.tell(),
|
| 1727 |
+
'model': record['model'], 'smoothed': record['smoothed'],
|
| 1728 |
+
'matting': record['matting'], 'dropout_recovered': record['dropout_recovered'],
|
| 1729 |
+
'raw_foreground_fraction': record['raw_foreground_fraction'],
|
| 1730 |
+
'final_foreground_fraction': record['final_foreground_fraction'],
|
| 1731 |
+
'crop_retry': record['crop_retry'],
|
| 1732 |
+
'temporal_recovery': temporal_status,
|
| 1733 |
+
}
|
| 1734 |
+
if recovered_count:
|
| 1735 |
+
logger.info('[video-mask] bidirectional recovery applied to %d/%d frames',
|
| 1736 |
+
recovered_count, len(results))
|
| 1737 |
return results
|
| 1738 |
|
| 1739 |
|