J Z commited on
Update float_lipsync.py
Browse files- float_lipsync.py +2 -16
float_lipsync.py
CHANGED
|
@@ -229,7 +229,6 @@ class FloatLipsync:
|
|
| 229 |
img = self._crop_face(img)
|
| 230 |
|
| 231 |
self.preprocessed_ref_image = self.transform(image=img)['image'].unsqueeze(0).to(self.device)
|
| 232 |
-
self._original_ref_image = self.preprocessed_ref_image.clone()
|
| 233 |
logger.info(f"[FLOAT] Reference image preprocessed: {self.preprocessed_ref_image.shape}")
|
| 234 |
|
| 235 |
def _crop_face(self, img: np.ndarray) -> np.ndarray:
|
|
@@ -274,7 +273,7 @@ class FloatLipsync:
|
|
| 274 |
return False
|
| 275 |
|
| 276 |
@torch.no_grad()
|
| 277 |
-
def generate(self, audio_path: str, output_path: str = None, emo: str = 'S2E'
|
| 278 |
"""
|
| 279 |
Generate a lipsync video from an audio file.
|
| 280 |
|
|
@@ -282,7 +281,6 @@ class FloatLipsync:
|
|
| 282 |
audio_path: Path to the input WAV audio file
|
| 283 |
output_path: Where to save the output video (auto-generated if None)
|
| 284 |
emo: Emotion mode ('S2E' for speech-to-emotion auto-detect)
|
| 285 |
-
chain_frames: If True, use the last frame as reference for next call
|
| 286 |
|
| 287 |
Returns:
|
| 288 |
Path to the generated video file
|
|
@@ -326,25 +324,13 @@ class FloatLipsync:
|
|
| 326 |
)['d_hat']
|
| 327 |
logger.info(f"[FLOAT] Model inference: {time.time() - gen_start:.2f}s")
|
| 328 |
|
| 329 |
-
# 3.
|
| 330 |
-
if chain_frames and d_hat.shape[0] > 0:
|
| 331 |
-
# d_hat is (T, C, H, W) in [-1, 1] range — same as preprocessed_ref_image
|
| 332 |
-
self.preprocessed_ref_image = d_hat[-1:].clone().to(self.device)
|
| 333 |
-
logger.info(f"[FLOAT] Chained last frame as next reference")
|
| 334 |
-
|
| 335 |
-
# 4. Save video with audio
|
| 336 |
self._save_video(d_hat, output_path, audio_path)
|
| 337 |
|
| 338 |
logger.info(f"[FLOAT] ✓ Total generation: {time.time() - inference_start:.2f}s")
|
| 339 |
logger.info(f"[FLOAT] ✓ Output: {output_path}")
|
| 340 |
return output_path
|
| 341 |
|
| 342 |
-
def reset_reference(self):
|
| 343 |
-
"""Reset the reference image back to the original ref.png."""
|
| 344 |
-
if hasattr(self, '_original_ref_image') and self._original_ref_image is not None:
|
| 345 |
-
self.preprocessed_ref_image = self._original_ref_image.clone()
|
| 346 |
-
logger.info("[FLOAT] Reference reset to original")
|
| 347 |
-
|
| 348 |
def _save_video(self, vid_tensor: torch.Tensor, video_path: str, audio_path: str):
|
| 349 |
"""Save video frames tensor to mp4 with audio using ffmpeg."""
|
| 350 |
# Prepare frames: (T, C, H, W) -> (T, H, W, C) uint8
|
|
|
|
| 229 |
img = self._crop_face(img)
|
| 230 |
|
| 231 |
self.preprocessed_ref_image = self.transform(image=img)['image'].unsqueeze(0).to(self.device)
|
|
|
|
| 232 |
logger.info(f"[FLOAT] Reference image preprocessed: {self.preprocessed_ref_image.shape}")
|
| 233 |
|
| 234 |
def _crop_face(self, img: np.ndarray) -> np.ndarray:
|
|
|
|
| 273 |
return False
|
| 274 |
|
| 275 |
@torch.no_grad()
|
| 276 |
+
def generate(self, audio_path: str, output_path: str = None, emo: str = 'S2E') -> str:
|
| 277 |
"""
|
| 278 |
Generate a lipsync video from an audio file.
|
| 279 |
|
|
|
|
| 281 |
audio_path: Path to the input WAV audio file
|
| 282 |
output_path: Where to save the output video (auto-generated if None)
|
| 283 |
emo: Emotion mode ('S2E' for speech-to-emotion auto-detect)
|
|
|
|
| 284 |
|
| 285 |
Returns:
|
| 286 |
Path to the generated video file
|
|
|
|
| 324 |
)['d_hat']
|
| 325 |
logger.info(f"[FLOAT] Model inference: {time.time() - gen_start:.2f}s")
|
| 326 |
|
| 327 |
+
# 3. Save video with audio
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 328 |
self._save_video(d_hat, output_path, audio_path)
|
| 329 |
|
| 330 |
logger.info(f"[FLOAT] ✓ Total generation: {time.time() - inference_start:.2f}s")
|
| 331 |
logger.info(f"[FLOAT] ✓ Output: {output_path}")
|
| 332 |
return output_path
|
| 333 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 334 |
def _save_video(self, vid_tensor: torch.Tensor, video_path: str, audio_path: str):
|
| 335 |
"""Save video frames tensor to mp4 with audio using ffmpeg."""
|
| 336 |
# Prepare frames: (T, C, H, W) -> (T, H, W, C) uint8
|