Spaces:
Sleeping
Sleeping
Updated WavLM processing from linear to 30-second sequential
Browse filesUpdated WavLM processing from linear to 30-second sequential to overcome resource (RAM) constraints.
main.py
CHANGED
|
@@ -112,11 +112,44 @@ def _extract_text_features(text: str) -> torch.Tensor:
|
|
| 112 |
return out.last_hidden_state[:, 0, :] # Shape: [1, 768]
|
| 113 |
|
| 114 |
def _extract_wavlm_features(wav_path: str) -> torch.Tensor:
|
|
|
|
| 115 |
audio, _ = librosa.load(wav_path, sr=16000)
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
|
| 121 |
def _extract_vision_features() -> torch.Tensor:
|
| 122 |
# Modality Imputation: We return a neutral 68-dim tensor since OpenFace C++ binaries aren't running
|
|
|
|
| 112 |
return out.last_hidden_state[:, 0, :] # Shape: [1, 768]
|
| 113 |
|
| 114 |
def _extract_wavlm_features(wav_path: str) -> torch.Tensor:
|
| 115 |
+
log.info(" [WavLM] Starting chunked acoustic extraction to preserve RAM...")
|
| 116 |
audio, _ = librosa.load(wav_path, sr=16000)
|
| 117 |
+
|
| 118 |
+
chunk_length_s = 30 # Process exactly 30 seconds of audio at a time
|
| 119 |
+
chunk_samples = chunk_length_s * 16000
|
| 120 |
+
|
| 121 |
+
all_chunk_features = []
|
| 122 |
+
|
| 123 |
+
# Loop through the massive audio file in 30-second windows
|
| 124 |
+
for i in range(0, len(audio), chunk_samples):
|
| 125 |
+
chunk = audio[i : i + chunk_samples]
|
| 126 |
+
|
| 127 |
+
# Skip tiny micro-chunks at the very end (less than 1 second)
|
| 128 |
+
if len(chunk) < 16000:
|
| 129 |
+
continue
|
| 130 |
+
|
| 131 |
+
inputs = wavlm_extractor(chunk, sampling_rate=16000, return_tensors="pt").to(DEVICE)
|
| 132 |
+
|
| 133 |
+
with torch.no_grad():
|
| 134 |
+
out = wavlm_model(**inputs)
|
| 135 |
+
|
| 136 |
+
# Get the mean representation for this specific 30-second chunk
|
| 137 |
+
chunk_feat = out.last_hidden_state.mean(dim=1) # Shape: [1, 768]
|
| 138 |
+
all_chunk_features.append(chunk_feat)
|
| 139 |
+
|
| 140 |
+
# Force the server to dump the RAM before loading the next chunk
|
| 141 |
+
import gc
|
| 142 |
+
gc.collect()
|
| 143 |
+
if torch.cuda.is_available():
|
| 144 |
+
torch.cuda.empty_cache()
|
| 145 |
+
|
| 146 |
+
log.info(f" [WavLM] Successfully processed {len(all_chunk_features)} audio chunks.")
|
| 147 |
+
|
| 148 |
+
# Mathematically average all the chunks together into ONE final tensor
|
| 149 |
+
# This perfectly mimics the temporal pooling your model expects!
|
| 150 |
+
final_audio_feat = torch.stack(all_chunk_features).mean(dim=0) # Shape: [1, 768]
|
| 151 |
+
|
| 152 |
+
return final_audio_feat # Shape: [1, 768]
|
| 153 |
|
| 154 |
def _extract_vision_features() -> torch.Tensor:
|
| 155 |
# Modality Imputation: We return a neutral 68-dim tensor since OpenFace C++ binaries aren't running
|