Santos-Loc commited on
Commit
05385c6
·
verified ·
1 Parent(s): a9474cf

Updated WavLM processing from linear to 30-second sequential

Browse files

Updated WavLM processing from linear to 30-second sequential to overcome resource (RAM) constraints.

Files changed (1) hide show
  1. main.py +37 -4
main.py CHANGED
@@ -112,11 +112,44 @@ def _extract_text_features(text: str) -> torch.Tensor:
112
  return out.last_hidden_state[:, 0, :] # Shape: [1, 768]
113
 
114
  def _extract_wavlm_features(wav_path: str) -> torch.Tensor:
 
115
  audio, _ = librosa.load(wav_path, sr=16000)
116
- inputs = wavlm_extractor(audio, sampling_rate=16000, return_tensors="pt").to(DEVICE)
117
- with torch.no_grad():
118
- out = wavlm_model(**inputs)
119
- return out.last_hidden_state.mean(dim=1) # Shape: [1, 768]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
120
 
121
  def _extract_vision_features() -> torch.Tensor:
122
  # Modality Imputation: We return a neutral 68-dim tensor since OpenFace C++ binaries aren't running
 
112
  return out.last_hidden_state[:, 0, :] # Shape: [1, 768]
113
 
114
  def _extract_wavlm_features(wav_path: str) -> torch.Tensor:
115
+ log.info(" [WavLM] Starting chunked acoustic extraction to preserve RAM...")
116
  audio, _ = librosa.load(wav_path, sr=16000)
117
+
118
+ chunk_length_s = 30 # Process exactly 30 seconds of audio at a time
119
+ chunk_samples = chunk_length_s * 16000
120
+
121
+ all_chunk_features = []
122
+
123
+ # Loop through the massive audio file in 30-second windows
124
+ for i in range(0, len(audio), chunk_samples):
125
+ chunk = audio[i : i + chunk_samples]
126
+
127
+ # Skip tiny micro-chunks at the very end (less than 1 second)
128
+ if len(chunk) < 16000:
129
+ continue
130
+
131
+ inputs = wavlm_extractor(chunk, sampling_rate=16000, return_tensors="pt").to(DEVICE)
132
+
133
+ with torch.no_grad():
134
+ out = wavlm_model(**inputs)
135
+
136
+ # Get the mean representation for this specific 30-second chunk
137
+ chunk_feat = out.last_hidden_state.mean(dim=1) # Shape: [1, 768]
138
+ all_chunk_features.append(chunk_feat)
139
+
140
+ # Force the server to dump the RAM before loading the next chunk
141
+ import gc
142
+ gc.collect()
143
+ if torch.cuda.is_available():
144
+ torch.cuda.empty_cache()
145
+
146
+ log.info(f" [WavLM] Successfully processed {len(all_chunk_features)} audio chunks.")
147
+
148
+ # Mathematically average all the chunks together into ONE final tensor
149
+ # This perfectly mimics the temporal pooling your model expects!
150
+ final_audio_feat = torch.stack(all_chunk_features).mean(dim=0) # Shape: [1, 768]
151
+
152
+ return final_audio_feat # Shape: [1, 768]
153
 
154
  def _extract_vision_features() -> torch.Tensor:
155
  # Modality Imputation: We return a neutral 68-dim tensor since OpenFace C++ binaries aren't running