student-v3 / code /precompute_shard_v3.py
amir0907's picture
Surgical emotion module v3: 50k-utt multi-corpus distillation (RAVDESS+CREMA-D+TESS+ESD, 2xT4)
2448b4a verified
Raw History Blame Contribute Delete
5.95 kB
"""Precompute frozen-teacher caches for v3 — multi-corpus, sharded over workers.
Corpora (train pool, >=10x the v2 set):
RAVDESS speech+song, actors 1-18 (reuse is NOT done here; v3 recomputes all
shards uniformly so any worker can take any slice)
CREMA-D (en, 7442) | TESS (en, 2800) | SAVEE (en, 480) | Emo-DB (de, 535)
| ESD (en+zh, ~35000)
Usage: python3 precompute_shard_v3.py <device> <shard_idx> <n_shards>
Deterministic global file list -> this worker takes every n_shards-th file.
Saves /tmp/work/teacher_cache_v3/shard_<idx>.pt
"""
import glob
import os
import sys
import warnings
warnings.filterwarnings("ignore")
import logging
logging.disable(logging.CRITICAL)
import torch
import torchaudio
sys.path.insert(0, "/tmp/work/s2st_emotion")
from funasr import AutoModel
from surgical_emotion import SurgicalConfig, SurgicalEmotionEncoder
DEV = sys.argv[1] if len(sys.argv) > 1 else "cuda:0"
SHARD = int(sys.argv[2]) if len(sys.argv) > 2 else 0
N_SHARDS = int(sys.argv[3]) if len(sys.argv) > 3 else 4
MAX_SEC = 12.0
OUT_DIR = "/tmp/work/teacher_cache_v3"
os.makedirs(OUT_DIR, exist_ok=True)
def gather_files():
"""Corpus roots -> file lists; RAVDESS only train actors, speech+song."""
pools = {
"ravdess": [],
"cremad": [],
"tess": [],
"savee": [],
"emodb": [],
"esd": [],
}
for f in glob.glob("/tmp/work/ravdess_kag/**/*.wav", recursive=True):
b = os.path.basename(f)
p = b.split("-")
try:
if int(p[-1].split(".")[0]) <= 18 and p[1] in ("01", "02"):
pools["ravdess"].append(f)
except Exception:
pass
pools["cremad"] = glob.glob("/tmp/work/cremad/**/*.wav", recursive=True)
pools["tess"] = glob.glob("/tmp/work/tess/**/*.wav", recursive=True)
pools["savee"] = glob.glob("/tmp/work/savee/**/*.wav", recursive=True)
pools["emodb"] = glob.glob("/tmp/work/emodb/**/*.wav", recursive=True)
esd = []
for f in glob.glob("/tmp/work/esd/**/*.wav", recursive=True):
# ESD layout .../Speaker/Emotion/xxx.wav ; keep emotion folders only
if any(e in f.upper().split(os.sep)[-2] for e in
("ANGRY", "HAPPY", "NEUTRAL", "SAD", "SURPRISE")):
esd.append(f)
pools["esd"] = esd
# deterministic interleave: plain global sort (roots differ per corpus), so a
# stride-n slice still samples every corpus proportionally
order = sorted(f for v in pools.values() for f in v)
return order, {k: len(v) for k, v in pools.items()}
def rms_envelope_50hz(wav, T_frames):
frames = torch.zeros(T_frames)
for i in range(T_frames):
seg = wav[i * 320: (i + 1) * 320]
frames[i] = float(seg.pow(2).mean().sqrt()) if seg.numel() else (frames[i - 1] if i else 0.0)
if T_frames >= 9:
frames = torch.nn.functional.conv1d(
frames[None, None], torch.ones(9)[None, None] / 9.0, padding=4)[0, 0]
lo, hi = frames.min(), frames.max()
return torch.full_like(frames, 0.5) if hi - lo < 1e-6 else (frames - lo) / (hi - lo)
def main():
files, counts = gather_files()
mine = files[SHARD::N_SHARDS]
print(f"[shard {SHARD}/{N_SHARDS} on {DEV}] corpus sizes={counts} total={len(files)} mine={len(mine)}", flush=True)
# resume support: partial cache keeps work safe across restarts
partial_path = f"{OUT_DIR}/shard_{SHARD}_partial.pt"
cache = []
if os.path.exists(partial_path):
cache = torch.load(partial_path, map_location="cpu", weights_only=False)
print(f"[shard {SHARD}] resuming with {len(cache)} done, skipping ahead", flush=True)
mine = mine[len(cache):]
fun = AutoModel(model="/tmp/work/emotion2vec_plus_base", trust_remote_code=False,
device=DEV, disable_update=True)
enc = SurgicalEmotionEncoder(fun.model, SurgicalConfig()).to(DEV)
enc.eval()
torch.set_grad_enabled(False)
# cached GPU resamplers: torchaudio functional.resample on CPU rebuilds a huge
# sinc kernel per call for odd rates (e.g. TESS 24414 Hz) -> seconds per file;
# a cached transform applied on GPU is milliseconds.
resamplers = {}
def to_16k(wav, sr):
if sr == 16000:
return wav
if sr not in resamplers:
resamplers[sr] = torchaudio.transforms.Resample(sr, 16000).to(DEV)
return resamplers[sr](wav.to(DEV)).cpu()
skipped = 0
for i, f in enumerate(mine):
try:
wav, sr = torchaudio.load(f)
wav = wav.mean(0)
if sr != 16000:
wav = to_16k(wav, sr)
except Exception:
skipped += 1
continue
if wav.numel() < 8000:
skipped += 1
continue
if wav.numel() > MAX_SEC * 16000:
wav = wav[: int(MAX_SEC * 16000)]
wav = wav / wav.abs().max().clamp(min=1e-4)
states, lengths = enc.extract_teacher_states(wav[None].to(DEV),
torch.tensor([wav.numel()], device=DEV))
probs, inten = enc.teacher_frame_probs(states, lengths)
T = int(lengths[0].item())
rms = rms_envelope_50hz(wav, T)
inten = (inten.squeeze(0)[:, 0].cpu().clamp(min=0, max=1).sqrt() * rms).unsqueeze(-1)
cache.append({
"states": states.squeeze(0).half().cpu(),
"probs": probs.squeeze(0).half().cpu(),
"inten": inten.half().cpu(),
"file": os.path.basename(f),
})
done = len(cache)
if done % 500 == 0:
print(f"[shard {SHARD}] {done}/{12551}", flush=True)
if done % 2000 == 0:
torch.save(cache, partial_path)
torch.save(cache, f"{OUT_DIR}/shard_{SHARD}.pt")
if os.path.exists(partial_path):
os.remove(partial_path)
print(f"[shard {SHARD}] saved {len(cache)} utts (skipped {skipped})", flush=True)
if __name__ == "__main__":
main()