zeechimp commited on
Commit
242bde2
·
verified ·
1 Parent(s): 6ca0d2e

Create examples/reverb_ood.py

Browse files
Files changed (1) hide show
  1. examples/reverb_ood.py +77 -0
examples/reverb_ood.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Out-of-distribution reverb test.
4
+
5
+ The main demo's reverb test uses rt60_s=0.35 for both training
6
+ augmentation and evaluation. That measures whether the classifier
7
+ handles reverb from the family it saw during training. It does
8
+ not measure robustness to reverb it has never seen.
9
+
10
+ This script trains with rt60_s=0.35 and evaluates at several
11
+ decay times the classifier never saw. The resulting accuracy
12
+ curve is the honest robustness measurement.
13
+
14
+ Run: python examples/reverb_ood.py
15
+ """
16
+ import numpy as np
17
+ from whisper_decoder import (
18
+ MLP, train_mlp, generate_corpus, apply_reverb,
19
+ synth_word, extract_features, FEATURE_DIM,
20
+ WORD_ORDER, SR,
21
+ )
22
+
23
+
24
+ def eval_at_rt60(model, mu, sigma, rt60, n_per_word=20,
25
+ seed_base=200):
26
+ import random
27
+ rng = random.Random(seed_base + int(rt60 * 100))
28
+ n_words = len(WORD_ORDER)
29
+ X, y = [], []
30
+ for wi, word in enumerate(WORD_ORDER):
31
+ for k in range(n_per_word):
32
+ s = rng.randint(1, 1 << 30)
33
+ sig = synth_word(word, 'whisper', s)
34
+ sig = apply_reverb(sig, rt60_s=rt60,
35
+ seed=rng.randint(1, 1 << 30))
36
+ X.append(extract_features(sig))
37
+ y.append(wi)
38
+ X = (np.stack(X) - mu) / sigma
39
+ y = np.array(y)
40
+ return float((model.predict(X) == y).mean())
41
+
42
+
43
+ def main():
44
+ print("training on whisper + 25% reverb (rt60=0.35)...")
45
+ train = generate_corpus('whisper', 200, 40, seed=0,
46
+ reverb_train_frac=0.25)
47
+ mu = train.X_train.mean(axis=0)
48
+ sigma = train.X_train.std(axis=0) + 1e-9
49
+ X_tr = (train.X_train - mu) / sigma
50
+
51
+ model = MLP(FEATURE_DIM, 64, 32, 10, seed=0)
52
+ train_mlp(model, X_tr, train.y_train, epochs=200,
53
+ batch=64, lr=3e-3, seed=0)
54
+
55
+ print()
56
+ print(f" {'rt60 (s)':>10} {'accuracy':>10} note")
57
+ print(" " + "-" * 50)
58
+ for rt60 in [0.0, 0.10, 0.20, 0.35, 0.50, 0.70, 1.00, 1.50]:
59
+ acc = eval_at_rt60(model, mu, sigma, rt60)
60
+ if rt60 == 0.0:
61
+ note = "dry"
62
+ elif rt60 <= 0.40:
63
+ note = "in-distribution"
64
+ elif rt60 <= 0.70:
65
+ note = "mildly OOD"
66
+ else:
67
+ note = "strongly OOD"
68
+ print(f" {rt60:>10.2f} {acc*100:>9.1f}% {note}")
69
+
70
+ print()
71
+ print(" The training reverb distribution is centered at "
72
+ "0.35 s.")
73
+ print(" Accuracy at rt60 > 0.5 is out-of-distribution.")
74
+
75
+
76
+ if __name__ == "__main__":
77
+ main()