import gradio as gr
import numpy as np
import librosa
import parselmouth
import json
import csv
import os
import math
import hashlib
import time
import threading
import io
import base64
from dataclasses import dataclass, asdict
from typing import Optional, Dict, Any, Tuple, List
try:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
_HAS_MPL = True
except ImportError:
_HAS_MPL = False
# =============================================================================
# Constants & Content
# =============================================================================
RAINBOW_PASSAGE = (
"When the sunlight strikes raindrops in the air, they act like a prism "
"and form a rainbow. The rainbow is a division of white light into many "
"beautiful colors. These take the shape of a long round arch, with its "
"path high above, and its two ends apparently beyond the horizon. "
"There is, according to legend, a boiling pot of gold at one end. "
"People look, but no one ever finds it. When a person looks for something "
"beyond reach, their friends say they are looking for the pot of gold "
"at the end of the rainbow."
)
PROMPTS = {
"Rainbow Passage": RAINBOW_PASSAGE,
"Sustained vowels /i/ then /a/ (5s each)":
"Say 'eeee' for ~5 seconds, then 'ahhh' for ~5 seconds.",
"Counting 1-20":
"Count from 1 to 20 at a comfortable pace.",
"Spontaneous speech (10-20s)":
"Describe what you did yesterday in 2-3 sentences.",
}
# ---------------------------------------------------------------------------
# Distribution anchors for slider mapping
# ---------------------------------------------------------------------------
# Each entry defines a Normal(mu, sigma) clipped to [lo, hi].
# A raw metric is mapped to a CDF percentile (0-1) then scaled to 0-100.
#
# Slider polarity (matching Concept Slider design):
# 100 = High / Light / Bright / Bouncy / Soft (feminine-associated end)
# 0 = Low / Heavy / Dark / Monotone / Loud (masculine-associated end)
#
# Weight: Higher H1*-H2* -> lighter -> higher slider. No inversion needed.
# Volume: Higher RMS -> louder -> *inverted* so 100 = Soft.
# ---------------------------------------------------------------------------
# Sources:
# F0: Titze 1994, Hillenbrand 1995 — cis-M ≈ 120 Hz (σ≈20), cis-F ≈ 210 Hz (σ≈25)
# F0 SD: Henton 1989, Pépiot 2014 — reading ~2.5 st (σ≈1.0)
# H1*-H2*: Keating & Esposito 2006 — cis-M ≈ 2.5 dB, cis-F ≈ 6.5 dB
# F2-F1: Hillenbrand 1995 — cis-M ≈ 780 Hz, cis-F ≈ 1020 Hz
# Formant dispersion: Fitch 1997 — cis-M ≈ 1050 Hz, cis-F ≈ 1150 Hz
# ---------------------------------------------------------------------------
GLOBAL_DISTS = {
"f0_med_hz": {"mu": 165.0, "sigma": 50.0, "lo": 65.0, "hi": 350.0},
"f0_sd_st": {"mu": 2.5, "sigma": 1.0, "lo": 0.2, "hi": 7.0},
"rms_dbfs": {"mu": -22.0, "sigma": 6.0, "lo": -45.0, "hi": -8.0},
"resonance_combo": {"mu": 0.0, "sigma": 1.0, "lo": -3.0, "hi": 3.0},
"h1_h2_corrected": {"mu": 4.5, "sigma": 4.0, "lo": -8.0, "hi": 18.0},
}
ANCHORED_DISTS = {
**GLOBAL_DISTS,
"f0_med_hz": {"mu": 160.0, "sigma": 55.0, "lo": 70.0, "hi": 340.0},
"h1_h2_corrected": {"mu": 3.5, "sigma": 4.5, "lo": -8.0, "hi": 16.0},
}
# Legacy keys for baseline compatibility — if a baseline was built with old keys,
# select_dists will still find them.
_LEGACY_DIST_MAP = {
"f2_minus_f1_hz": "resonance_combo",
"h1_h2_db": "h1_h2_corrected",
}
# ── Resonance method definitions ──
# Each method maps a Metrics field to a distribution and polarity.
# "field": attribute name on the Metrics dataclass
# "dist": distribution dict (mu/sigma/lo/hi) for percentile mapping
# "invert": if True, lower raw value → higher (brighter) slider position
# "unit": display unit for the report
# "source": brief literature reference
RESONANCE_METHODS = {
"F2 − F1 (Hz)": {
"field": "f2_minus_f1_hz",
"dist": {"mu": 900.0, "sigma": 200.0, "lo": 300.0, "hi": 1600.0},
"invert": False, # higher = brighter
"unit": "Hz",
"source": "Hillenbrand 1995 — cis-M ≈ 780, cis-F ≈ 1020",
},
"Formant Dispersion Df": {
"field": "formant_dispersion_hz",
"dist": {"mu": 1100.0, "sigma": 150.0, "lo": 700.0, "hi": 1500.0},
"invert": False, # higher spacing = shorter VTL = brighter
"unit": "Hz",
"source": "Fitch 1997 — cis-M ≈ 1050, cis-F ≈ 1150",
},
"Est. VTL (cm)": {
"field": "vtl_cm",
"dist": {"mu": 16.0, "sigma": 1.8, "lo": 10.0, "hi": 22.0},
"invert": True, # shorter VTL = brighter → invert
"unit": "cm",
"source": "Fitch 1997; VTL = c / (4 × Df)",
},
"F2/F1 Ratio": {
"field": "f2_f1_ratio",
"dist": {"mu": 2.0, "sigma": 0.5, "lo": 1.0, "hi": 4.0},
"invert": False, # higher ratio = brighter
"unit": "",
"source": "Derived; normalizes vowel-height confound",
},
"Composite Z (default)": {
"field": "resonance_combo",
"dist": {"mu": 0.0, "sigma": 1.0, "lo": -3.0, "hi": 3.0},
"invert": False, # z-score: positive = brighter
"unit": "z",
"source": "Z-scored mean of F2−F1 + Df",
},
"Spectral Mean (Hz)": {
"field": "spectral_mean_hz",
"dist": {"mu": 1400.0, "sigma": 350.0, "lo": 600.0, "hi": 3000.0},
"invert": False, # higher CoG = brighter
"unit": "Hz",
"source": "Power-spectrum center of gravity above F0; formant-free",
},
"F3 (Hz)": {
"field": "f3_hz",
"dist": {"mu": 2800.0, "sigma": 250.0, "lo": 2000.0, "hi": 3800.0},
"invert": False, # higher F3 = shorter pharynx = brighter
"unit": "Hz",
"source": "Direct pharyngeal-space cue; Vorperian & Kent 2007",
},
}
RESONANCE_METHOD_DEFAULT = "Composite Z (default)"
DEFAULT_SR = 22050
FRAME_LEN = 0.040
HOP_LEN = 0.010
PITCH_FLOOR_DEFAULT = 60
PITCH_CEIL_DEFAULT = 500
MAX_FORMANT_DEFAULT = 5500
# Speed of sound in cm/s for VTL estimation (Fitch 1997)
SPEED_OF_SOUND_CM = 35300.0
HISTORY_CAP = 200
MAX_UPLOAD_SECONDS = 120
MAX_LIVE_BUFFER_SECONDS = 12
LIVE_ANALYZE_SECONDS = 8
EXPORT_DIR = "exports"
PROFILE_JSON_PATH = os.path.join(EXPORT_DIR, "profile.json")
HISTORY_CSV_PATH = os.path.join(EXPORT_DIR, "history.csv")
BASELINE_JSON_PATH = os.path.join(EXPORT_DIR, "baseline.json")
# =============================================================================
# CSS - colour-coded sliders matching the Concept Slider design
# =============================================================================
CUSTOM_CSS = """
@import url('https://fonts.googleapis.com/css2?family=DM+Sans:ital,opsz,wght@0,9..40,300;0,9..40,500;0,9..40,700;1,9..40,400&family=JetBrains+Mono:wght@400;600&display=swap');
:root {
--vts-pitch: #5b7de8;
--vts-weight: #9b6dba;
--vts-resonance: #e8956e;
--vts-prosody: #d4c24a;
--vts-volume: #e878c8;
--vts-surface: rgba(255,255,255,0.04);
--vts-border: rgba(255,255,255,0.08);
--vts-text: rgba(255,255,255,0.88);
--vts-muted: rgba(255,255,255,0.45);
--vts-radius: 16px;
}
.gradio-container {
max-width: 960px !important;
font-family: 'DM Sans', -apple-system, sans-serif !important;
}
/* --- Header --- */
.vts-header {
text-align: center;
padding: 2.2rem 1rem 1.4rem;
position: relative;
overflow: hidden;
}
.vts-header::before {
content: '';
position: absolute;
inset: 0;
background: radial-gradient(ellipse 70% 50% at 50% 0%, rgba(91,125,232,0.12), transparent),
radial-gradient(ellipse 50% 40% at 80% 20%, rgba(232,135,110,0.08), transparent);
pointer-events: none;
}
.vts-header h1 {
font-family: 'DM Sans', sans-serif;
font-weight: 700;
font-size: 2rem;
letter-spacing: -0.03em;
margin: 0 0 0.3rem;
background: linear-gradient(135deg, var(--vts-pitch), var(--vts-resonance));
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
background-clip: text;
}
.vts-header p {
color: var(--vts-muted);
font-size: 0.92rem;
line-height: 1.5;
margin: 0;
}
.vts-header .vts-tag {
display: inline-block;
margin-top: 0.6rem;
padding: 0.25rem 0.75rem;
border-radius: 100px;
font-size: 0.72rem;
font-weight: 500;
letter-spacing: 0.04em;
text-transform: uppercase;
background: rgba(91,125,232,0.12);
color: var(--vts-pitch);
border: 1px solid rgba(91,125,232,0.2);
}
/* --- Tabs --- */
.tabs > .tab-nav > button {
font-family: 'DM Sans', sans-serif !important;
font-weight: 600 !important;
font-size: 0.95rem !important;
letter-spacing: -0.01em !important;
}
/* --- Slider track overrides --- */
.slider-pitch input[type=range]::-webkit-slider-runnable-track { background: linear-gradient(90deg, #2a3578, #5b7de8) !important; border-radius: 6px !important; }
.slider-pitch input[type=range]::-moz-range-track { background: linear-gradient(90deg, #2a3578, #5b7de8) !important; border-radius: 6px !important; }
.slider-pitch input[type=range]::-webkit-slider-thumb { background: var(--vts-pitch) !important; }
.slider-pitch input[type=range]::-moz-range-thumb { background: var(--vts-pitch) !important; }
.slider-pitch label span, .slider-pitch .label-wrap span { color: var(--vts-pitch) !important; font-weight: 600 !important; }
.slider-weight input[type=range]::-webkit-slider-runnable-track { background: linear-gradient(90deg, #3d1f5c, #9b6dba) !important; border-radius: 6px !important; }
.slider-weight input[type=range]::-moz-range-track { background: linear-gradient(90deg, #3d1f5c, #9b6dba) !important; border-radius: 6px !important; }
.slider-weight input[type=range]::-webkit-slider-thumb { background: var(--vts-weight) !important; }
.slider-weight input[type=range]::-moz-range-thumb { background: var(--vts-weight) !important; }
.slider-weight label span, .slider-weight .label-wrap span { color: var(--vts-weight) !important; font-weight: 600 !important; }
.slider-resonance input[type=range]::-webkit-slider-runnable-track { background: linear-gradient(90deg, #8b4a2a, #e8956e) !important; border-radius: 6px !important; }
.slider-resonance input[type=range]::-moz-range-track { background: linear-gradient(90deg, #8b4a2a, #e8956e) !important; border-radius: 6px !important; }
.slider-resonance input[type=range]::-webkit-slider-thumb { background: var(--vts-resonance) !important; }
.slider-resonance input[type=range]::-moz-range-thumb { background: var(--vts-resonance) !important; }
.slider-resonance label span, .slider-resonance .label-wrap span { color: var(--vts-resonance) !important; font-weight: 600 !important; }
.slider-prosody input[type=range]::-webkit-slider-runnable-track { background: linear-gradient(90deg, #6b5a10, #d4c24a) !important; border-radius: 6px !important; }
.slider-prosody input[type=range]::-moz-range-track { background: linear-gradient(90deg, #6b5a10, #d4c24a) !important; border-radius: 6px !important; }
.slider-prosody input[type=range]::-webkit-slider-thumb { background: var(--vts-prosody) !important; }
.slider-prosody input[type=range]::-moz-range-thumb { background: var(--vts-prosody) !important; }
.slider-prosody label span, .slider-prosody .label-wrap span { color: var(--vts-prosody) !important; font-weight: 600 !important; }
.slider-volume input[type=range]::-webkit-slider-runnable-track { background: linear-gradient(90deg, #8a2070, #e878c8) !important; border-radius: 6px !important; }
.slider-volume input[type=range]::-moz-range-track { background: linear-gradient(90deg, #8a2070, #e878c8) !important; border-radius: 6px !important; }
.slider-volume input[type=range]::-webkit-slider-thumb { background: var(--vts-volume) !important; }
.slider-volume input[type=range]::-moz-range-thumb { background: var(--vts-volume) !important; }
.slider-volume label span, .slider-volume .label-wrap span { color: var(--vts-volume) !important; font-weight: 600 !important; }
.slider-confidence input[type=range]::-webkit-slider-runnable-track { background: linear-gradient(90deg, #444, #888) !important; border-radius: 6px !important; }
.slider-confidence input[type=range]::-moz-range-track { background: linear-gradient(90deg, #444, #888) !important; border-radius: 6px !important; }
/* --- Prompt box --- */
.prompt-box textarea {
font-size: 1.02em !important;
line-height: 1.65 !important;
font-family: 'DM Sans', sans-serif !important;
opacity: 0.85;
}
/* --- Coaching card styling --- */
.coaching-card p { line-height: 1.6 !important; }
.coaching-card strong { letter-spacing: -0.01em; }
/* --- Clean up accordions --- */
.gradio-accordion { border: 1px solid var(--vts-border) !important; border-radius: var(--vts-radius) !important; }
/* --- Monospace for reports --- */
.report-box textarea { font-family: 'JetBrains Mono', monospace !important; font-size: 0.82em !important; }
"""
# =============================================================================
# JSON helper - handles NaN / Inf that stdlib json cannot serialise
# =============================================================================
class _SafeEncoder(json.JSONEncoder):
def encode(self, o):
return super().encode(self._sanitise(o))
def _sanitise(self, o):
# Handle numpy scalars first (np.float32 is NOT a subclass of float)
if isinstance(o, (np.floating, float)):
if math.isnan(float(o)) or math.isinf(float(o)):
return None
return float(o)
if isinstance(o, np.integer):
return int(o)
if isinstance(o, dict):
return {k: self._sanitise(v) for k, v in o.items()}
if isinstance(o, (list, tuple)):
return [self._sanitise(v) for v in o]
return o
def _json_dump(obj, fh, **kw):
json.dump(obj, fh, cls=_SafeEncoder, **kw)
# =============================================================================
# Utilities
# =============================================================================
def ensure_exports_dir():
os.makedirs(EXPORT_DIR, exist_ok=True)
def sanitize_audio(x):
x = np.asarray(x, dtype=np.float32)
x = np.nan_to_num(x, nan=0.0, posinf=0.0, neginf=0.0)
if x.size:
x = x - float(np.mean(x))
return np.clip(x, -1.0, 1.0)
def to_mono_float32(y):
if y is None:
return None
x = np.asarray(y)
if x.size == 0:
return None
x = x.astype(np.float32, copy=False)
if x.ndim == 2:
x = np.mean(x, axis=0) if x.shape[0] < x.shape[1] else np.mean(x, axis=1)
mx = float(np.max(np.abs(x))) if x.size else 0.0
if mx > 1.5:
x = x / mx
return sanitize_audio(x)
def normalize_audio_tuple(audio_like):
if audio_like is None:
return None
if isinstance(audio_like, (tuple, list)) and len(audio_like) == 2:
sr, y = audio_like
try:
sr = int(sr) if sr is not None else DEFAULT_SR
except Exception:
sr = DEFAULT_SR
y = np.asarray(y, dtype=np.float32)
return sr, y
if isinstance(audio_like, np.ndarray):
return DEFAULT_SR, audio_like.astype(np.float32, copy=False)
try:
arr = np.asarray(audio_like, dtype=np.float32)
if arr.ndim >= 1 and arr.size > 0:
return DEFAULT_SR, arr
except Exception:
pass
return None
def resample_if_needed(y, sr, target_sr=DEFAULT_SR):
if sr != target_sr:
y = librosa.resample(y, orig_sr=sr, target_sr=target_sr)
return y.astype(np.float32), target_sr
return y, sr
def hard_cap_length(y, sr, max_seconds):
if y is None or y.size == 0:
return y
max_len = int(max_seconds * sr)
return y[-max_len:] if y.size > max_len else y
def select_analysis_window(y, sr, window_choice):
if y is None or y.size == 0:
return y
if window_choice == "Full":
return y
secs = {"Last 10s": 10, "Last 20s": 20}.get(window_choice, 0)
if secs > 0:
n = int(secs * sr)
return y[-n:] if y.size > n else y
return y
def dbfs_rms(x):
eps = 1e-12
rms = np.sqrt(np.mean(np.square(x)) + eps)
return float(20.0 * np.log10(rms + eps))
def detect_clipping(x, thresh=0.999):
if x is None or x.size == 0:
return 0.0
return float(np.mean(np.abs(x) >= thresh))
def estimate_snr_db(y, sr):
frame = int(FRAME_LEN * sr)
hop = int(HOP_LEN * sr)
if y.size < frame:
return float("nan")
try:
rms = librosa.feature.rms(y=y, frame_length=frame, hop_length=hop)[0]
except Exception:
return float("nan")
if rms.size < 10:
return float("nan")
rms_sorted = np.sort(rms)
lo = np.mean(rms_sorted[: max(1, rms_sorted.size // 5)])
hi = np.mean(rms_sorted[-max(1, rms_sorted.size // 5):])
eps = 1e-12
return float(20.0 * np.log10((hi + eps) / (lo + eps)))
def normal_cdf(z):
return 0.5 * (1.0 + math.erf(z / math.sqrt(2.0)))
def pct_to_slider(p):
if not np.isfinite(p):
return float("nan")
return float(np.clip(p, 0.0, 1.0) * 100.0)
def safe(v, fallback=0.0):
return float(v) if np.isfinite(v) else float(fallback)
def cap_history(history):
return (history or [])[-HISTORY_CAP:]
def audio_hash(y, sr, extra=""):
h = hashlib.sha256()
h.update(str(sr).encode())
h.update(extra.encode())
x = y
if x.size > 32000:
x = x[::max(1, x.size // 32000)]
h.update(x.tobytes())
return h.hexdigest()
def apply_preemphasis(y, coef=0.97):
try:
return librosa.effects.preemphasis(y, coef=coef).astype(np.float32)
except Exception:
return y
def loudness_normalize_dbfs(y, target_dbfs=-20.0):
cur = dbfs_rms(y)
if not np.isfinite(cur) or cur < -45.0:
return y
gain = float(10.0 ** ((target_dbfs - cur) / 20.0))
return np.clip(y * gain, -1.0, 1.0).astype(np.float32)
def preprocess_audio(y, use_preemph, use_loudnorm):
if use_preemph:
y = apply_preemphasis(y)
if use_loudnorm:
y = loudness_normalize_dbfs(y)
return y.astype(np.float32)
def robust_center_scale(x):
x = np.asarray(x, dtype=np.float32)
x = x[np.isfinite(x)]
if x.size < 5:
return float("nan"), float("nan")
med = float(np.median(x))
iqr = float(np.percentile(x, 75) - np.percentile(x, 25))
sigma = float(max(iqr / 1.349, 1e-6))
return med, sigma
def robust_clip(x, k=3.5):
mu, sigma = robust_center_scale(x)
if not (np.isfinite(mu) and np.isfinite(sigma)):
return x
lo, hi = mu - k * sigma, mu + k * sigma
return x[(x >= lo) & (x <= hi)]
def history_preview(history, max_rows=20):
if not history:
return []
cols = ["timestamp", "label", "confidence",
"Pitch_slider", "Weight_slider", "Resonance_slider",
"Prosody_slider", "Volume_slider"]
rows = history[-max_rows:]
out = []
for r in rows:
row = []
for c in cols:
v = r.get(c, "")
if isinstance(v, float):
if math.isnan(v) or math.isinf(v):
row.append("\u2014")
elif c == "confidence":
row.append(round(v, 2))
else:
row.append(round(v, 1))
else:
row.append(v)
out.append(row)
return out
def _now_iso():
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# =============================================================================
# Metrics
# =============================================================================
@dataclass
class Metrics:
# Pitch
f0_med_hz: float
f0_iqr_hz: float
f0_sd_st: float
voicing_ratio: float
# Prosody (composite)
f0_velocity_st_s: float # pitch velocity: semitones per second
f0_direction_rate: float # directional changes per second
prosody_composite: float # weighted combination for slider
# Volume / signal quality
rms_dbfs: float
snr_db: float
clipping_ratio: float
# Formants / resonance
f1_hz: float
f2_hz: float
f3_hz: float
f2_minus_f1_hz: float
formant_dispersion_hz: float # (F3 - F1) / 2
vtl_cm: float # estimated vocal tract length
f2_f1_ratio: float # F2/F1, dimensionless resonance ratio
resonance_combo: float # z-scored composite of F2-F1 + Df
spectral_mean_hz: float # center of gravity of power spectrum above F0
# Voice quality / weight
h1_h2_db: float # uncorrected H1-H2
h1_h2_corrected_db: float # H1*-H2* with Iseli-Alwan formant correction
cpp_db: float # real Cepstral Peak Prominence
jitter_local: float # local jitter (%)
shimmer_local: float # local shimmer (%)
hnr_db: float # harmonics-to-noise ratio
# Meta
confidence: float
warnings: List[str]
def empty_metrics(warnings=None):
nan = float("nan")
return Metrics(
nan, nan, nan, nan, # pitch
nan, nan, nan, # prosody
nan, nan, nan, # volume
nan, nan, nan, nan, nan, nan, nan, nan, nan, # formants/resonance (9)
nan, nan, nan, nan, nan, nan, # voice quality
0.0, warnings or [], # meta
)
def _compute_cpp(y, sr, f0_med, frame_len_samp, hop_samp,
pitch_floor_hz, pitch_ceil_hz):
"""Compute Cepstral Peak Prominence (Hillenbrand et al. 1994).
CPP = height of the cepstral peak (at the expected quefrency for F0)
above a regression line fitted to the cepstrum. This is the gold-standard
voice quality metric, distinct from HNR.
"""
if not np.isfinite(f0_med) or f0_med <= 0:
return float("nan")
# Quefrency range corresponding to plausible F0
q_lo = int(sr / min(pitch_ceil_hz, 600)) # high pitch → low quefrency
q_hi = int(sr / max(pitch_floor_hz, 50)) # low pitch → high quefrency
if q_lo >= q_hi or q_hi <= 0:
return float("nan")
n_fft = max(2048, int(2 ** np.ceil(np.log2(frame_len_samp * 2))))
window = np.hanning(frame_len_samp).astype(np.float32)
cpp_vals = []
# Process frames
for start in range(0, max(1, y.size - frame_len_samp), hop_samp * 4):
seg = y[start:start + frame_len_samp]
if seg.size < frame_len_samp:
break
seg_w = seg * window
# Power spectrum → log → IFFT = power cepstrum
spec = np.fft.rfft(seg_w, n=n_fft)
log_power = np.log(np.abs(spec) ** 2 + 1e-20)
cepstrum = np.fft.irfft(log_power)
# Extract the quefrency range of interest
if q_hi >= len(cepstrum):
continue
cep_region = cepstrum[q_lo:q_hi + 1]
if cep_region.size < 3:
continue
# Find cepstral peak
peak_idx_local = int(np.argmax(cep_region))
peak_val = float(cep_region[peak_idx_local])
# Fit regression line to cepstrum in the region
x_reg = np.arange(cep_region.size, dtype=np.float64)
if x_reg.size >= 2:
coeffs = np.polyfit(x_reg, cep_region, 1)
regression_at_peak = float(np.polyval(coeffs, peak_idx_local))
cpp_frame = peak_val - regression_at_peak
if np.isfinite(cpp_frame):
cpp_vals.append(cpp_frame)
if not cpp_vals:
return float("nan")
return float(np.median(cpp_vals))
def extract_distributions(y, sr, pitch_floor_hz, pitch_ceil_hz, max_formant_hz):
"""Extract comprehensive acoustic metrics with clinical-grade accuracy.
Improvements over naive approach:
1. to_pitch_cc(very_accurate=True) for robust F0 estimation
2. Adaptive max_formant based on detected F0 range
3. Formant bandwidth-based vowel detection
4. F3 tracking + formant dispersion + VTL estimation for resonance
5. H1*-H2* with Iseli-Alwan formant correction for vocal weight
6. Real CPP from power cepstrum
7. Jitter and shimmer from Praat for voice quality / confidence
8. Prosody composite: pitch velocity + directional change rate
9. n_fft=4096 for better frequency resolution in H1-H2
10. Composite resonance metric (z-scored F2-F1 + formant dispersion)
"""
warnings = []
if y is None or y.size == 0:
return empty_metrics(["No audio."]), {}
try:
y_trim, _ = librosa.effects.trim(y, top_db=35)
if y_trim.size >= int(0.25 * sr):
y = y_trim
else:
warnings.append("Trim removed most audio; using untrimmed signal.")
except Exception:
warnings.append("Trim failed; using untrimmed signal.")
audio_duration_s = y.size / sr
if audio_duration_s < 0.5:
warnings.append("Audio is short (<0.5 s). Results may be unstable.")
rms_db = dbfs_rms(y)
clip_ratio = detect_clipping(y)
snr = estimate_snr_db(y, sr)
if np.isfinite(snr) and snr < 10:
warnings.append("High noise / low SNR. Record in a quieter space.")
if clip_ratio > 0.002:
warnings.append("Clipping detected. Lower input gain or increase mic distance.")
try:
snd = parselmouth.Sound(y, sampling_frequency=sr)
except Exception:
m = empty_metrics(warnings + ["Praat Sound creation failed."])
m.rms_dbfs, m.snr_db, m.clipping_ratio = rms_db, snr, clip_ratio
return m, {}
# ── 1. Pitch: cross-correlation with very_accurate ──
try:
pitch = snd.to_pitch_cc(time_step=HOP_LEN, pitch_floor=pitch_floor_hz,
pitch_ceiling=pitch_ceil_hz, very_accurate=True)
f0 = pitch.selected_array["frequency"]
times = pitch.xs()
except Exception:
# Fallback to standard autocorrelation if cc fails
try:
pitch = snd.to_pitch(time_step=HOP_LEN, pitch_floor=pitch_floor_hz,
pitch_ceiling=pitch_ceil_hz)
f0 = pitch.selected_array["frequency"]
times = pitch.xs()
warnings.append("Fell back to autocorrelation pitch tracker.")
except Exception:
m = empty_metrics(warnings + ["Pitch extraction failed."])
m.rms_dbfs, m.snr_db, m.clipping_ratio = rms_db, snr, clip_ratio
return m, {}
if f0.size == 0:
m = empty_metrics(warnings + ["Pitch output empty."])
m.rms_dbfs, m.snr_db, m.clipping_ratio = rms_db, snr, clip_ratio
return m, {}
n = min(len(f0), len(times))
f0, times = f0[:n], times[:n]
voiced = (f0 > 0) & np.isfinite(f0)
f0_voiced = f0[voiced].astype(np.float32)
f0_voiced = f0_voiced[(f0_voiced >= pitch_floor_hz) & (f0_voiced <= pitch_ceil_hz)]
# Median filter to remove isolated octave errors
if f0_voiced.size >= 9:
f0_voiced = librosa.decompose.nn_filter(f0_voiced[None, :], aggregate=np.median)[0]
voicing_ratio = float(np.mean(voiced))
if voicing_ratio < 0.35:
warnings.append("Low voicing ratio. Speak more continuously for better estimates.")
if f0_voiced.size == 0:
m = empty_metrics(warnings + ["No voiced frames detected."])
m.voicing_ratio = voicing_ratio
m.rms_dbfs, m.snr_db, m.clipping_ratio = rms_db, snr, clip_ratio
return m, {}
f0_med = float(np.median(f0_voiced))
f0_iqr = float(np.percentile(f0_voiced, 75) - np.percentile(f0_voiced, 25))
st = 12.0 * np.log2(f0_voiced / max(f0_med, 1e-6))
f0_sd_st = float(np.std(st))
# ── 7. Prosody dynamics: velocity + directional changes ──
# Use the time-ordered voiced F0 contour for temporal features
voiced_times = times[voiced][:f0_voiced.size]
f0_velocity_st_s = float("nan")
f0_direction_rate = float("nan")
prosody_composite = f0_sd_st # fallback: just SD
if f0_voiced.size >= 5 and voiced_times.size >= 5:
# Semitone contour
st_contour = 12.0 * np.log2(f0_voiced / max(f0_med, 1e-6))
dt = np.diff(voiced_times)
dt = np.maximum(dt, 1e-6) # avoid divide-by-zero
dst = np.abs(np.diff(st_contour))
velocities = dst / dt
f0_velocity_st_s = float(np.median(velocities))
# Directional changes: sign changes in pitch derivative
pitch_diff = np.diff(st_contour)
signs = np.sign(pitch_diff)
signs = signs[signs != 0] # remove flat segments
if signs.size >= 2:
direction_changes = float(np.sum(np.abs(np.diff(signs)) > 0))
total_time = voiced_times[-1] - voiced_times[0]
if total_time > 0.1:
f0_direction_rate = direction_changes / total_time
# Composite: weighted combination (SD 60%, velocity 25%, direction 15%)
# Normalize velocity to semitone-scale (typical range: 5-30 st/s)
vel_norm = float(np.clip(f0_velocity_st_s / 5.0, 0, 6.0)) if np.isfinite(f0_velocity_st_s) else f0_sd_st
dir_norm = float(np.clip(f0_direction_rate / 3.0, 0, 6.0)) if np.isfinite(f0_direction_rate) else f0_sd_st
prosody_composite = 0.60 * f0_sd_st + 0.25 * vel_norm + 0.15 * dir_norm
# ── RMS frames ──
frame = int(FRAME_LEN * sr)
hop = int(HOP_LEN * sr)
if y.size >= frame:
rms_frames = librosa.feature.rms(y=y, frame_length=frame, hop_length=hop)[0]
rms_frames_dbfs = (20.0 * np.log10(np.maximum(rms_frames, 1e-12))).astype(np.float32)
else:
rms_frames_dbfs = np.array([rms_db], dtype=np.float32)
# ── 2. Adaptive formant settings based on detected F0 ──
if f0_med < 140:
adaptive_max_formant = min(max_formant_hz, 5000)
adaptive_num_formants = 5
elif f0_med > 200:
adaptive_max_formant = max(max_formant_hz, 5500)
adaptive_num_formants = 5
else:
adaptive_max_formant = max_formant_hz
adaptive_num_formants = 5
# ── 3. Vowel mask with formant bandwidth check ──
formant = None
try:
formant = snd.to_formant_burg(time_step=HOP_LEN,
max_number_of_formants=adaptive_num_formants,
maximum_formant=adaptive_max_formant)
except Exception:
warnings.append("Formant tracking failed (resonance unavailable).")
# Build improved vowel mask: voiced + ZCR + flatness + bandwidth
if y.size < frame:
vowel_mask = voiced.copy()
else:
try:
zcr = librosa.feature.zero_crossing_rate(y, frame_length=frame, hop_length=hop)[0]
n_fft = max(512, int(2 ** np.ceil(np.log2(frame))))
flat = librosa.feature.spectral_flatness(y=y, n_fft=n_fft, hop_length=hop)[0]
n_frames = min(len(zcr), len(flat), len(rms_frames_dbfs))
zcr, flat = zcr[:n_frames], flat[:n_frames]
idx = np.clip(np.rint(times * sr / hop).astype(int), 0, n_frames - 1)
# Base mask: voiced + low ZCR + low flatness
vowel_mask = voiced & (zcr[idx] < 0.12) & (flat[idx] < 0.35)
# ── 5. Add formant bandwidth criterion ──
if formant is not None:
bw_ok = np.ones(len(times), dtype=bool)
for i, t in enumerate(times):
if not vowel_mask[i]:
continue
b1 = formant.get_bandwidth_at_time(1, float(t))
if np.isfinite(b1) and b1 > 400.0:
bw_ok[i] = False # Wide B1 → not a clean vowel
vowel_mask = vowel_mask & bw_ok
except Exception:
vowel_mask = voiced.copy()
warnings.append("Frame features failed; using voiced-only mask.")
# ── 4. Formant extraction: F1, F2, F3 + dispersion + VTL ──
f1_vals, f2_vals, f3_vals, f2_minus_f1_vals, df_vals = [], [], [], [], []
if formant is not None:
for t, ok in zip(times, vowel_mask):
if not ok:
continue
f1 = formant.get_value_at_time(1, float(t))
f2 = formant.get_value_at_time(2, float(t))
f3 = formant.get_value_at_time(3, float(t))
if not (np.isfinite(f1) and np.isfinite(f2) and
150 < f1 < 1200 and 500 < f2 < 3500):
continue
f1_vals.append(float(f1))
f2_vals.append(float(f2))
f2_minus_f1_vals.append(float(f2 - f1))
if np.isfinite(f3) and f3 > f2 and f3 < 4500:
f3_vals.append(float(f3))
df_vals.append(float((f3 - f1) / 2.0))
f2m = np.array(f2_minus_f1_vals, dtype=np.float32)
if f2m.size:
f2m = robust_clip(f2m)
if formant is None or f2m.size < 10:
warnings.append("Few stable vowel frames for formants. Resonance estimate may be noisy.")
f1_med = float(np.median(f1_vals)) if f1_vals else float("nan")
f2_med = float(np.median(f2_vals)) if f2_vals else float("nan")
f3_med = float(np.median(f3_vals)) if f3_vals else float("nan")
f2_minus_f1_med = float(np.median(f2m)) if f2m.size else float("nan")
# Formant dispersion (Fitch 1997)
df_arr = np.array(df_vals, dtype=np.float32)
if df_arr.size:
df_arr = robust_clip(df_arr)
formant_disp_med = float(np.median(df_arr)) if df_arr.size else float("nan")
# ── 6. VTL estimation: VTL = c / (4 * Δf) ──
vtl_cm = float("nan")
if np.isfinite(formant_disp_med) and formant_disp_med > 50:
vtl_cm = SPEED_OF_SOUND_CM / (4.0 * formant_disp_med)
# Composite resonance: z-score F2-F1 and Df, then average
# Normative: F2-F1 mu≈900 σ≈200, Df mu≈1050 σ≈150
resonance_combo = float("nan")
z_parts = []
if np.isfinite(f2_minus_f1_med):
z_parts.append((f2_minus_f1_med - 900.0) / 200.0)
if np.isfinite(formant_disp_med):
z_parts.append((formant_disp_med - 1050.0) / 150.0)
if z_parts:
resonance_combo = float(np.mean(z_parts))
# ── F2/F1 ratio (dimensionless resonance) ──
f2_f1_ratio = float("nan")
if f1_vals and f2_vals:
f1_arr_r = np.array(f1_vals, dtype=np.float32)
f2_arr_r = np.array(f2_vals, dtype=np.float32)
# Avoid division by zero; exclude extreme F1 values
valid = f1_arr_r > 200
if valid.sum() > 3:
ratios = f2_arr_r[valid] / f1_arr_r[valid]
ratios = robust_clip(ratios)
f2_f1_ratio = float(np.median(ratios))
# ── Spectral mean / center of gravity above F0 (formant-free resonance) ──
spectral_mean_hz = float("nan")
if np.isfinite(f0_med) and y.size >= frame:
# Compute average power spectrum across voiced frames, then find CoG
n_fft_cog = 4096
freqs_cog = np.fft.rfftfreq(n_fft_cog, d=1.0 / sr)
window_cog = np.hanning(frame).astype(np.float32)
sum_power = np.zeros(n_fft_cog // 2 + 1, dtype=np.float64)
n_frames_cog = 0
centers_cog = (times * sr).astype(int)
half_cog = frame // 2
for i, (c, v) in enumerate(zip(centers_cog, voiced)):
if not v:
continue
s0 = max(0, c - half_cog)
s1 = s0 + frame
if s1 > y.size:
continue
seg = y[s0:s1] * window_cog
power = np.abs(np.fft.rfft(seg, n=n_fft_cog)) ** 2
sum_power += power
n_frames_cog += 1
if n_frames_cog > 5:
avg_power = sum_power / n_frames_cog
# Mask below 1.5 × F0 to exclude fundamental and first harmonic
f_cutoff = 1.5 * f0_med
mask_above_f0 = freqs_cog >= f_cutoff
if mask_above_f0.sum() > 10:
p_above = avg_power[mask_above_f0]
f_above = freqs_cog[mask_above_f0]
total = p_above.sum()
if total > 0:
spectral_mean_hz = float(np.sum(f_above * p_above) / total)
# ── 9. H1*-H2* with Iseli-Alwan formant correction ──
h1h2_raw_vals = []
h1h2_corrected_vals = []
if y.size >= frame:
n_fft2 = 4096 # ← doubled from 2048 for better frequency resolution
window = np.hanning(frame).astype(np.float32)
centers = (times * sr).astype(int)
half = frame // 2
freqs = np.fft.rfftfreq(n_fft2, d=1.0 / sr)
def amp_db_at(mag, f_target):
"""Get amplitude in dB at frequency f_target with parabolic interpolation."""
i0 = int(np.clip(int(np.argmin(np.abs(freqs - f_target))), 1, mag.size - 2))
# Parabolic interpolation around peak
a, b, c = float(mag[i0 - 1]), float(mag[i0]), float(mag[i0 + 1])
if b > 0 and b >= a and b >= c:
val = b
else:
val = float(np.mean(mag[max(0, i0 - 1):i0 + 2]))
return float(20.0 * np.log10(val + 1e-12)), (b >= a and b >= c)
def formant_correction_db(f_signal, f_formant, b_formant):
"""Iseli-Alwan (2004) single-formant transfer function magnitude in dB.
H(f) = 1 / sqrt((1 - (f/F)^2)^2 + (f*B/(F^2))^2)
Returns 20*log10(|H(f)|)."""
if not (np.isfinite(f_formant) and np.isfinite(b_formant)
and f_formant > 0 and b_formant > 0):
return 0.0
r = f_signal / f_formant
term1 = (1.0 - r * r) ** 2
term2 = (f_signal * b_formant / (f_formant * f_formant)) ** 2
denom = term1 + term2
if denom < 1e-12:
return 0.0
return float(-10.0 * np.log10(denom))
for t, ok, c in zip(times, vowel_mask, centers):
if not ok or c - half < 0 or c + half >= y.size:
continue
f0_t = float(pitch.get_value_at_time(float(t)))
if not np.isfinite(f0_t) or f0_t <= 0 or 2.0 * f0_t >= freqs[-1]:
continue
seg_w = y[c - half:c + half] * window
mag = np.abs(np.fft.rfft(seg_w, n=n_fft2))
h1_db, h1_peak = amp_db_at(mag, f0_t)
h2_db, h2_peak = amp_db_at(mag, 2.0 * f0_t)
if not (h1_peak and h2_peak and np.isfinite(h1_db) and np.isfinite(h2_db)):
continue
h1h2_raw = h1_db - h2_db
h1h2_raw_vals.append(h1h2_raw)
# Apply Iseli-Alwan correction using F1 and B1
if formant is not None:
f1_t = formant.get_value_at_time(1, float(t))
b1_t = formant.get_bandwidth_at_time(1, float(t))
corr_h1 = formant_correction_db(f0_t, f1_t, b1_t)
corr_h2 = formant_correction_db(2.0 * f0_t, f1_t, b1_t)
h1_star = h1_db - corr_h1
h2_star = h2_db - corr_h2
h1h2_corrected_vals.append(h1_star - h2_star)
else:
h1h2_corrected_vals.append(h1h2_raw)
h12_raw = np.array(h1h2_raw_vals, dtype=np.float32)
h12_corr = np.array(h1h2_corrected_vals, dtype=np.float32)
if h12_raw.size:
h12_raw = robust_clip(h12_raw)
if h12_corr.size:
h12_corr = robust_clip(h12_corr)
if h12_corr.size < 10:
warnings.append("Few stable frames for H1*-H2*. Weight estimate may be noisy.")
h1_h2_med = float(np.median(h12_raw)) if h12_raw.size else float("nan")
h1_h2_corrected_med = float(np.median(h12_corr)) if h12_corr.size else float("nan")
# ── 6. Real CPP from power cepstrum ──
cpp_db_val = float("nan")
try:
cpp_db_val = _compute_cpp(y, sr, f0_med, frame, hop, pitch_floor_hz, pitch_ceil_hz)
except Exception:
warnings.append("CPP computation failed; using NaN.")
# ── HNR (keep as supplementary metric) ──
hnr_db = float("nan")
try:
harm = snd.to_harmonicity_cc(time_step=HOP_LEN, minimum_pitch=pitch_floor_hz)
harm_vals = np.array(harm.values).flatten()
harm_vals = harm_vals[np.isfinite(harm_vals) & (harm_vals > -200)]
hnr_db = float(np.median(harm_vals)) if harm_vals.size else float("nan")
except Exception:
warnings.append("Harmonicity calculation failed.")
# ── 7. Jitter and shimmer from Praat ──
jitter_local = float("nan")
shimmer_local = float("nan")
try:
pp = parselmouth.praat.call(snd, "To PointProcess (periodic, cc)...",
pitch_floor_hz, pitch_ceil_hz)
jitter_local = float(parselmouth.praat.call(pp, "Get jitter (local)", 0.0, 0.0,
0.0001, 0.02, 1.3) * 100.0)
shimmer_local = float(parselmouth.praat.call(
[snd, pp], "Get shimmer (local)", 0.0, 0.0, 0.0001, 0.02, 1.3, 1.6) * 100.0)
except Exception:
pass # Jitter/shimmer are supplementary — silently skip
# ── Confidence scoring (improved) ──
conf_parts = [
# Voicing quality (weight: 0.20)
float(np.clip((voicing_ratio - 0.2) / 0.6, 0.0, 1.0)),
# SNR (weight: 0.15)
float(np.clip((snr - 5.0) / 20.0, 0.0, 1.0)) if np.isfinite(snr) else 0.5,
# No clipping (weight: 0.10)
float(1.0 - np.clip(clip_ratio / 0.01, 0.0, 1.0)),
# Formant frame count (weight: 0.15)
float(np.clip(f2m.size / 50.0, 0.0, 1.0)),
# H1-H2 frame count (weight: 0.15)
float(np.clip(h12_corr.size / 50.0, 0.0, 1.0)),
# Audio duration (weight: 0.15) — at least 3s, ideal 8+s
float(np.clip((audio_duration_s - 1.0) / 7.0, 0.0, 1.0)),
# Low jitter → good quality (weight: 0.10)
float(np.clip(1.0 - jitter_local / 5.0, 0.0, 1.0)) if np.isfinite(jitter_local) else 0.5,
]
conf_weights = [0.20, 0.15, 0.10, 0.15, 0.15, 0.15, 0.10]
confidence = float(sum(c * w for c, w in zip(conf_parts, conf_weights)))
m = Metrics(
f0_med_hz=f0_med, f0_iqr_hz=f0_iqr, f0_sd_st=f0_sd_st,
voicing_ratio=voicing_ratio,
f0_velocity_st_s=f0_velocity_st_s,
f0_direction_rate=f0_direction_rate,
prosody_composite=prosody_composite,
rms_dbfs=rms_db, snr_db=snr, clipping_ratio=clip_ratio,
f1_hz=f1_med, f2_hz=f2_med, f3_hz=f3_med,
f2_minus_f1_hz=f2_minus_f1_med,
formant_dispersion_hz=formant_disp_med,
vtl_cm=vtl_cm,
f2_f1_ratio=f2_f1_ratio,
resonance_combo=resonance_combo,
spectral_mean_hz=spectral_mean_hz,
h1_h2_db=h1_h2_med,
h1_h2_corrected_db=h1_h2_corrected_med,
cpp_db=cpp_db_val,
jitter_local=jitter_local,
shimmer_local=shimmer_local,
hnr_db=hnr_db,
confidence=confidence, warnings=warnings,
)
dists = {
"f0_voiced": np.asarray(f0_voiced, dtype=np.float32),
"st": np.asarray(st, dtype=np.float32),
"rms_frames_dbfs": np.asarray(rms_frames_dbfs, dtype=np.float32),
"f2_minus_f1_vals": np.asarray(f2m, dtype=np.float32),
"h1_h2_corrected": np.asarray(h12_corr, dtype=np.float32),
"resonance_combo": np.asarray([resonance_combo], dtype=np.float32) if np.isfinite(resonance_combo) else np.array([], dtype=np.float32),
"formant_dispersion": np.asarray(df_arr, dtype=np.float32),
}
return m, dists
def extract_metrics(y, sr, pitch_floor, pitch_ceil, max_formant):
m, _ = extract_distributions(y, sr, pitch_floor, pitch_ceil, max_formant)
return m
# =============================================================================
# Mapping: metrics -> 0-100 sliders
# =============================================================================
def dist_to_percentile(value, dist):
if not np.isfinite(value):
return float("nan")
mu, sigma = dist.get("mu", float("nan")), dist.get("sigma", float("nan"))
if not (np.isfinite(mu) and np.isfinite(sigma) and sigma > 0):
return float("nan")
lo, hi = dist.get("lo", float("-inf")), dist.get("hi", float("inf"))
v = float(np.clip(value, lo, hi))
z = (v - mu) / max(sigma, 1e-6)
return float(np.clip(normal_cdf(z), 0.0, 1.0))
def select_dists(mapping_mode, baseline_state):
if mapping_mode == "Baseline (personal)" and baseline_state:
dists = baseline_state.get("dists", GLOBAL_DISTS)
# Handle baselines built with legacy key names
resolved = {}
for new_key in GLOBAL_DISTS:
if new_key in dists:
resolved[new_key] = dists[new_key]
else:
# Check legacy mappings
for old_key, mapped_new in _LEGACY_DIST_MAP.items():
if mapped_new == new_key and old_key in dists:
resolved[new_key] = dists[old_key]
break
else:
resolved[new_key] = GLOBAL_DISTS[new_key]
return resolved
if mapping_mode == "Anchored (literature)":
return ANCHORED_DISTS
return GLOBAL_DISTS
def metrics_to_sliders(m, dists, resonance_method=None):
pitch_p = dist_to_percentile(m.f0_med_hz, dists["f0_med_hz"])
prosody_p = dist_to_percentile(m.prosody_composite, dists["f0_sd_st"])
volume_p = dist_to_percentile(m.rms_dbfs, dists["rms_dbfs"])
weight_p = dist_to_percentile(m.h1_h2_corrected_db, dists["h1_h2_corrected"])
# Resonance: use selected method's field and distribution
method_name = resonance_method or RESONANCE_METHOD_DEFAULT
method_cfg = RESONANCE_METHODS.get(method_name, RESONANCE_METHODS[RESONANCE_METHOD_DEFAULT])
res_field = method_cfg["field"]
res_val = getattr(m, res_field, float("nan"))
# Use baseline dist for resonance_combo key if available, else method-specific dist
res_dist = dists.get("resonance_combo", method_cfg["dist"])
# If mapping mode is Global/Anchored, always use the method-specific dist
if res_dist is dists.get("resonance_combo") and method_name != RESONANCE_METHOD_DEFAULT:
res_dist = method_cfg["dist"]
resonance_p = dist_to_percentile(res_val, res_dist)
if method_cfg.get("invert") and np.isfinite(resonance_p):
resonance_p = 1.0 - resonance_p
# Volume: higher RMS -> louder -> 100 should be Soft -> invert.
if np.isfinite(volume_p):
volume_p = 1.0 - volume_p
return {
"Pitch": pct_to_slider(pitch_p),
"Weight": pct_to_slider(weight_p),
"Resonance": pct_to_slider(resonance_p),
"Prosody": pct_to_slider(prosody_p),
"Volume": pct_to_slider(volume_p),
}
# =============================================================================
# Formatting
# =============================================================================
def _fmt(val, fmt_spec, unit=""):
if np.isfinite(val):
return format(val, fmt_spec) + unit
return "\u2014"
def format_report(m):
lines = [
"Confidence: {:.2f}".format(m.confidence),
"",
"── Pitch ──",
" Median F0: {}".format(_fmt(m.f0_med_hz, '.1f', ' Hz')),
" Pitch spread (IQR): {}".format(_fmt(m.f0_iqr_hz, '.1f', ' Hz')),
" Voicing ratio: {}".format(_fmt(m.voicing_ratio, '.2f')),
"",
"── Prosody ──",
" F0 SD (semitones): {}".format(_fmt(m.f0_sd_st, '.2f', ' st')),
" Pitch velocity: {}".format(_fmt(m.f0_velocity_st_s, '.1f', ' st/s')),
" Direction changes: {}".format(_fmt(m.f0_direction_rate, '.1f', ' /s')),
" Composite score: {}".format(_fmt(m.prosody_composite, '.2f')),
"",
"── Resonance ──",
" F1 (median): {}".format(_fmt(m.f1_hz, '.0f', ' Hz')),
" F2 (median): {}".format(_fmt(m.f2_hz, '.0f', ' Hz')),
" F3 (median): {}".format(_fmt(m.f3_hz, '.0f', ' Hz')),
" F2 − F1: {}".format(_fmt(m.f2_minus_f1_hz, '.0f', ' Hz')),
" F2/F1 ratio: {}".format(_fmt(m.f2_f1_ratio, '.2f')),
" Formant dispersion: {}".format(_fmt(m.formant_dispersion_hz, '.0f', ' Hz')),
" Est. VTL: {}".format(_fmt(m.vtl_cm, '.1f', ' cm')),
" Spectral mean: {}".format(_fmt(m.spectral_mean_hz, '.0f', ' Hz')),
" Resonance composite:{}".format(_fmt(m.resonance_combo, '.2f', ' z')),
"",
"── Voice Quality / Weight ──",
" H1-H2 (raw): {}".format(_fmt(m.h1_h2_db, '.2f', ' dB')),
" H1*-H2* (corrected):{}".format(_fmt(m.h1_h2_corrected_db, '.2f', ' dB')),
" CPP: {}".format(_fmt(m.cpp_db, '.2f', ' dB')),
" HNR: {}".format(_fmt(m.hnr_db, '.1f', ' dB')),
" Jitter (local): {}".format(_fmt(m.jitter_local, '.3f', ' %')),
" Shimmer (local): {}".format(_fmt(m.shimmer_local, '.3f', ' %')),
"",
"── Signal ──",
" Volume (RMS): {}".format(_fmt(m.rms_dbfs, '.1f', ' dBFS')),
" SNR (rough): {}".format(_fmt(m.snr_db, '.1f', ' dB')),
" Clipping: {}".format(_fmt(m.clipping_ratio, '.4f')),
]
if m.warnings:
lines += ["", "Warnings:"] + [" ▸ {}".format(w) for w in m.warnings]
return "\n".join(lines)
def format_sliders(s):
return "\n".join(
"{}: {:.0f}".format(k, s[k]) if np.isfinite(s.get(k, float("nan")))
else "{}: \u2014".format(k)
for k in ["Pitch", "Weight", "Resonance", "Prosody", "Volume"]
)
def compute_delta(user_s, target_s):
d = {}
for k in ["Pitch", "Weight", "Resonance", "Prosody", "Volume"]:
u, t = user_s.get(k, float("nan")), target_s.get(k, float("nan"))
d[k] = (u - t) if np.isfinite(u) and np.isfinite(t) else float("nan")
return d
def format_delta(d):
return "\n".join(
"{}: {:+.0f}".format(k, d[k]) if np.isfinite(d.get(k, float("nan")))
else "{}: \u2014".format(k)
for k in ["Pitch", "Weight", "Resonance", "Prosody", "Volume"]
)
def append_gain_warning(target_state, user_sliders, warnings):
if not target_state:
return
t = (target_state or {}).get("sliders", {})
tv = t.get("Volume", float("nan"))
uv = user_sliders.get("Volume", float("nan"))
if np.isfinite(tv) and np.isfinite(uv) and abs(uv - tv) > 30:
warnings.append(
"Large volume delta likely reflects mic gain/distance "
"mismatch (not loudness).")
# =============================================================================
# Export / history
# =============================================================================
def export_profile_json(name, m, sliders):
ensure_exports_dir()
payload = {"name": name, "metrics": asdict(m), "sliders_0_100": sliders}
with open(PROFILE_JSON_PATH, "w") as f:
_json_dump(payload, f, indent=2)
return PROFILE_JSON_PATH
def export_history_csv(history):
ensure_exports_dir()
fnames = sorted({k for row in history for k in row})
# Sanitise NaN/Inf to empty string for clean CSV output
clean_rows = []
for row in history:
clean = {}
for k, v in row.items():
if isinstance(v, float) and (math.isnan(v) or math.isinf(v)):
clean[k] = ""
else:
clean[k] = v
clean_rows.append(clean)
with open(HISTORY_CSV_PATH, "w", newline="") as f:
w = csv.DictWriter(f, fieldnames=fnames)
w.writeheader()
w.writerows(clean_rows)
return HISTORY_CSV_PATH
def history_row(label, m, sliders):
return {
"timestamp": _now_iso(),
"label": label, "confidence": m.confidence,
"f0_med_hz": m.f0_med_hz, "f0_iqr_hz": m.f0_iqr_hz,
"f0_sd_st": m.f0_sd_st, "voicing_ratio": m.voicing_ratio,
"prosody_composite": m.prosody_composite,
"f0_velocity_st_s": m.f0_velocity_st_s,
"rms_dbfs": m.rms_dbfs, "snr_db": m.snr_db,
"clipping_ratio": m.clipping_ratio,
"f1_hz": m.f1_hz, "f2_hz": m.f2_hz, "f3_hz": m.f3_hz,
"f2_minus_f1_hz": m.f2_minus_f1_hz,
"f2_f1_ratio": m.f2_f1_ratio,
"formant_dispersion_hz": m.formant_dispersion_hz,
"vtl_cm": m.vtl_cm,
"spectral_mean_hz": m.spectral_mean_hz,
"resonance_combo": m.resonance_combo,
"h1_h2_db": m.h1_h2_db,
"h1_h2_corrected_db": m.h1_h2_corrected_db,
"cpp_db": m.cpp_db,
"hnr_db": m.hnr_db,
"jitter_local": m.jitter_local,
"shimmer_local": m.shimmer_local,
"Pitch_slider": sliders["Pitch"],
"Weight_slider": sliders["Weight"],
"Resonance_slider": sliders["Resonance"],
"Prosody_slider": sliders["Prosody"],
"Volume_slider": sliders["Volume"],
}
def export_baseline_json(baseline_state):
if not baseline_state:
return None
ensure_exports_dir()
with open(BASELINE_JSON_PATH, "w") as f:
_json_dump(baseline_state, f, indent=2)
return BASELINE_JSON_PATH
# =============================================================================
# Cache
# =============================================================================
ANALYSIS_CACHE = {}
_CACHE_LOCK = threading.Lock()
def cached_analyze(y, sr, pf, pc, mf, cache_key, dists, resonance_method=None):
rm = resonance_method or RESONANCE_METHOD_DEFAULT
k = audio_hash(y, sr, extra="{}-{}-{}-{}-rm:{}".format(pf, pc, mf, cache_key, rm))
with _CACHE_LOCK:
if k in ANALYSIS_CACHE:
return ANALYSIS_CACHE[k]
m = extract_metrics(y, sr, pf, pc, mf)
sliders = metrics_to_sliders(m, dists, resonance_method=rm)
with _CACHE_LOCK:
ANALYSIS_CACHE[k] = (m, sliders)
if len(ANALYSIS_CACHE) > 128:
ANALYSIS_CACHE.pop(next(iter(ANALYSIS_CACHE)))
return m, sliders
# =============================================================================
# Baseline
# =============================================================================
def build_baseline_from_audio(y, sr, pf, pc, mf):
m, d = extract_distributions(y, sr, pf, pc, mf)
if not d:
return None, "Baseline failed: could not compute distributions."
f0_mu, f0_sigma = robust_center_scale(d["f0_voiced"])
if not np.isfinite(f0_mu) or d["f0_voiced"].size < 30:
return None, "Baseline failed: not enough voiced frames. Speak continuously for 10-15 s."
st_mu, st_sigma = robust_center_scale(d["st"])
rms_mu, rms_sigma = robust_center_scale(d["rms_frames_dbfs"])
# Resonance composite z-score distribution
res_combo_arr = d.get("resonance_combo", np.array([], dtype=np.float32))
if res_combo_arr.size > 0:
rc_mu, rc_sigma = float(np.median(res_combo_arr)), max(float(np.std(res_combo_arr)), 0.3)
else:
# Fallback: derive from F2-F1
f2m = d.get("f2_minus_f1_vals", np.array([], dtype=np.float32))
if f2m.size > 0:
rc_mu = float((np.median(f2m) - 900.0) / 200.0)
rc_sigma = 1.0
else:
rc_mu, rc_sigma = 0.0, 1.0
h12_mu, h12_sigma = robust_center_scale(d.get("h1_h2_corrected", np.array([], dtype=np.float32)))
def mk(mu, sigma, min_sigma, pad=3.0):
if not (np.isfinite(mu) and np.isfinite(sigma) and sigma > 0):
return {"mu": float("nan"), "sigma": float("nan"), "lo": -1e9, "hi": 1e9}
s = max(sigma, min_sigma)
return {"mu": float(mu), "sigma": float(s),
"lo": float(mu - pad * s), "hi": float(mu + pad * s)}
baseline_state = {
"note": "Baseline distributions derived from your sample.",
"metrics_snapshot": asdict(m),
"dists": {
"f0_med_hz": mk(f0_mu, f0_sigma, 5.0),
"f0_sd_st": mk(max(float(np.std(d["st"])), 0.01), st_sigma, 0.25),
"rms_dbfs": mk(rms_mu, rms_sigma, 2.0),
"resonance_combo": mk(rc_mu, rc_sigma, 0.3),
"h1_h2_corrected": mk(h12_mu, h12_sigma, 1.0),
},
"counts": {
"f0_voiced": int(d["f0_voiced"].size),
"f2_minus_f1_vals": int(d["f2_minus_f1_vals"].size),
"h1_h2_corrected": int(d.get("h1_h2_corrected", np.array([])).size),
}
}
return baseline_state, "Baseline set."
# =============================================================================
# Audio preprocessing pipeline
# =============================================================================
def _prep_audio(audio_like, window_choice, pitch_floor, pitch_ceil,
max_formant, use_preemph, use_loudnorm,
max_secs=MAX_UPLOAD_SECONDS):
normalized = normalize_audio_tuple(audio_like)
if normalized is None:
raise ValueError("No audio provided.")
sr, y = normalized
y = to_mono_float32(y)
if y is None:
raise ValueError("Audio is empty after conversion.")
y, sr = resample_if_needed(y, sr, DEFAULT_SR)
y = hard_cap_length(y, sr, max_secs)
y = select_analysis_window(y, sr, window_choice)
y = preprocess_audio(y, use_preemph, use_loudnorm)
pf = int(np.clip(int(round(pitch_floor)), 40, 150))
pc = int(np.clip(int(round(pitch_ceil)), 250, 800))
if pf >= pc:
pf = min(pf, pc - 1)
mf = int(np.clip(int(round(max_formant)), 4500, 6500))
return y, sr, pf, pc, mf
# =============================================================================
# Resonance method comparison
# =============================================================================
def resonance_comparison_table(m, dists, active_method=None):
"""Generate an HTML comparison table showing all 7 resonance methods side-by-side.
For each method, shows: raw value, unit, slider position (0-100), and
whether it's the currently active method.
"""
active = active_method or RESONANCE_METHOD_DEFAULT
rows = []
for name, cfg in RESONANCE_METHODS.items():
field = cfg["field"]
raw_val = getattr(m, field, float("nan"))
unit = cfg["unit"]
is_active = (name == active)
# Compute slider position using method-specific dist
d = cfg["dist"]
pct = dist_to_percentile(raw_val, d)
if cfg.get("invert") and np.isfinite(pct):
pct = 1.0 - pct
slider_val = pct_to_slider(pct) if np.isfinite(pct) else float("nan")
# Format values
if np.isfinite(raw_val):
if unit == "cm":
raw_str = "{:.1f}".format(raw_val)
elif unit == "z":
raw_str = "{:+.2f}".format(raw_val)
elif unit == "":
raw_str = "{:.2f}".format(raw_val)
else:
raw_str = "{:.0f}".format(raw_val)
else:
raw_str = "—"
slider_str = "{:.0f}".format(slider_val) if np.isfinite(slider_val) else "—"
# Color-code slider value: gradient from dark (0) to bright (100)
if np.isfinite(slider_val):
hue = 25 + (slider_val / 100.0) * 10 # orange range 25-35
sat = 70
light = 30 + (slider_val / 100.0) * 35 # 30% to 65%
bar_pct = max(3, slider_val)
bar_color = "hsl({}, {}%, {}%)".format(hue, sat, light)
else:
bar_pct = 0
bar_color = "#555"
active_marker = " ✦" if is_active else ""
row_class = "res-row-active" if is_active else "res-row"
rows.append(
''
'| {name}{mark} | '
'{raw} {u} | '
''
' '
' | '
'{src} | '
'
'.format(
cls=row_class,
fw="700" if is_active else "400",
name=name, mark=active_marker,
raw=raw_str, u=unit,
bc=bar_color, bp=bar_pct, sv=slider_str,
src=cfg["source"].split("—")[0].strip() if "—" in cfg["source"] else cfg["source"][:25],
)
)
html = (
''
'
'
''
'| Method | '
'Value | '
'Slider (0–100) | '
'Source | '
'
'
)
for r in rows:
html += r
html += '
![]()
of a radar chart comparing target vs user."""
if not _HAS_MPL:
return "matplotlib not available for charts.
"
if not target_sliders and not user_sliders:
return ""
dims = _DIM_ORDER
n = len(dims)
angles = [i * 2 * math.pi / n for i in range(n)]
angles.append(angles[0])
fig = None
try:
fig, ax = plt.subplots(figsize=(3.8, 3.8), subplot_kw={"polar": True})
fig.patch.set_facecolor("#1a1a2e")
ax.set_facecolor("#1a1a2e")
ax.set_thetagrids([a * 180 / math.pi for a in angles[:-1]], dims,
fontsize=9, fontweight="bold", color="#aaa")
ax.set_ylim(0, 100)
ax.set_yticks([20, 40, 60, 80])
ax.set_yticklabels(["20", "40", "60", "80"], fontsize=7, color="#555")
ax.spines["polar"].set_visible(False)
ax.tick_params(colors="#555")
ax.grid(color="#333", linewidth=0.5)
if target_sliders:
t_vals = [safe(target_sliders.get(d, 0)) for d in dims] + \
[safe(target_sliders.get(dims[0], 0))]
ax.plot(angles, t_vals, "o-", color="#888", linewidth=2,
markersize=5, label="Target", zorder=3)
ax.fill(angles, t_vals, alpha=0.08, color="#888")
if user_sliders:
u_vals = [safe(user_sliders.get(d, 0)) for d in dims] + \
[safe(user_sliders.get(dims[0], 0))]
ax.plot(angles, u_vals, "o-", color="#5b7de8", linewidth=2.2,
markersize=5, label="You", zorder=4)
ax.fill(angles, u_vals, alpha=0.12, color="#5b7de8")
ax.legend(loc="upper right", bbox_to_anchor=(1.25, 1.15), fontsize=8,
facecolor="#1a1a2e", edgecolor="#333", labelcolor="#aaa")
plt.tight_layout()
uri = _fig_to_base64(fig)
return '
'.format(uri)
except Exception:
return ""
finally:
if fig is not None:
plt.close(fig)
def render_trend_plot(history, max_points=40):
"""Return HTML ![]()
of a trend line chart from session history."""
if not _HAS_MPL:
return "matplotlib not available for charts.
"
user_rows = [r for r in (history or []) if r.get("label", "").startswith("You")]
if len(user_rows) < 2:
return "Record at least two samples to see trends.
"
rows = user_rows[-max_points:]
x = list(range(len(rows)))
x_labels = [r.get("timestamp", "")[11:16] for r in rows] # HH:MM
fig = None
try:
fig, ax = plt.subplots(figsize=(6, 2.8))
fig.patch.set_facecolor("#1a1a2e")
ax.set_facecolor("#1a1a2e")
for dim in _DIM_ORDER:
key = dim + "_slider"
vals = []
for r in rows:
v = r.get(key, float("nan"))
vals.append(v if np.isfinite(v) else float("nan"))
ax.plot(x, vals, "o-", color=_DIM_COLORS[dim], linewidth=1.8,
markersize=4, label=dim, alpha=0.85)
ax.set_ylim(0, 100)
ax.set_ylabel("Slider value", fontsize=9, color="#888")
ax.set_xlabel("Session recordings", fontsize=9, color="#888")
ax.tick_params(colors="#666")
if len(x_labels) <= 15:
ax.set_xticks(x)
ax.set_xticklabels(x_labels, fontsize=7, rotation=45, color="#666")
else:
step = max(1, len(x) // 10)
ax.set_xticks(x[::step])
ax.set_xticklabels(x_labels[::step], fontsize=7, rotation=45, color="#666")
ax.legend(fontsize=7, ncol=5, loc="upper center", bbox_to_anchor=(0.5, 1.18),
facecolor="#1a1a2e", edgecolor="#333", labelcolor="#aaa")
ax.grid(True, alpha=0.15, color="#555")
for spine in ax.spines.values():
spine.set_color("#333")
plt.tight_layout()
uri = _fig_to_base64(fig)
return ''.format(uri)
except Exception:
return ""
finally:
if fig is not None:
plt.close(fig)
# =============================================================================
# Custom HTML bar visualization (matches Concept Slider design)
# =============================================================================
_BAR_COLORS = {
"Pitch": {"grad": "linear-gradient(to top, #1e2a5c, #3b5abd, #5b7de8, #8da6f0)",
"marker": "#5b7de8", "glow": "rgba(91,125,232,0.4)", "bg": "rgba(91,125,232,0.08)"},
"Weight": {"grad": "linear-gradient(to top, #2d1548, #5a2d82, #9b6dba, #c4a0d8)",
"marker": "#9b6dba", "glow": "rgba(155,109,186,0.4)", "bg": "rgba(155,109,186,0.08)"},
"Resonance": {"grad": "linear-gradient(to top, #6b3a1a, #b86040, #e8956e, #f0c0a0)",
"marker": "#e8956e", "glow": "rgba(232,149,110,0.4)", "bg": "rgba(232,149,110,0.08)"},
"Prosody": {"grad": "linear-gradient(to top, #4a4010, #8a7520, #d4c24a, #e8dc80)",
"marker": "#d4c24a", "glow": "rgba(212,194,74,0.4)", "bg": "rgba(212,194,74,0.08)"},
"Volume": {"grad": "linear-gradient(to top, #6a1850, #b84090, #e878c8, #f0a8d8)",
"marker": "#e878c8", "glow": "rgba(232,120,200,0.4)", "bg": "rgba(232,120,200,0.08)"},
}
_BAR_LABELS = {
"Pitch": ("Low", "High"),
"Weight": ("Heavy", "Light"),
"Resonance": ("Dark", "Bright"),
"Prosody": ("Mono", "Bouncy"),
"Volume": ("Loud", "Soft"),
}
def render_slider_bars(target_sliders=None, user_sliders=None,
confidence=None, context="target"):
"""Render vertical colour-coded slider bars as a dark-mode-native HTML block.
context: "target" | "user" — changes empty-state messaging.
"""
has_target = target_sliders and any(
np.isfinite(target_sliders.get(d, float("nan"))) for d in _DIM_ORDER)
has_user = user_sliders and any(
np.isfinite(user_sliders.get(d, float("nan"))) for d in _DIM_ORDER)
bars_html = []
for dim in _DIM_ORDER:
colors = _BAR_COLORS[dim]
lo_label, hi_label = _BAR_LABELS[dim]
t_val = safe(target_sliders.get(dim, float("nan")), -1) if target_sliders else -1
u_val = safe(user_sliders.get(dim, float("nan")), -1) if user_sliders else -1
# Clamp display positions
t_pct = max(2.0, min(98.0, t_val)) if t_val >= 0 else -1
u_pct = max(2.0, min(98.0, u_val)) if u_val >= 0 else -1
# Target marker — horizontal line with subtle triangles
target_marker = ""
if t_pct >= 0:
target_marker = (
''
).format(pct=t_pct, val=t_val)
# User marker — glowing circle
user_marker = ""
if u_pct >= 0:
user_marker = (
''
).format(pct=u_pct, val=u_val, color=colors["marker"],
glow=colors["glow"], glow2=colors["glow"].replace("0.4", "0.15"))
# Delta badge
delta_badge = ""
if t_val >= 0 and u_val >= 0:
d = u_val - t_val
if abs(d) <= 8:
d_color = "#4ade80"
d_bg = "rgba(74,222,128,0.1)"
elif abs(d) <= 20:
d_color = "#fbbf24"
d_bg = "rgba(251,191,36,0.1)"
else:
d_color = "#f87171"
d_bg = "rgba(248,113,113,0.1)"
delta_badge = (
'{delta:+.0f}
'
).format(color=d_color, bg=d_bg, delta=d)
# Value display
val_display = ""
if u_val >= 0:
val_display = (
'{v:.0f}
'
).format(c=colors["marker"], v=u_val)
elif t_val >= 0:
val_display = (
'{v:.0f}
'
).format(v=t_val)
bar = (
''
# Top label
'
{hi}
'
# Bar track
'
'
# Inner glow overlay
'
'
' {target}{user}'
'
'
# Bottom label
'
{lo}
'
# Dimension name
'
{dim}
'
' {val}{delta}'
'
'
'
Confidence'
'
'
'
'
'{pct:.0f}% {label}'
'
{}
'
).format("".join(legend_items))
# Empty state
empty_hint = ""
if not has_target and not has_user:
if context == "target":
empty_hint = (
''
'Upload a voice sample above to see where it lands on each dimension'
'
')
else:
empty_hint = (
''
'Record or upload your voice to see your slider positions'
'
')
return (
''
'
'
+ "".join(bars_html) +
'
' + legend + conf_html + empty_hint +
'