Spaces:
Running on Zero
Running on Zero
File size: 5,099 Bytes
a63c2eb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | """
Lunaron Evolution Engine - Candidate Evaluator
Evaluates challenger models against champion baseline on Holdout Dataset.
Strict Regression Gate:
- If ANY stem degrades by > 5.0%, regression fails.
- Prevents "Vocal improved at the expense of Drums" regressions.
"""
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Any
import torch
import torch.nn.functional as F
@dataclass
class RegressionResult:
passed: bool
overall_improved: bool
stem_regressions: List[str]
reasons: List[str]
summary_text: str
@dataclass
class EvaluationReport:
champion_id: str
challenger_id: str
stem_metrics: Dict[str, Dict[str, float]]
regression_result: RegressionResult
overall_quality_delta_pct: float
leakage_delta_pct: float
mixture_consistency_delta_pct: float
class CandidateEvaluator:
"""
Offline validation & regression test engine.
Ensures safe, measured model advancement without degrading any stem.
"""
@staticmethod
def compute_stem_metrics(
pred_stem: torch.Tensor,
ref_stem: torch.Tensor,
eps: float = 1e-8
) -> Dict[str, float]:
"""Calculates energy, SDR proxy, and spectral convergence for a stem."""
min_len = min(pred_stem.shape[-1], ref_stem.shape[-1])
p = pred_stem[..., :min_len]
r = ref_stem[..., :min_len]
p_mono = p.mean(dim=0) if p.dim() > 1 else p
r_mono = r.mean(dim=0) if r.dim() > 1 else r
l1_err = float(F.l1_loss(p_mono, r_mono).item())
p_energy = float(torch.sum(p_mono ** 2).item())
r_energy = float(torch.sum(r_mono ** 2).item())
# Spectral difference
win = torch.hann_window(2048, device=p.device)
stft_p = torch.abs(torch.stft(p_mono, n_fft=2048, hop_length=512, window=win, return_complex=True)) + eps
stft_r = torch.abs(torch.stft(r_mono, n_fft=2048, hop_length=512, window=win, return_complex=True)) + eps
spectral_conv = float((torch.norm(stft_p - stft_r, p="fro") / (torch.norm(stft_r, p="fro") + eps)).item())
return {
"l1_error": round(l1_err, 5),
"spectral_convergence": round(spectral_conv, 4),
"energy_ratio": round(p_energy / (r_energy + eps), 4)
}
@classmethod
def check_regression(
cls,
champion_metrics: Dict[str, Dict[str, float]],
challenger_metrics: Dict[str, Dict[str, float]],
tolerance_pct: float = 5.0
) -> RegressionResult:
"""
Verifies whether challenger has any regression compared to champion.
A stem is considered regressed if spectral convergence or error is worse by > tolerance_pct.
"""
regressed_stems = []
reasons = []
total_champ_sc = 0.0
total_chall_sc = 0.0
count = 0
for stem_name, c_m in champion_metrics.items():
if stem_name not in challenger_metrics:
continue
h_m = challenger_metrics[stem_name]
count += 1
c_sc = c_m.get("spectral_convergence", 1.0)
h_sc = h_m.get("spectral_convergence", 1.0)
total_champ_sc += c_sc
total_chall_sc += h_sc
# Higher spectral convergence means more error
degradation = ((h_sc - c_sc) / max(c_sc, 1e-6)) * 100.0
if degradation > tolerance_pct:
regressed_stems.append(stem_name)
reasons.append(f"Stem '{stem_name}' degraded by {round(degradation, 1)}% in spectral error.")
overall_improved = total_chall_sc < total_champ_sc if count > 0 else False
passed = (len(regressed_stems) == 0) and overall_improved
if not overall_improved:
reasons.append("Challenger overall error is not lower than Champion.")
summary = (
f"Regression Status: {'PASS' if passed else 'FAIL'}. "
f"Active Regressions: {len(regressed_stems)} stems."
)
return RegressionResult(
passed=passed,
overall_improved=overall_improved,
stem_regressions=regressed_stems,
reasons=reasons,
summary_text=summary
)
@classmethod
def generate_report_text(cls, report: EvaluationReport) -> str:
"""Generates formatted text report following the design spec."""
reg = report.regression_result
return f"""
========================================
Lunaron Evolution Engine Evaluation Report
========================================
Previous Champion: {report.champion_id}
Candidate : {report.challenger_id}
Overall Quality Delta : {report.overall_quality_delta_pct:+.1f}%
Leakage Delta : {report.leakage_delta_pct:+.1f}%
Mixture Consistency Delta: {report.mixture_consistency_delta_pct:+.1f}%
Regression Gate Status : {'PASS' if reg.passed else 'FAIL'}
Stem Regressions Detected: {', '.join(reg.stem_regressions) if reg.stem_regressions else 'None'}
{chr(10).join(['- ' + r for r in reg.reasons])}
========================================
"""
|