recruto / validation /validate_interview_system.py
badarfaisal989u9's picture
Initial deployment
04b8502
Raw History Blame Contribute Delete
7.16 kB
"""
Validation utility for interview system.
This script validates:
1) Face verification (binary classification) using confusion matrix metrics.
2) Speech-to-text output quality using WER/CER.
Input CSV formats:
Face CSV columns:
- sample_id
- y_true (0 or 1)
- y_pred (0 or 1)
STT CSV columns:
- sample_id
- reference_text
- predicted_text
Usage example:
python validation/validate_interview_system.py \
--face_csv validation/face_labels.csv \
--stt_csv validation/stt_labels.csv \
--output_dir validation/results
"""
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
from typing import Dict, List, Tuple
def _safe_div(numerator: float, denominator: float) -> float:
return numerator / denominator if denominator else 0.0
def _normalize_text(text: str) -> str:
# Minimal normalization so metrics are fair and consistent.
return " ".join((text or "").strip().lower().split())
def _levenshtein(a: List[str], b: List[str]) -> int:
if not a:
return len(b)
if not b:
return len(a)
prev_row = list(range(len(b) + 1))
for i, token_a in enumerate(a, start=1):
curr_row = [i]
for j, token_b in enumerate(b, start=1):
ins = curr_row[j - 1] + 1
delete = prev_row[j] + 1
replace = prev_row[j - 1] + (token_a != token_b)
curr_row.append(min(ins, delete, replace))
prev_row = curr_row
return prev_row[-1]
def compute_face_metrics(face_csv: Path) -> Dict[str, float]:
tp = tn = fp = fn = 0
total = 0
with face_csv.open("r", encoding="utf-8", newline="") as f:
reader = csv.DictReader(f)
required = {"sample_id", "y_true", "y_pred"}
if not required.issubset(reader.fieldnames or []):
raise ValueError(
f"Face CSV must contain columns: {sorted(required)}. "
f"Found: {reader.fieldnames}"
)
for row in reader:
y_true = int(row["y_true"])
y_pred = int(row["y_pred"])
if y_true not in (0, 1) or y_pred not in (0, 1):
raise ValueError(
f"y_true/y_pred must be 0 or 1. Row sample_id={row.get('sample_id')}"
)
total += 1
if y_true == 1 and y_pred == 1:
tp += 1
elif y_true == 0 and y_pred == 0:
tn += 1
elif y_true == 0 and y_pred == 1:
fp += 1
elif y_true == 1 and y_pred == 0:
fn += 1
accuracy = _safe_div(tp + tn, total)
precision = _safe_div(tp, tp + fp)
recall = _safe_div(tp, tp + fn)
specificity = _safe_div(tn, tn + fp)
f1 = _safe_div(2 * precision * recall, precision + recall)
return {
"samples": total,
"tp": tp,
"tn": tn,
"fp": fp,
"fn": fn,
"accuracy": round(accuracy, 4),
"precision": round(precision, 4),
"recall": round(recall, 4),
"specificity": round(specificity, 4),
"f1_score": round(f1, 4),
}
def compute_stt_metrics(stt_csv: Path) -> Dict[str, float]:
total_word_dist = 0
total_words = 0
total_char_dist = 0
total_chars = 0
samples = 0
with stt_csv.open("r", encoding="utf-8", newline="") as f:
reader = csv.DictReader(f)
required = {"sample_id", "reference_text", "predicted_text"}
if not required.issubset(reader.fieldnames or []):
raise ValueError(
f"STT CSV must contain columns: {sorted(required)}. "
f"Found: {reader.fieldnames}"
)
for row in reader:
ref = _normalize_text(row.get("reference_text", ""))
hyp = _normalize_text(row.get("predicted_text", ""))
ref_words = ref.split()
hyp_words = hyp.split()
ref_chars = list(ref)
hyp_chars = list(hyp)
total_word_dist += _levenshtein(ref_words, hyp_words)
total_words += len(ref_words)
total_char_dist += _levenshtein(ref_chars, hyp_chars)
total_chars += len(ref_chars)
samples += 1
wer = _safe_div(total_word_dist, total_words)
cer = _safe_div(total_char_dist, total_chars)
return {
"samples": samples,
"word_errors": total_word_dist,
"total_reference_words": total_words,
"wer": round(wer, 4),
"char_errors": total_char_dist,
"total_reference_chars": total_chars,
"cer": round(cer, 4),
}
def write_reports(
output_dir: Path,
face_metrics: Dict[str, float] | None,
stt_metrics: Dict[str, float] | None,
) -> None:
output_dir.mkdir(parents=True, exist_ok=True)
payload = {"face_validation": face_metrics, "stt_validation": stt_metrics}
json_path = output_dir / "validation_report.json"
json_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
text_lines: List[str] = ["Interview System Validation Report", "=" * 36, ""]
if face_metrics:
text_lines.extend(
[
"Face Verification:",
f"- Samples: {face_metrics['samples']}",
f"- Confusion Matrix: TP={face_metrics['tp']}, TN={face_metrics['tn']}, FP={face_metrics['fp']}, FN={face_metrics['fn']}",
f"- Accuracy: {face_metrics['accuracy']:.4f}",
f"- Precision: {face_metrics['precision']:.4f}",
f"- Recall: {face_metrics['recall']:.4f}",
f"- Specificity: {face_metrics['specificity']:.4f}",
f"- F1-Score: {face_metrics['f1_score']:.4f}",
"",
]
)
if stt_metrics:
text_lines.extend(
[
"Answer Recording (STT):",
f"- Samples: {stt_metrics['samples']}",
f"- WER: {stt_metrics['wer']:.4f}",
f"- CER: {stt_metrics['cer']:.4f}",
"",
]
)
(output_dir / "validation_report.txt").write_text(
"\n".join(text_lines), encoding="utf-8"
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Validate interview system metrics.")
parser.add_argument("--face_csv", type=Path, default=None, help="Face labels CSV.")
parser.add_argument("--stt_csv", type=Path, default=None, help="STT labels CSV.")
parser.add_argument(
"--output_dir",
type=Path,
default=Path("validation/results"),
help="Directory to store validation reports.",
)
return parser.parse_args()
def main() -> None:
args = parse_args()
if not args.face_csv and not args.stt_csv:
raise ValueError("Provide at least one input file: --face_csv or --stt_csv")
face_metrics = compute_face_metrics(args.face_csv) if args.face_csv else None
stt_metrics = compute_stt_metrics(args.stt_csv) if args.stt_csv else None
write_reports(args.output_dir, face_metrics, stt_metrics)
print(f"Validation report generated in: {args.output_dir}")
if __name__ == "__main__":
main()