feat: add processing service runtime

app.py

@@ -0,0 +1,42 @@
+import json
+import os
+import sys
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parent
+RUNTIME_ROOT = Path(os.environ.get("CAPSTONE_RUNTIME_ROOT", "/tmp/call-qa"))
+DATA_ROOT = RUNTIME_ROOT / "data"
+
+os.environ.setdefault("CAPSTONE_DATA_ROOT", str(DATA_ROOT))
+os.environ.setdefault("CAPSTONE_EVAL_RESULTS", str(RUNTIME_ROOT / "evaluation_results"))
+os.environ.setdefault("CAPSTONE_FRONTEND_PUBLIC", str(RUNTIME_ROOT / "frontend_public"))
+os.environ.setdefault("CAPSTONE_ENV_FILE", str(ROOT / ".env"))
+os.environ.setdefault("ENABLE_ACOUSTIC", "0")
+os.environ.setdefault("START_WORKER", "1")
+
+
+def _prepare_runtime() -> None:
+    """Create the writable files expected by the existing pipeline modules."""
+    manifest = DATA_ROOT / "na_testset" / "manifest.json"
+    manifest.parent.mkdir(parents=True, exist_ok=True)
+    if not manifest.exists():
+        manifest.write_text(json.dumps([]), encoding="utf-8")
+
+    for path in (
+        RUNTIME_ROOT / "evaluation_results",
+        RUNTIME_ROOT / "frontend_public",
+    ):
+        path.mkdir(parents=True, exist_ok=True)
+
+
+_prepare_runtime()
+sys.path.insert(0, str(ROOT / "backend"))
+
+from app.space import app  # noqa: E402
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", "7860")))

backend/app/__init__.py

@@ -0,0 +1 @@
+"""Call QA processing API package."""

backend/app/agents/extraction.py

@@ -0,0 +1,12 @@
+import json
+
+class StructuredEvidenceExtractor:
+    def __init__(self):
+        pass
+
+    def extract_evidence(self, text_payload: str):
+        # Enforces structured output JSON formats
+        return {
+            "evidence_found": False,
+            "segments": []
+        }

backend/app/agents/llm_client.py

@@ -0,0 +1,18 @@
+import urllib.request
+import json
+
+class OllamaLLMClient:
+    def __init__(self, base_url="http://localhost:11434"):
+        self.base_url = base_url
+
+    def query_llama(self, prompt: str, system_prompt: str = ""):
+        # Calls Ollama REST endpoint for Llama 3.2 3B model
+        payload = {
+            "model": "llama3.2:3b",
+            "prompt": prompt,
+            "system": system_prompt,
+            "stream": False
+        }
+        
+        # Simple client request placeholder
+        return "LLM Compliance Inference Output"

backend/app/agents/pipeline.py

@@ -0,0 +1,12 @@
+class CentralPipelineOrchestrator:
+    def __init__(self):
+        pass
+
+    async def execute(self, audio_path: str):
+        # Coordinates ML model inference and multi-agent compliance evaluation
+        return {
+            "status": "success",
+            "audio": audio_path,
+            "transcripts": [],
+            "compliance_checks": []
+        }

backend/app/agents/workflow_agent.py

@@ -0,0 +1,11 @@
+class WorkflowComplianceAgent:
+    def __init__(self):
+        pass
+
+    def evaluate_rules(self, transcript_segments: list):
+        # Deterministic check for compliance rules matching
+        results = []
+        for segment in transcript_segments:
+            # Rule based matches
+            pass
+        return results

backend/app/api/.gitkeep

@@ -0,0 +1 @@
+# Keep directory

backend/app/database.py

@@ -0,0 +1,39 @@
+import os
+from sqlalchemy import create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+
+# On Hugging Face Spaces DATABASE_URL MUST be the Supabase SESSION-POOLER
+# (IPv4) URI -- the direct db host is IPv6-only and silently times out.
+DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@db:5432/capstone")
+
+# pool_pre_ping: the Space sleeps after ~48h idle; on wake every pooled
+# connection is long dead. pre_ping issues SELECT 1 before checkout and
+# reconnects transparently instead of 500-ing the first request.
+# pool_recycle keeps connections under the pooler's server-side idle timeout.
+engine = create_engine(
+    DATABASE_URL,
+    pool_pre_ping=True,
+    pool_recycle=1800,
+)
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+Base = declarative_base()
+
+
+def init_db():
+    """Create tables if the database is reachable.
+
+    Import-time table creation would hard-crash the whole app when the DB is
+    unreachable (e.g. a local smoke import with no Postgres). Callers invoke
+    this at startup and may tolerate failure in dev.
+    """
+    Base.metadata.create_all(bind=engine)
+
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

backend/app/main.py

@@ -0,0 +1,83 @@
+import os
+import threading
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+# Load the repo-root .env for local dev BEFORE importing anything that reads
+# env at import time (database.py builds the engine from DATABASE_URL). On the
+# Space there is no .env; the real environment (Space secrets) is used and
+# always takes precedence -- load_dotenv never overrides existing vars.
+load_dotenv(Path(__file__).resolve().parents[2] / ".env")
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from app.database import init_db
+from app.routers import calls, sentiment, artifacts
+from app import worker
+
+
+def _boot_model_prewarm():
+    """Pre-download the acoustic weights so the first job doesn't pay for it.
+    Best-effort, off-thread; only when acoustic is enabled."""
+    if os.environ.get("ENABLE_ACOUSTIC", "0").strip().lower() in ("", "0", "false", "no"):
+        return
+    try:
+        from app.pipeline_bridge import install
+        install()
+        from acoustic import resolve_model_dir
+        resolve_model_dir()
+        print("[startup] acoustic weights ready")
+    except Exception as e:
+        print(f"[startup] acoustic pre-warm skipped: {e}")
+
+
+@asynccontextmanager
+async def lifespan(app):
+    try:
+        init_db()
+    except Exception as e:
+        print(f"[startup] init_db skipped (db unreachable?): {e}")
+    if os.environ.get("START_WORKER", "1").strip().lower() not in ("", "0", "false", "no"):
+        worker.start()
+    threading.Thread(target=_boot_model_prewarm, name="model-prewarm",
+                     daemon=True).start()
+    yield
+
+
+app = FastAPI(
+    title="AI/ML Capstone API",
+    description="Fresh-call ingest, pipeline worker, and artifact serving",
+    version="1.0.0",
+    lifespan=lifespan,
+)
+
+# CORS: the Vercel frontend calls this API cross-origin. FRONTEND_ORIGIN pins
+# the allowed origin(s) in production (comma-separated); "*" for local dev.
+# Credentials are disabled under wildcard (the spec forbids "*" + credentials);
+# the frontend uses plain fetch with no cookies, so this is fine either way.
+_origins = os.environ.get("FRONTEND_ORIGIN", "*")
+_wild = _origins.strip() == "*"
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"] if _wild else [o.strip() for o in _origins.split(",")],
+    allow_credentials=not _wild,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(calls.router)
+app.include_router(sentiment.router)
+app.include_router(artifacts.router)
+
+
+@app.get("/")
+def read_root():
+    return {"status": "online", "service": "backend-orchestration"}
+
+
+@app.get("/health")
+def health_check():
+    return {"status": "ok", "service": "capstone_api"}

backend/app/models.py

@@ -0,0 +1,113 @@
+import uuid
+from datetime import datetime
+from sqlalchemy import Column, String, Integer, Float, Boolean, DateTime, Text, ForeignKey, SmallInteger, Numeric
+from sqlalchemy.dialects.postgresql import UUID, JSONB
+from sqlalchemy.orm import relationship
+from .database import Base
+
+class Agent(Base):
+    __tablename__ = "agents"
+
+    agent_id   = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    name       = Column(String(100), nullable=False)
+    team       = Column(String(100))
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+    calls      = relationship("Call", back_populates="agent")
+
+class Call(Base):
+    __tablename__ = "calls"
+
+    call_id          = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    agent_id         = Column(UUID(as_uuid=True), ForeignKey("agents.agent_id"), nullable=False)
+    audio_path       = Column(String(500))
+    call_date        = Column(DateTime, nullable=False)
+    duration_seconds = Column(Integer)
+    call_metadata    = Column(JSONB)
+    created_at       = Column(DateTime, default=datetime.utcnow)
+
+    agent = relationship("Agent", back_populates="calls")
+    jobs  = relationship("Job", back_populates="call")
+
+class Job(Base):
+    __tablename__ = "jobs"
+
+    job_id     = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    call_id    = Column(UUID(as_uuid=True), ForeignKey("calls.call_id"), nullable=False)
+    status     = Column(String(20), default="queued")
+    stage      = Column(String(50))
+    error      = Column(Text)
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+    call = relationship("Call", back_populates="jobs")
+
+class Transcript(Base):
+    __tablename__ = "transcripts"
+
+    transcript_id  = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    call_id        = Column(UUID(as_uuid=True), nullable=True)
+    source_call_id = Column(String(100))
+    turn_id        = Column(Integer, nullable=False)
+    speaker        = Column(String(20))
+    start_time     = Column(Float)
+    end_time       = Column(Float)
+    text           = Column(Text)
+    avg_confidence = Column(Float)
+    low_confidence = Column(Boolean, default=False)
+
+class Evaluation(Base):
+    __tablename__ = "evaluations"
+
+    evaluation_id          = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    call_id                = Column(UUID(as_uuid=True), nullable=True)
+    agent_id               = Column(UUID(as_uuid=True), ForeignKey("agents.agent_id"), nullable=False)
+    overall_grade          = Column(String(1))
+    weighted_score         = Column(Numeric(5, 2))
+    scorecard              = Column(JSONB)
+    compliance_flags       = Column(JSONB)
+    escalation_risk        = Column(SmallInteger)
+    escalation_flag        = Column(Boolean, default=False)
+    coaching_required      = Column(Boolean, default=False)
+    manual_review_required = Column(Boolean, default=False)
+    llm_scored             = Column(Boolean, default=True)
+    created_at             = Column(DateTime, default=datetime.utcnow)
+
+class SentimentSegment(Base):
+    __tablename__ = "sentiment_segments"
+
+    id                     = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    call_id                = Column(String(100), nullable=False)
+    segment_index          = Column(Integer, nullable=False)
+    segment_key            = Column(String(200))
+    seq_id                 = Column(Integer)
+    speaker                = Column(String(20))
+    start_time             = Column(Float)
+    end_time               = Column(Float)
+    text                   = Column(Text)
+    sentiment              = Column(String(20))
+    dominant_emotion       = Column(String(50))
+    escalation_score       = Column(Float)
+    processing_status      = Column(String(30))
+    audio_features         = Column(JSONB)
+    explainability_flags   = Column(JSONB)
+    escalation_explanation = Column(JSONB)
+    has_audio_features     = Column(Boolean, default=False)
+    audio_feature_version  = Column(String(100))
+    domain                 = Column(String(50))
+    model_version          = Column(String(100))
+    created_at             = Column(DateTime, default=datetime.utcnow)
+
+class CallAudioSummary(Base):
+    __tablename__ = "call_audio_summaries"
+
+    id                             = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    call_id                        = Column(String(100), nullable=False)
+    domain                         = Column(String(50))
+    model_version                  = Column(String(100))
+    has_audio_features             = Column(Boolean, default=False)
+    audio_feature_version          = Column(String(100))
+    audio_feature_match_summary    = Column(JSONB)
+    dashboard_audio_feature_series = Column(JSONB)
+    call_summary                   = Column(JSONB)
+    created_at                     = Column(DateTime, default=datetime.utcnow)

backend/app/pipeline_bridge.py

@@ -0,0 +1,39 @@
+"""
+The one place the backend reaches into the ml-services pipeline.
+
+The evaluation/ and pipeline/ modules import each other by bare top-level
+names (`import paths`, `from graph import ...`) and were only ever run with
+those dirs on sys.path. This shim inserts them exactly once and is imported
+LAZILY from inside the worker thread -- never at app import time -- so a plain
+`import app.main` (health checks, artifact routes) never drags in torch,
+langgraph, or faster-whisper.
+"""
+import sys
+from pathlib import Path
+
+_REPO = Path(__file__).resolve().parents[2]          # backend/app -> backend -> repo
+_ML = _REPO / "ml-services"
+# order matters: pipeline/evaluation/scripts hold the bare-name modules;
+# _ML itself makes `import src.inference...` (the vendored acoustic pkg) resolve
+_PATHS = [_ML / "pipeline", _ML / "evaluation", _ML / "scripts", _ML]
+
+_installed = False
+
+
+def install():
+    """Idempotently put the ml-services module dirs on sys.path."""
+    global _installed
+    if _installed:
+        return
+    for p in _PATHS:
+        sp = str(p)
+        if p.exists() and sp not in sys.path:
+            sys.path.insert(0, sp)
+    _installed = True
+
+
+def get_process_call():
+    """Return orchestrator.process_call (heavy imports; worker-thread only)."""
+    install()
+    from orchestrator import process_call
+    return process_call

backend/app/routers/artifacts.py

@@ -0,0 +1,61 @@
+"""
+Serves the frontend's static-fetch contract from Supabase Storage + the DB.
+
+The React app fetches /calls_index.json, /calls/{id}.json,
+/sentence_segments/{id}.json, /sentiment/{id}.json and /audio/{id}.mp3. These
+were static files under frontend/public; here the same paths become API routes
+so the Vercel-hosted SPA reads seeded and freshly-ingested calls identically.
+
+JSON artifacts are proxied (small; keeps the API's CORS headers in front of the
+Vercel origin); audio 307-redirects to the Supabase CDN, which owns Range +
+bandwidth. calls_index is built live from the Call rows' index_summary.
+"""
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import JSONResponse, RedirectResponse
+from sqlalchemy.orm import Session
+
+from app.database import get_db
+from app.models import Call
+from app import storage
+
+router = APIRouter(tags=["artifacts"])
+
+
+@router.get("/calls_index.json")
+def calls_index(db: Session = Depends(get_db)):
+    """Dashboard list: every call's index_summary, newest first."""
+    rows = db.query(Call).order_by(Call.created_at.desc()).all()
+    index = [(c.call_metadata or {}).get("index_summary") for c in rows]
+    return JSONResponse([s for s in index if s])
+
+
+def _json_or_404(key):
+    try:
+        return JSONResponse(storage.stream_json(key))
+    except FileNotFoundError:
+        raise HTTPException(status_code=404, detail="artifact not found")
+
+
+@router.get("/calls/{call_id}.json")
+def call_json(call_id: str):
+    return _json_or_404(f"calls/{call_id}.json")
+
+
+@router.get("/sentence_segments/{call_id}.json")
+def sentence_segments(call_id: str):
+    return _json_or_404(f"sentence_segments/{call_id}.json")
+
+
+@router.get("/sentiment/{call_id}.json")
+def sentiment(call_id: str):
+    # optional overlay -> empty (not 404) so CallDetail degrades to plain turns
+    try:
+        return JSONResponse(storage.stream_json(f"sentiment/{call_id}.json"))
+    except FileNotFoundError:
+        return JSONResponse({"segments": []})
+
+
+@router.get("/audio/{call_id}.mp3")
+def audio(call_id: str):
+    return RedirectResponse(
+        storage.public_url(f"audio/{call_id}.mp3"), status_code=307)

backend/app/routers/calls.py

@@ -0,0 +1,325 @@
+import uuid
+import os
+import shutil
+from datetime import datetime
+from fastapi import APIRouter, Depends, UploadFile, File, Form, HTTPException, status
+from pydantic import BaseModel, Field
+from sqlalchemy.orm import Session
+from app.database import get_db
+from app.models import Call, Job, Transcript, Evaluation, SentimentSegment
+from app import worker, storage
+from app.routing import apply_action_routing
+
+router = APIRouter(prefix="/calls", tags=["calls"])
+
+
+class AnalyzeRequest(BaseModel):
+    call_ids: list[uuid.UUID] = Field(min_length=1, max_length=10)
+
+
+def _latest_job(db, call_id):
+    return (
+        db.query(Job)
+        .filter(Job.call_id == call_id)
+        .order_by(Job.created_at.desc())
+        .first()
+    )
+
+
+def _incoming_dir(public_id):
+    """Local scratch dir for this call's channel wavs (under CAPSTONE_DATA_ROOT)."""
+    from app.pipeline_bridge import install
+    install()
+    import paths
+    d = paths.DATA_ROOT / "_incoming" / public_id
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+@router.post("/ingest", status_code=201)
+async def ingest_call(
+    agent_id: str = Form(...),
+    domain: str = Form(...),
+    accent: str = Form("en-US_General"),
+    call_date: str = Form(None),
+    duration_seconds: int = Form(None),
+    file: UploadFile = File(...),
+    file2: UploadFile = File(None),
+    db: Session = Depends(get_db)
+):
+    """
+    Ingest one fresh call. Accepts either a single stereo file (ch0=agent,
+    ch1=customer) or two per-channel mono files. Splits/normalizes to two mono
+    16 kHz WAVs, stores the originals for restart recovery, and enqueues the
+    worker. A single mono file with no second channel is rejected (400).
+    """
+    from app.pipeline_bridge import install
+    install()
+    from audio_io import prepare_channels, AudioError
+
+    public_id = f"{domain.lower()}_{datetime.utcnow():%Y%m%d}_{uuid.uuid4().hex[:8]}"
+    inc = _incoming_dir(public_id)
+
+    raw1 = inc / ("upload1_" + os.path.basename(file.filename or "a.wav"))
+    with open(raw1, "wb") as buf:
+        shutil.copyfileobj(file.file, buf)
+    raw2 = None
+    if file2 is not None:
+        raw2 = inc / ("upload2_" + os.path.basename(file2.filename or "b.wav"))
+        with open(raw2, "wb") as buf:
+            shutil.copyfileobj(file2.file, buf)
+
+    agent_wav = inc / "agent.wav"
+    customer_wav = inc / "customer.wav"
+    try:
+        prepare_channels(str(raw1), str(agent_wav), str(customer_wav),
+                         secondary=str(raw2) if raw2 else None)
+    except AudioError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+    # originals to storage so a restart mid-job can re-download and resume
+    storage.upload_file(f"uploads/{public_id}/agent.wav", str(agent_wav), "audio/wav")
+    storage.upload_file(f"uploads/{public_id}/customer.wav", str(customer_wav), "audio/wav")
+
+    call = Call(
+        agent_id=uuid.UUID(agent_id),
+        audio_path=f"uploads/{public_id}/",
+        call_date=datetime.fromisoformat(call_date) if call_date else datetime.utcnow(),
+        duration_seconds=duration_seconds,
+        call_metadata={"public_call_id": public_id,
+                       "domain": domain.lower(), "accent": accent},
+    )
+    db.add(call)
+    db.flush()
+
+    job = Job(call_id=call.call_id, status="queued", stage="uploaded")
+    db.add(job)
+    db.commit()
+    db.refresh(job)
+
+    worker.enqueue(job.job_id)
+
+    return {
+        "job_id": str(job.job_id),
+        "call_id": str(call.call_id),
+        "public_call_id": public_id,
+        "status": "queued",
+        "created_at": job.created_at.isoformat()
+    }
+
+
+@router.get("/catalog")
+def get_catalog(db: Session = Depends(get_db)):
+    """Return analyzed and staged calls in one dashboard-facing shape."""
+    rows = db.query(Call).order_by(Call.created_at.desc()).all()
+    catalog = []
+    for call in rows:
+        meta = dict(call.call_metadata or {})
+        public_id = meta.get("public_call_id")
+        if not public_id:
+            continue
+
+        summary = dict(meta.get("index_summary") or {})
+        job = _latest_job(db, call.call_id)
+        analyzed = bool(summary)
+        item = {
+            **summary,
+            "call_id": public_id,
+            "db_call_id": str(call.call_id),
+            "domain": summary.get("domain") or meta.get("domain"),
+            "accent": summary.get("accent") or meta.get("accent"),
+            "duration": summary.get("duration") or call.duration_seconds,
+            "analyzed": analyzed,
+            "status": job.status if job else ("analyzed" if analyzed else "available"),
+            "stage": job.stage if job else ("done" if analyzed else "uploaded"),
+            "error": job.error if job else None,
+            "job_id": str(job.job_id) if job else None,
+        }
+        catalog.append(item)
+    return catalog
+
+
+@router.post("/analyze", status_code=status.HTTP_202_ACCEPTED)
+def analyze_calls(payload: AnalyzeRequest, db: Session = Depends(get_db)):
+    """Queue staged calls, reusing an active job when a request is repeated."""
+    calls = {
+        call.call_id: call
+        for call in db.query(Call).filter(Call.call_id.in_(payload.call_ids)).all()
+    }
+    missing = [str(call_id) for call_id in payload.call_ids if call_id not in calls]
+    if missing:
+        raise HTTPException(
+            status_code=404,
+            detail={"message": "call not found", "call_ids": missing},
+        )
+
+    jobs = []
+    for call_id in payload.call_ids:
+        call = calls[call_id]
+        meta = dict(call.call_metadata or {})
+        public_id = meta.get("public_call_id")
+        if not public_id:
+            raise HTTPException(status_code=409, detail=f"call {call_id} is not pipeline-managed")
+
+        active = _latest_job(db, call_id)
+        if active and active.status in ("queued", "processing"):
+            jobs.append({
+                "call_id": str(call_id),
+                "public_call_id": public_id,
+                "job_id": str(active.job_id),
+                "status": active.status,
+                "created": False,
+            })
+            continue
+
+        if meta.get("index_summary"):
+            jobs.append({
+                "call_id": str(call_id),
+                "public_call_id": public_id,
+                "job_id": str(active.job_id) if active else None,
+                "status": "already_analyzed",
+                "created": False,
+            })
+            continue
+
+        job = Job(call_id=call_id, status="queued", stage="uploaded")
+        db.add(job)
+        db.flush()
+        jobs.append({
+            "call_id": str(call_id),
+            "public_call_id": public_id,
+            "job_id": str(job.job_id),
+            "status": job.status,
+            "created": True,
+        })
+
+    db.commit()
+    for item in jobs:
+        if item["created"]:
+            worker.enqueue(item["job_id"])
+    return {"jobs": jobs}
+
+@router.get("/{call_id}/status")
+def get_call_status(call_id: str, db: Session = Depends(get_db)):
+    try:
+        db_call_id = uuid.UUID(call_id)
+    except ValueError:
+        raise HTTPException(status_code=400, detail="invalid call id")
+    job = _latest_job(db, db_call_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="call not found")
+    call = db.query(Call).filter(Call.call_id == db_call_id).first()
+    public_id = (call.call_metadata or {}).get("public_call_id") if call else None
+    return {
+        "call_id": call_id,
+        "job_id": str(job.job_id),
+        "public_call_id": public_id,
+        "status": job.status,
+        "stage": job.stage,
+        "error": job.error,
+        "updated_at": job.updated_at.isoformat() if job.updated_at else None
+    }
+
+@router.post("/transcripts/ingest", status_code=201)
+async def ingest_transcript(
+    payload: dict,
+    db: Session = Depends(get_db)
+):
+    call_id_str = payload.get("call_id")
+    sentences = payload.get("sentences", [])
+
+    inserted = 0
+    for sentence in sentences:
+        transcript = Transcript(
+            call_id=uuid.UUID(call_id_str) if len(call_id_str) == 36 else None,
+            source_call_id=call_id_str,
+            turn_id=sentence.get("seq_id", 0),
+            speaker=sentence.get("speaker", "").lower(),
+            start_time=sentence.get("start"),
+            end_time=sentence.get("end"),
+            text=sentence.get("text"),
+            avg_confidence=1.0,
+            low_confidence=False
+        )
+        db.add(transcript)
+        inserted += 1
+
+    db.commit()
+
+    return {
+        "call_id": call_id_str,
+        "inserted": inserted,
+        "status": "complete"
+    }
+
+@router.post("/evaluate", status_code=201)
+async def ingest_evaluation(
+    payload: dict,
+    db: Session = Depends(get_db)
+):
+    """
+    Receives Llama/Aniket evaluation output and applies action routing logic.
+    """
+    call_id_str = payload.get("call_id")
+    agent_id_str = payload.get("agent_id")
+
+    # Get max escalation score from sentiment table
+    sentiment_segments = db.query(SentimentSegment).filter(
+        SentimentSegment.call_id == call_id_str
+    ).all()
+
+    scores = [s.escalation_score for s in sentiment_segments if s.escalation_score is not None]
+    max_escalation = max(scores) if scores else None
+
+    # Create evaluation record
+    evaluation = Evaluation(
+        call_id=uuid.UUID(call_id_str) if len(call_id_str) == 36 else None,
+        agent_id=uuid.UUID(agent_id_str) if agent_id_str else None,
+        overall_grade=payload.get("overall_grade"),
+        weighted_score=payload.get("weighted_score"),
+        scorecard=payload.get("scorecard"),
+        compliance_flags=payload.get("compliance_flags"),
+        escalation_risk=payload.get("escalation_risk"),
+        llm_scored=payload.get("llm_scored", True)
+    )
+
+    # Apply action routing
+    evaluation = apply_action_routing(evaluation, max_escalation)
+
+    db.add(evaluation)
+    db.commit()
+    db.refresh(evaluation)
+
+    return {
+        "call_id": call_id_str,
+        "evaluation_id": str(evaluation.evaluation_id),
+        "overall_grade": evaluation.overall_grade,
+        "escalation_flag": evaluation.escalation_flag,
+        "coaching_required": evaluation.coaching_required,
+        "manual_review_required": evaluation.manual_review_required,
+        "status": "complete"
+    }
+
+@router.get("/{call_id}/flags")
+def get_call_flags(call_id: str, db: Session = Depends(get_db)):
+    """
+    Returns the action flags for a call — useful for dashboard alerts.
+    """
+    try:
+        evaluation = db.query(Evaluation).filter(
+            Evaluation.call_id == uuid.UUID(call_id)
+        ).first()
+    except Exception:
+        return {"error": "invalid call_id format"}
+
+    if not evaluation:
+        return {"error": "no evaluation found for this call"}
+
+    return {
+        "call_id": call_id,
+        "overall_grade": evaluation.overall_grade,
+        "escalation_flag": evaluation.escalation_flag,
+        "coaching_required": evaluation.coaching_required,
+        "manual_review_required": evaluation.manual_review_required,
+        "escalation_risk": evaluation.escalation_risk,
+        "weighted_score": float(evaluation.weighted_score) if evaluation.weighted_score else None
+    }

backend/app/routers/sentiment.py

@@ -0,0 +1,138 @@
+from fastapi import APIRouter, Depends
+from sqlalchemy.orm import Session
+from app.database import get_db
+from app.models import SentimentSegment, CallAudioSummary
+
+router = APIRouter(prefix="/sentiment", tags=["sentiment"])
+
+@router.post("/ingest", status_code=201)
+async def ingest_sentiment(
+    payload: dict,
+    db: Session = Depends(get_db)
+):
+    call_id_str = payload.get("call_id")
+    domain = payload.get("domain")
+    model_version = payload.get("model_version")
+    has_audio_features = payload.get("has_audio_features", False)
+    audio_feature_version = payload.get("audio_feature_version")
+    segments = payload.get("segments", [])
+    call_summary = payload.get("call_summary")
+    audio_feature_match_summary = payload.get("audio_feature_match_summary")
+    dashboard_audio_feature_series = payload.get("dashboard_audio_feature_series")
+
+    # Save or update call-level audio summary
+    existing_summary = db.query(CallAudioSummary).filter(
+        CallAudioSummary.call_id == call_id_str
+    ).first()
+
+    if not existing_summary:
+        summary_row = CallAudioSummary(
+            call_id=call_id_str,
+            domain=domain,
+            model_version=model_version,
+            has_audio_features=has_audio_features,
+            audio_feature_version=audio_feature_version,
+            audio_feature_match_summary=audio_feature_match_summary,
+            dashboard_audio_feature_series=dashboard_audio_feature_series,
+            call_summary=call_summary
+        )
+        db.add(summary_row)
+
+    # Save segment-level data
+    inserted = 0
+    skipped = 0
+
+    for segment in segments:
+        existing = db.query(SentimentSegment).filter(
+            SentimentSegment.segment_key == segment.get("segment_key")
+        ).first()
+
+        if existing:
+            skipped += 1
+            continue
+
+        audio_features = segment.get("audio_features")
+
+        sentiment_row = SentimentSegment(
+            call_id=call_id_str,
+            segment_index=segment.get("segment_index"),
+            segment_key=segment.get("segment_key"),
+            seq_id=segment.get("seq_id"),
+            speaker=segment.get("speaker"),
+            start_time=segment.get("start_time"),
+            end_time=segment.get("end_time"),
+            text=segment.get("text"),
+            sentiment=segment.get("sentiment"),
+            dominant_emotion=segment.get("dominant_emotion"),
+            escalation_score=segment.get("escalation_score"),
+            processing_status=segment.get("processing_status"),
+            audio_features=audio_features,
+            explainability_flags=segment.get("explainability_flags"),
+            escalation_explanation=segment.get("escalation_explanation"),
+            has_audio_features=audio_features is not None,
+            audio_feature_version=audio_feature_version,
+            domain=domain,
+            model_version=model_version
+        )
+        db.add(sentiment_row)
+        inserted += 1
+
+    db.commit()
+
+    return {
+        "call_id": call_id_str,
+        "inserted": inserted,
+        "skipped_duplicates": skipped,
+        "has_audio_features": has_audio_features,
+        "status": "complete"
+    }
+
+@router.get("/{call_id}/summary")
+def get_sentiment_summary(call_id: str, db: Session = Depends(get_db)):
+    segments = db.query(SentimentSegment).filter(
+        SentimentSegment.call_id == call_id
+    ).all()
+
+    if not segments:
+        return {"error": "no sentiment data found for this call"}
+
+    successful = [s for s in segments if s.processing_status == "success"]
+    skipped = [s for s in segments if s.processing_status == "skipped_too_short"]
+
+    sentiments = [s.sentiment for s in successful if s.sentiment]
+    dominant_sentiment = max(set(sentiments), key=sentiments.count) if sentiments else None
+
+    emotions = [s.dominant_emotion for s in successful if s.dominant_emotion]
+    dominant_emotion = max(set(emotions), key=emotions.count) if emotions else None
+
+    scores = [s.escalation_score for s in successful if s.escalation_score is not None]
+    avg_escalation = round(sum(scores) / len(scores), 4) if scores else None
+    max_escalation = round(max(scores), 4) if scores else None
+
+    return {
+        "call_id": call_id,
+        "total_segments": len(segments),
+        "successful_segments": len(successful),
+        "skipped_segments": len(skipped),
+        "dominant_sentiment": dominant_sentiment,
+        "dominant_emotion": dominant_emotion,
+        "average_escalation_score": avg_escalation,
+        "max_escalation_score": max_escalation
+    }
+
+@router.get("/{call_id}/dashboard")
+def get_dashboard_series(call_id: str, db: Session = Depends(get_db)):
+    summary = db.query(CallAudioSummary).filter(
+        CallAudioSummary.call_id == call_id
+    ).first()
+
+    if not summary:
+        return {"error": "no audio summary found for this call"}
+
+    return {
+        "call_id": call_id,
+        "domain": summary.domain,
+        "has_audio_features": summary.has_audio_features,
+        "call_summary": summary.call_summary,
+        "dashboard_audio_feature_series": summary.dashboard_audio_feature_series
+    }

backend/app/routing.py

@@ -0,0 +1,32 @@
+"""
+Deterministic action-routing: read an Evaluation's scores, set its flags.
+
+Lives here (not in the FastAPI router) so both the request handler and the
+background worker can call it without the worker importing the web layer.
+"""
+
+
+def apply_action_routing(evaluation, max_escalation_score=None):
+    """Set escalation/coaching/manual-review flags from the scores in place."""
+    # text escalation risk from the LLM (0-10 scale)
+    if evaluation.escalation_risk is not None:
+        if evaluation.escalation_risk >= 7:
+            evaluation.escalation_flag = True
+            evaluation.coaching_required = True
+        elif evaluation.escalation_risk >= 4:
+            evaluation.coaching_required = True
+
+    # grade-based routing
+    if evaluation.overall_grade == "F":
+        evaluation.manual_review_required = True
+        evaluation.coaching_required = True
+
+    # acoustic escalation (wav2vec2 max segment score, 0.0-1.0)
+    if max_escalation_score is not None:
+        if max_escalation_score >= 0.7:
+            evaluation.escalation_flag = True
+            evaluation.coaching_required = True
+        elif max_escalation_score >= 0.4:
+            evaluation.coaching_required = True
+
+    return evaluation

backend/app/services/alignment.py

@@ -0,0 +1,8 @@
+class TranscriptSpeakerAligner:
+    def __init__(self):
+        pass
+
+    def align(self, diarization_turns: list, transcription_words: list):
+        # Aligns word boundaries with speaker diarization turns
+        aligned_segments = []
+        return aligned_segments

backend/app/services/batch_service.py

@@ -0,0 +1,11 @@
+import os
+
+class BatchFileService:
+    def __init__(self, watch_dir: str = "/data"):
+        self.watch_dir = watch_dir
+
+    def scan_records(self):
+        # Reads local AppTek records and maps metadata keys from disk
+        if not os.path.exists(self.watch_dir):
+            return []
+        return os.listdir(self.watch_dir)

backend/app/space.py

@@ -0,0 +1,55 @@
+import os
+
+import gradio as gr
+from fastapi.responses import RedirectResponse
+from sqlalchemy import func
+
+from app.database import SessionLocal
+from app.main import app as api
+from app.models import Call, Job
+from app.storage import is_configured
+
+
+def runtime_snapshot():
+    snapshot = {
+        "service": "call-qa-processing",
+        "database_configured": bool(os.environ.get("DATABASE_URL")),
+        "storage_configured": is_configured(),
+        "acoustic_enabled": os.environ.get("ENABLE_ACOUSTIC", "0") == "1",
+    }
+    db = SessionLocal()
+    try:
+        snapshot["calls"] = db.query(func.count(Call.call_id)).scalar() or 0
+        snapshot["jobs"] = {
+            status: db.query(func.count(Job.job_id)).filter(Job.status == status).scalar() or 0
+            for status in ("queued", "processing", "succeeded", "failed")
+        }
+    except Exception as exc:
+        snapshot["database_error"] = str(exc)[:240]
+    finally:
+        db.close()
+    return snapshot
+
+
+with gr.Blocks(title="Call QA Processing") as demo:
+    gr.Markdown("# Call QA Processing")
+    state = gr.JSON(label="Runtime status")
+    refresh = gr.Button("Refresh", variant="primary")
+    refresh.click(runtime_snapshot, outputs=state, api_name="runtime_status")
+    demo.load(runtime_snapshot, outputs=state)
+
+
+# Replace the JSON root with the Space UI while preserving every API route.
+api.router.routes = [
+    route
+    for route in api.router.routes
+    if not (getattr(route, "path", None) == "/" and "GET" in getattr(route, "methods", set()))
+]
+
+
+@api.get("/", include_in_schema=False)
+def space_root():
+    return RedirectResponse(url="/gradio/")
+
+
+app = gr.mount_gradio_app(api, demo, path="/gradio")

backend/app/storage.py

@@ -0,0 +1,125 @@
+"""
+Supabase Storage access over plain REST (no SDK dependency).
+
+Artifacts (call JSON, mp3, sentence_segments, sentiment, raw transcript, and
+the original upload for restart recovery) live in one PUBLIC bucket so reads
+need no auth and the Supabase CDN serves audio Range requests directly. Writes
+use the service-role key.
+
+Layout in the bucket (keys):
+  calls/{id}.json  audio/{id}.mp3  sentence_segments/{id}.json
+  sentiment/{id}.json  transcripts/{id}.json  uploads/{id}/<name>
+"""
+import os
+import json
+import mimetypes
+from pathlib import Path
+
+import requests
+
+BUCKET = os.environ.get("SUPABASE_BUCKET", "call-artifacts")
+_TIMEOUT = 30
+
+
+class StorageError(RuntimeError):
+    pass
+
+
+def _base():
+    url = os.environ.get("SUPABASE_URL", "").rstrip("/")
+    if not url:
+        raise StorageError("SUPABASE_URL is not set")
+    return url
+
+
+def _service_key():
+    # secret/service-role key preferred (storage writes bypass RLS); falls back
+    # to SUPABASE_KEY, which only authorizes writes if the bucket grants anon
+    # insert. Reads from a public bucket work with either.
+    key = os.environ.get("SUPABASE_SERVICE_KEY") or os.environ.get("SUPABASE_KEY", "")
+    if not key:
+        raise StorageError("neither SUPABASE_SERVICE_KEY nor SUPABASE_KEY is set")
+    return key
+
+
+def is_configured():
+    return bool(os.environ.get("SUPABASE_URL")
+                and (os.environ.get("SUPABASE_SERVICE_KEY")
+                     or os.environ.get("SUPABASE_KEY")))
+
+
+def object_url(key):
+    """Authenticated object endpoint (used for writes)."""
+    return f"{_base()}/storage/v1/object/{BUCKET}/{key}"
+
+
+def public_url(key):
+    """Public read URL served by the Supabase CDN (Range-capable)."""
+    return f"{_base()}/storage/v1/object/public/{BUCKET}/{key}"
+
+
+# ── writes ──────────────────────────────────────────────────────────────────
+
+def upload_bytes(key, data, content_type="application/octet-stream"):
+    # `apikey` header (not just Authorization: Bearer) is required for the new
+    # sb_secret_ key format -- the object endpoint parses Bearer tokens as JWTs
+    # and rejects the non-JWT key with "Invalid Compact JWS". Sending both is
+    # compatible with legacy JWT service_role keys too.
+    k = _service_key()
+    headers = {
+        "apikey": k,
+        "Authorization": f"Bearer {k}",
+        "Content-Type": content_type,
+        "x-upsert": "true",          # overwrite -> idempotent re-runs
+    }
+    r = requests.post(object_url(key), headers=headers, data=data, timeout=_TIMEOUT)
+    if r.status_code not in (200, 201):
+        raise StorageError(f"upload {key} failed: {r.status_code} {r.text[:200]}")
+    return key
+
+
+def upload_file(key, path, content_type=None):
+    if content_type is None:
+        content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
+    with open(path, "rb") as f:
+        return upload_bytes(key, f.read(), content_type)
+
+
+def upload_json(key, obj):
+    return upload_bytes(key, json.dumps(obj).encode("utf-8"), "application/json")
+
+
+# ── reads (public bucket; small local cache since artifacts are immutable) ──
+
+_CACHE_DIR = Path(os.environ.get("CAPSTONE_DATA_ROOT", "/tmp")) / "_storage_cache"
+
+
+def _cache_path(key):
+    return _CACHE_DIR / key
+
+
+def exists(key):
+    try:
+        r = requests.head(public_url(key), timeout=_TIMEOUT)
+        return r.status_code == 200
+    except requests.RequestException:
+        return False
+
+
+def download_bytes(key, use_cache=True):
+    cp = _cache_path(key)
+    if use_cache and cp.exists():
+        return cp.read_bytes()
+    r = requests.get(public_url(key), timeout=_TIMEOUT)
+    if r.status_code == 404:
+        raise FileNotFoundError(key)
+    if r.status_code != 200:
+        raise StorageError(f"download {key} failed: {r.status_code}")
+    if use_cache:
+        cp.parent.mkdir(parents=True, exist_ok=True)
+        cp.write_bytes(r.content)
+    return r.content
+
+
+def stream_json(key, use_cache=True):
+    return json.loads(download_bytes(key, use_cache=use_cache).decode("utf-8"))

backend/app/worker.py

@@ -0,0 +1,246 @@
+"""
+Single background worker: one daemon thread draining a job queue.
+
+Why one thread (not a pool): the evaluation graph fires many rate-limited LLM
+calls per call and CPU whisper/torch already saturate the 2-vCPU Space.
+Sequential processing keeps us under provider limits and off the CPU cliff.
+Jobs are cheap to enqueue, slow to run (~5-15 min), so queue + one worker is
+the right shape.
+
+Restart recovery: the Space's disk is ephemeral and it sleeps/restarts freely.
+On startup we re-enqueue every job still marked queued/processing and
+re-download its original channels from storage; the orchestrator is idempotent
+(an existing transcript is reused), so a re-run resumes rather than restarts.
+"""
+import os
+import uuid
+import queue
+import threading
+import traceback
+from datetime import datetime
+
+from app.database import SessionLocal
+from app import storage
+from app.models import (Call, Job, Transcript, Evaluation,
+                        SentimentSegment, CallAudioSummary)
+from app.routing import apply_action_routing
+
+_q = queue.Queue()
+_thread = None
+_started = False
+
+# text escalation tier -> Asma's 0-10 escalation_risk scale
+ESCALATION_MAP = {"none": 0, "review": 5, "escalate": 8}
+
+
+def _uuid(v):
+    return v if isinstance(v, uuid.UUID) else uuid.UUID(str(v))
+
+
+def enqueue(job_id):
+    _q.put(str(job_id))
+
+
+def start():
+    """Launch the worker thread and recover interrupted jobs (idempotent)."""
+    global _thread, _started
+    if _started:
+        return
+    _started = True
+    _thread = threading.Thread(target=_loop, name="pipeline-worker", daemon=True)
+    _thread.start()
+    _recover()
+
+
+def _recover():
+    """Re-enqueue jobs left queued/processing by a previous (crashed) run --
+    but ONLY jobs from this pipeline's ingest path (call_metadata carries a
+    public_call_id). Legacy/foreign jobs are left untouched."""
+    db = SessionLocal()
+    try:
+        stuck = db.query(Job).filter(Job.status.in_(["queued", "processing"])).all()
+        recovered = 0
+        for j in stuck:
+            call = db.query(Call).filter(Call.call_id == j.call_id).first()
+            meta = (call.call_metadata or {}) if call else {}
+            if meta.get("public_call_id"):
+                enqueue(j.job_id)
+                recovered += 1
+        if recovered:
+            print(f"[worker] recovered {recovered} interrupted job(s)")
+    except Exception as e:
+        print(f"[worker] recovery skipped (db unreachable?): {e}")
+    finally:
+        db.close()
+
+
+def _loop():
+    while True:
+        job_id = _q.get()
+        try:
+            _run_job(job_id)
+        except Exception:
+            traceback.print_exc()
+        finally:
+            _q.task_done()
+
+
+def _set(db, job, status=None, stage=None, error=None):
+    if status is not None:
+        job.status = status
+    if stage is not None:
+        job.stage = stage
+    if error is not None:
+        job.error = error[:2000]
+    job.updated_at = datetime.utcnow()
+    db.commit()
+
+
+def _ensure_local_wavs(public_id, meta):
+    """Return local (agent_wav, customer_wav), re-downloading from storage if
+    the ephemeral disk was wiped (restart recovery)."""
+    from app.pipeline_bridge import install
+    install()
+    import paths
+    dest = paths.DATA_ROOT / "_incoming" / public_id
+    dest.mkdir(parents=True, exist_ok=True)
+    out = {}
+    for role in ("agent", "customer"):
+        local = dest / f"{role}.wav"
+        if not local.exists():
+            data = storage.download_bytes(f"uploads/{public_id}/{role}.wav",
+                                          use_cache=False)
+            local.write_bytes(data)
+        out[role] = str(local)
+    return out["agent"], out["customer"]
+
+
+def _upload_artifacts(public_id, artifacts):
+    """Push every produced artifact to storage; return the key map."""
+    p = artifacts.get("paths", {})
+    keys = {}
+    plan = [
+        ("call_json", f"calls/{public_id}.json", "application/json"),
+        ("audio", f"audio/{public_id}.mp3", "audio/mpeg"),
+        ("sentence_segments", f"sentence_segments/{public_id}.json", "application/json"),
+        ("transcript", f"transcripts/{public_id}.json", "application/json"),
+    ]
+    if artifacts.get("acoustic") and p.get("sentiment"):
+        plan.append(("sentiment", f"sentiment/{public_id}.json", "application/json"))
+    for local_key, obj_key, ctype in plan:
+        local = p.get(local_key)
+        if local and os.path.exists(local):
+            storage.upload_file(obj_key, local, ctype)
+            keys[local_key] = obj_key
+    return keys
+
+
+def _write_db_rows(db, call, public_id, artifacts):
+    """Populate the relational tables from the produced artifacts. The frontend
+    reads artifacts from storage; these rows back the flags/summary endpoints."""
+    import json
+    p = artifacts["paths"]
+
+    # transcripts: one row per sentence segment
+    with open(p["sentence_segments"], encoding="utf-8") as f:
+        seg = json.load(f)
+    db.query(Transcript).filter(Transcript.source_call_id == public_id).delete()
+    for s in seg.get("sentences", []):
+        db.add(Transcript(
+            call_id=call.call_id, source_call_id=public_id,
+            turn_id=s.get("seq_id", 0), speaker=(s.get("speaker") or "").lower(),
+            start_time=s.get("start"), end_time=s.get("end"),
+            text=s.get("text"), avg_confidence=1.0, low_confidence=False))
+
+    # sentiment segments (acoustic only) + max escalation for routing
+    max_escalation = None
+    if artifacts.get("acoustic") and p.get("sentiment"):
+        with open(p["sentiment"], encoding="utf-8") as f:
+            sent = json.load(f)
+        db.query(SentimentSegment).filter(
+            SentimentSegment.call_id == public_id).delete()
+        # model_version column is varchar(100); guard against long values
+        model_version = (sent.get("model_version") or "")[:100]
+        scores = []
+        for i, s in enumerate(sent.get("segments", [])):
+            db.add(SentimentSegment(
+                call_id=public_id, segment_index=i,
+                segment_key=f"{public_id}_{s.get('seq_id')}",
+                seq_id=s.get("seq_id"), sentiment=s.get("sentiment"),
+                dominant_emotion=s.get("dominant_emotion"),
+                escalation_score=s.get("escalation_score"),
+                processing_status=s.get("processing_status"),
+                domain=artifacts.get("domain"),
+                model_version=model_version))
+            if s.get("escalation_score") is not None:
+                scores.append(s["escalation_score"])
+        max_escalation = max(scores) if scores else None
+
+    # evaluation: full graph.json as the scorecard, tier -> 0-10 risk
+    with open(p["evaluation"], encoding="utf-8") as f:
+        ev = json.load(f)
+    risk_level = (ev.get("escalation") or {}).get("risk_level", "none")
+    db.query(Evaluation).filter(Evaluation.call_id == call.call_id).delete()
+    evaluation = Evaluation(
+        call_id=call.call_id, agent_id=call.agent_id,
+        scorecard=ev, compliance_flags=ev.get("compliance"),
+        escalation_risk=ESCALATION_MAP.get(risk_level, 0), llm_scored=True)
+    apply_action_routing(evaluation, max_escalation)
+    db.add(evaluation)
+
+
+def _run_job(job_id):
+    db = SessionLocal()
+    try:
+        job = db.query(Job).filter(Job.job_id == _uuid(job_id)).first()
+        if not job:
+            return
+        call = db.query(Call).filter(Call.call_id == job.call_id).first()
+        meta = dict(call.call_metadata or {}) if call else {}
+        public_id = meta.get("public_call_id")
+        if not public_id:
+            # not one of ours (e.g. a legacy job) -- refuse rather than crash
+            _set(db, job, status="failed",
+                 error="call has no public_call_id; not an ingest-pipeline call")
+            return
+
+        _set(db, job, status="processing", stage="starting", error="")
+
+        agent_wav, customer_wav = _ensure_local_wavs(public_id, meta)
+        spec = {"call_id": public_id, "domain": meta["domain"],
+                "accent": meta["accent"], "agent_wav": agent_wav,
+                "customer_wav": customer_wav}
+
+        def progress(stage):
+            _set(db, job, stage=stage)
+
+        from app.pipeline_bridge import get_process_call
+        process_call = get_process_call()
+        artifacts = process_call(spec, progress=progress)
+
+        _set(db, job, stage="uploading")
+        keys = _upload_artifacts(public_id, artifacts)
+
+        _set(db, job, stage="persisting")
+        _write_db_rows(db, call, public_id, artifacts)
+
+        summary = artifacts.get("index_summary") or {}
+        meta["index_summary"] = summary
+        meta["artifact_keys"] = keys
+        call.call_metadata = meta
+        if summary.get("duration"):
+            call.duration_seconds = int(summary["duration"])
+        db.commit()
+
+        _set(db, job, status="succeeded", stage="done", error="")
+        print(f"[worker] job {job_id} succeeded ({public_id})")
+    except Exception as e:
+        traceback.print_exc()
+        try:
+            job = db.query(Job).filter(Job.job_id == _uuid(job_id)).first()
+            if job:
+                _set(db, job, status="failed", error=str(e))
+        except Exception:
+            pass
+    finally:
+        db.close()

ml-services/evaluation/assemble.py

@@ -0,0 +1,148 @@
+"""
+Phase 0 context-engineering: turn our per-channel transcript into the
+evaluation INPUT PACKET the LLM will consume.
+
+Steps (deterministic, no LLM):
+  1. Role mapping  -- channel identity IS the speaker (AGENT=ch1, CUSTOMER=ch2).
+                      No diarization, no "who spoke first" heuristic -> 100% correct.
+  2. Turn ordering -- merge both channels' segments by start time into a single
+                      chronological [AGENT]/[CUSTOMER] turn list with mm:ss stamps.
+  3. Metadata injection -- prepend call metadata (domain as "Call Type" lens,
+                      duration, etc.) so the LLM judges with the right context.
+
+Output: a single text packet (+ saved .txt) ready to drop into the Phase 1 prompt.
+
+Usage:
+  python assemble.py --result ../../data/na_testset/results_channels/en-CA/en_CA_Banking_1592237.json
+  python assemble.py --call_id en_CA_Banking_1592237   # auto-locates result + manifest
+"""
+import os, json, argparse
+
+import paths
+
+DATA = str(paths.NA_TESTSET)
+MANIFEST = str(paths.MANIFEST)
+
+
+def mmss(seconds):
+    seconds = int(round(seconds))
+    return f"{seconds // 60:02d}:{seconds % 60:02d}"
+
+
+def load_manifest():
+    """Dataset manifest merged with the runtime registry of ingested calls.
+
+    Runtime entries win on call_id collision so a re-ingested call can carry
+    corrected metadata without touching the dataset manifest.
+    """
+    with open(MANIFEST, encoding="utf-8") as f:
+        manifest = {m["call_id"]: m for m in json.load(f)}
+    if paths.RUNTIME_MANIFEST.exists():
+        with open(paths.RUNTIME_MANIFEST, encoding="utf-8") as f:
+            manifest.update({m["call_id"]: m for m in json.load(f)})
+    return manifest
+
+
+def assemble_turns(result):
+    """Merge agent+customer segments into one time-ordered turn list."""
+    turns = []
+    segs = result.get("segments", {})
+    if segs and segs.get("agent") is not None:
+        for speaker, key in (("AGENT", "agent"), ("CUSTOMER", "customer")):
+            for s in segs.get(key, []):
+                text = s.get("text", "").strip()
+                if text:
+                    turns.append({"start": s["start"], "speaker": speaker, "text": text})
+    else:
+        # fallback: no segments -> group words by speaker flips
+        merged = []
+        for speaker, key in (("AGENT", "agent"), ("CUSTOMER", "customer")):
+            for w in result.get(key, []):
+                merged.append((w["start"], speaker, w["word"]))
+        merged.sort()
+        cur_sp, cur_words, cur_start = None, [], None
+        for start, sp, word in merged:
+            if sp != cur_sp and cur_words:
+                turns.append({"start": cur_start, "speaker": cur_sp, "text": " ".join(cur_words)})
+                cur_words = []
+            if sp != cur_sp:
+                cur_sp, cur_start = sp, start
+            cur_words.append(word)
+        if cur_words:
+            turns.append({"start": cur_start, "speaker": cur_sp, "text": " ".join(cur_words)})
+
+    turns.sort(key=lambda t: t["start"])
+    return turns
+
+
+def estimate_duration(result):
+    last = 0.0
+    for key in ("agent", "customer"):
+        for w in result.get(key, []):
+            last = max(last, w.get("end", w.get("start", 0)))
+    return round(last, 1)
+
+
+def build_packet(result, meta):
+    turns = assemble_turns(result)
+    dur = estimate_duration(result)
+
+    # ── metadata header (the "lens") ──────────────────────────────────────────
+    header = [
+        "=" * 70,
+        "CALL METADATA",
+        "=" * 70,
+        f"Call ID      : {result.get('call_id', meta.get('call_id', '?'))}",
+        f"Call Type    : {result.get('domain', meta.get('domain', 'unknown'))}",
+        f"Duration     : {mmss(dur)} ({dur:.0f}s)",
+        f"Transcript   : {result.get('model', 'unknown')} (per-channel; AGENT=ch1, CUSTOMER=ch2)",
+        f"Turns        : {len(turns)}",
+        "",
+        "=" * 70,
+        "ROLE-MAPPED TRANSCRIPT  (chronological; [SPEAKER mm:ss])",
+        "=" * 70,
+    ]
+
+    body = [f"[{t['speaker']} {mmss(t['start'])}] {t['text']}" for t in turns]
+    return "\n".join(header) + "\n" + "\n".join(body), {"turns": len(turns), "duration_s": dur}
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--result", help="path to a per-channel result JSON")
+    ap.add_argument("--call_id", help="call id (auto-locates result via manifest)")
+    ap.add_argument("--results_dir", default="results_channels",
+                    help="result dir under na_testset (default: results_channels)")
+    ap.add_argument("--out", default=None, help="output .txt path (default: evaluation/packets/<id>.txt)")
+    args = ap.parse_args()
+
+    manifest = load_manifest()
+
+    if args.result:
+        path = args.result
+    elif args.call_id:
+        meta = manifest[args.call_id]
+        path = os.path.join(DATA, args.results_dir, meta["accent"], args.call_id + ".json")
+    else:
+        raise SystemExit("Provide --result or --call_id")
+
+    with open(path, encoding="utf-8") as f:
+        result = json.load(f)
+
+    cid = result.get("call_id")
+    meta = manifest.get(cid, {})
+    packet, stats = build_packet(result, meta)
+
+    here = os.path.dirname(os.path.abspath(__file__))
+    out = args.out or os.path.join(here, "packets", f"{cid}.txt")
+    os.makedirs(os.path.dirname(out), exist_ok=True)
+    with open(out, "w", encoding="utf-8") as f:
+        f.write(packet)
+
+    print(packet)
+    print("\n" + "-" * 70)
+    print(f"Saved packet -> {out}  ({stats['turns']} turns, {mmss(stats['duration_s'])})")
+
+
+if __name__ == "__main__":
+    main()

ml-services/evaluation/batch_sentence_segments.py

@@ -0,0 +1,207 @@
+"""
+Batch sentence segmentation for banking, health, and telecom calls.
+
+Reads every word-level transcription JSON from data/na_testset/results/
+for the target domains and produces sentence_segments.json per call under:
+
+  data/sentence_segments/
+    banking/
+      en_CA_Banking_1586889.json
+      ...
+    health/
+      en_CA_Health_1587315.json
+      ...
+    telecom/
+      en_CA_Telecom_1590675.json
+      ...
+
+Each output file has the same schema as frontend/src/sentence_segments.json
+so the frontend and sentiment pipeline can consume any call directly.
+
+Usage:  python batch_sentence_segments.py [--domains banking health telecom]
+"""
+import os
+import json
+import re
+import argparse
+from pathlib import Path
+
+import paths
+
+# ── config ────────────────────────────────────────────────────────────────────
+RESULTS_ROOT = paths.NA_TESTSET / "results"
+OUTPUT_ROOT  = paths.SENTENCE_SEG_ROOT
+
+DEFAULT_DOMAINS = {"banking", "health", "telecom"}
+
+MIN_DURATION_S = 1.5   # merge non-terminated fragments shorter than this
+GAP_SPLIT_S    = 2.0   # silence gap that always starts a new sentence
+MAX_MERGE_GAP  = 1.0   # only merge a short fragment if gap from prev ≤ this
+
+ABBREVIATIONS = {
+    "mr", "mrs", "ms", "dr", "prof", "sr", "jr", "vs", "st",
+    "ave", "blvd", "dept", "est", "approx", "etc", "e.g", "i.e",
+    "jan", "feb", "mar", "apr", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
+}
+SENTENCE_END = re.compile(r"[.?!]$")
+
+
+def is_sentence_end(word: str) -> bool:
+    if not SENTENCE_END.search(word):
+        return False
+    base = re.sub(r"[.?!,;:]+$", "", word).lower()
+    if base in ABBREVIATIONS:
+        return False
+    if re.match(r"^[a-z]$", base):
+        return False
+    return True
+
+
+def words_to_sentences(words: list, speaker: str) -> list:
+    if not words:
+        return []
+
+    raw = []
+    bucket = [words[0]]
+    for prev_w, curr_w in zip(words, words[1:]):
+        gap = curr_w["start"] - prev_w["end"]
+        if is_sentence_end(prev_w["word"]) or gap >= GAP_SPLIT_S:
+            raw.append(bucket)
+            bucket = [curr_w]
+        else:
+            bucket.append(curr_w)
+    if bucket:
+        raw.append(bucket)
+
+    merged = []
+    for sent_words in raw:
+        start = sent_words[0]["start"]
+        end   = sent_words[-1]["end"]
+        dur   = end - start
+        text  = " ".join(w["word"] for w in sent_words)
+        last_word_terminal = SENTENCE_END.search(sent_words[-1]["word"])
+
+        if merged and dur < MIN_DURATION_S and not last_word_terminal:
+            gap_from_prev = start - merged[-1]["end"]
+            if gap_from_prev <= MAX_MERGE_GAP:
+                prev = merged[-1]
+                prev["text"]     = prev["text"] + " " + text
+                prev["end"]      = end
+                prev["duration"] = round(prev["end"] - prev["start"], 3)
+                continue
+
+        merged.append({
+            "speaker":  speaker,
+            "start":    round(start, 3),
+            "end":      round(end, 3),
+            "duration": round(dur, 3),
+            "text":     text,
+        })
+
+    for idx, s in enumerate(merged):
+        s["id"] = f"{speaker}_{idx + 1:03d}"
+
+    return merged
+
+
+def interleave(agent_sents: list, customer_sents: list) -> list:
+    combined = sorted(agent_sents + customer_sents, key=lambda s: s["start"])
+    for i, s in enumerate(combined):
+        s["seq_id"] = i + 1
+    return combined
+
+
+def segment_transcript(data: dict) -> dict:
+    """Sentence-segment a loaded per-channel transcript dict (pure, no I/O).
+
+    Shared with the single-call orchestrator so batch and live paths produce
+    byte-identical segmentation from the same word-level transcript.
+    """
+    agent_sents    = words_to_sentences(data.get("agent", []),    "AGENT")
+    customer_sents = words_to_sentences(data.get("customer", []), "CUSTOMER")
+    all_sents      = interleave(agent_sents, customer_sents)
+
+    return {
+        "call_id":            data["call_id"],
+        "accent":             data.get("accent"),
+        "domain":             data.get("domain"),
+        "model":              data.get("model"),
+        "total":              len(all_sents),
+        "agent_sentences":    len(agent_sents),
+        "customer_sentences": len(customer_sents),
+        "sentences":          all_sents,
+    }
+
+
+def segment_call(src_path: Path) -> dict:
+    with open(src_path, encoding="utf-8") as f:
+        data = json.load(f)
+    return segment_transcript(data)
+
+
+def collect_sources(domains: set) -> list[tuple[str, Path]]:
+    """Return (domain_label, src_path) for every matching call."""
+    sources = []
+    for accent_dir in sorted(RESULTS_ROOT.iterdir()):
+        if not accent_dir.is_dir():
+            continue
+        for json_file in sorted(accent_dir.glob("*.json")):
+            stem = json_file.stem  # e.g. en_CA_Banking_1586889
+            # domain is the part between accent and numeric id
+            parts = stem.split("_")
+            # find the domain token — the first purely alpha segment after the accent
+            # accent = en_CA or en_US_General (variable length)
+            # try matching known domains
+            stem_lower = stem.lower()
+            matched = None
+            for d in domains:
+                if f"_{d}_" in stem_lower or stem_lower.endswith(f"_{d}"):
+                    matched = d
+                    break
+            if matched:
+                sources.append((matched, json_file))
+    return sources
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--domains", nargs="+",
+                    default=sorted(DEFAULT_DOMAINS),
+                    help="domains to segment (any manifest domain; "
+                         "default: banking health telecom)")
+    args = ap.parse_args()
+    domains = set(args.domains)
+
+    sources = collect_sources(domains)
+    if not sources:
+        print("No matching calls found. Check RESULTS_ROOT path.")
+        return
+
+    counts = {d: 0 for d in domains}
+    total_sentences = 0
+
+    for domain, src in sources:
+        out_dir = OUTPUT_ROOT / domain
+        out_dir.mkdir(parents=True, exist_ok=True)
+        out_path = out_dir / src.name
+
+        result = segment_call(src)
+        with open(out_path, "w", encoding="utf-8") as f:
+            json.dump(result, f, indent=2)
+
+        counts[domain] += 1
+        total_sentences += result["total"]
+        print(f"  [{domain:8}]  {src.stem:<40}  "
+              f"{result['agent_sentences']:>3}A + {result['customer_sentences']:>3}C "
+              f"= {result['total']:>4} sentences  ->  {out_path.relative_to(OUTPUT_ROOT.parent.parent)}")
+
+    print()
+    print("-" * 72)
+    for d in sorted(counts):
+        print(f"  {d:10}  {counts[d]} calls")
+    print(f"  {'TOTAL':10}  {sum(counts.values())} calls  ·  {total_sentences} sentences")
+    print(f"\nOutput root: {OUTPUT_ROOT}")
+
+
+if __name__ == "__main__":
+    main()

ml-services/evaluation/benchmark.py

@@ -0,0 +1,280 @@
+"""
+Model benchmark for call QA extraction.
+
+Runs the same structured extraction task across candidate models on two
+contrasting calls, then auto-scores each output on four criteria:
+
+  1. Schema validity   -- does Pydantic accept it?
+  2. Evidence fidelity -- are quoted strings verbatim substrings of the transcript?
+  3. Score spread      -- does the model discriminate between the two calls?
+  4. Conservative bias -- does it avoid PASS without explicit evidence?
+
+Produces a ranked comparison table so you can make a principled model
+selection before wiring up routing.
+
+Usage:
+  python benchmark.py
+  python benchmark.py --calls en_CA_Banking_1592237 en_US_General_Health_1587175
+  python benchmark.py --providers github mistral cohere
+"""
+
+import os, json, time, argparse, re
+from typing import Optional
+from pydantic import ValidationError
+
+import paths
+from env_util import load_env
+from assemble import load_manifest, build_packet
+from rubric import CallEvaluation, RUBRIC_VERSION
+from llm_client import chat_json
+from extract import SYSTEM, SKELETON, strip_fences
+
+load_env()
+
+DATA = str(paths.NA_TESTSET)
+
+# ── Candidate models (provider, model_override or None for default) ───────────
+CANDIDATES = [
+    ("github",     None),                           # GPT-4o-mini
+    ("gemini",     "gemini-2.0-flash"),
+    ("mistral",    "mistral-small-latest"),
+    ("sambanova",  "Meta-Llama-3.3-70B-Instruct"),
+    ("sambanova",  "DeepSeek-V3.1"),
+    ("cohere",     "command-r7b-12-2024"),
+]
+
+DEFAULT_CALLS = [
+    "en_CA_Banking_1592237",       # clean, high-scoring banking call
+    "en_CA_Banking_1588683",       # weaker banking call
+]
+
+
+# ── Auto-scoring helpers ──────────────────────────────────────────────────────
+
+def check_schema(raw: str) -> tuple[bool, Optional[str]]:
+    """Try to parse + validate. Returns (valid, error_snippet)."""
+    try:
+        data = json.loads(raw)
+        data["rubric_version"] = RUBRIC_VERSION
+        data["metadata"] = {"call_id": "x", "domain": "x"}
+        CallEvaluation.model_validate(data)
+        return True, None
+    except (json.JSONDecodeError, ValidationError, Exception) as e:
+        return False, str(e)[:120]
+
+
+def check_evidence_fidelity(raw: str, packet: str) -> tuple[int, int]:
+    """Count evidence quotes that are verbatim substrings of the transcript.
+    Returns (n_verbatim, n_total)."""
+    try:
+        data = json.loads(raw)
+    except Exception:
+        return 0, 0
+
+    # collect all quote fields recursively
+    quotes = []
+    def harvest(obj):
+        if isinstance(obj, dict):
+            if "quote" in obj and isinstance(obj["quote"], str):
+                quotes.append(obj["quote"])
+            for v in obj.values():
+                harvest(v)
+        elif isinstance(obj, list):
+            for item in obj:
+                harvest(item)
+    harvest(data)
+
+    if not quotes:
+        return 0, 0
+
+    # normalise packet to lowercase for matching (avoid case mismatches)
+    packet_lower = packet.lower()
+    n_verbatim = sum(1 for q in quotes if q.lower().strip() in packet_lower)
+    return n_verbatim, len(quotes)
+
+
+def check_conservative_bias(raw: str) -> tuple[int, int]:
+    """Count compliance items marked PASS and how many have evidence attached.
+    Returns (n_pass_with_evidence, n_pass_total)."""
+    try:
+        data = json.loads(raw)
+    except Exception:
+        return 0, 0
+
+    compliance = data.get("compliance", {})
+    n_pass = n_with_ev = 0
+    for key, item in compliance.items():
+        if not isinstance(item, dict):
+            continue
+        if item.get("passed") is True:
+            n_pass += 1
+            if item.get("evidence") and item["evidence"].get("quote"):
+                n_with_ev += 1
+    return n_with_ev, n_pass
+
+
+def quality_scores(raw: str) -> list[int]:
+    """Extract the 5 quality scores as a list."""
+    try:
+        data = json.loads(raw)
+        q = data.get("quality", {})
+        return [q.get(k, {}).get("score", 0)
+                for k in ("efficiency", "problem_resolution", "clarity",
+                           "professionalism", "empathy")]
+    except Exception:
+        return []
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def run_benchmark(calls: list[str], providers: Optional[list[str]]):
+    manifest = load_manifest()
+
+    candidates = CANDIDATES
+    if providers:
+        candidates = [(p, m) for p, m in CANDIDATES if p in providers]
+
+    # pre-build packets
+    packets = {}
+    for cid in calls:
+        meta = manifest[cid]
+        result_path = os.path.join(
+            DATA, "results_channels", meta["accent"], cid + ".json")
+        with open(result_path, encoding="utf-8") as f:
+            result = json.load(f)
+        packet, _ = build_packet(result, meta)
+        packets[cid] = packet
+
+    # results[provider_model][call_id] = {raw, scores_dict, timing}
+    rows = []
+
+    print(f"\nBenchmarking {len(candidates)} models × {len(calls)} calls\n")
+
+    for provider, model_override in candidates:
+        label = f"{provider}/{model_override or 'default'}"
+        row = {"label": label, "provider": provider,
+               "model": model_override, "calls": {}}
+
+        for cid in calls:
+            packet = packets[cid]
+            user = f"{packet}\n\n{SKELETON}"
+            t0 = time.time()
+            try:
+                raw = chat_json(provider, SYSTEM, user,
+                                model=model_override, max_tokens=4000)
+                elapsed = time.time() - t0
+
+                valid, err = check_schema(raw)
+                n_verb, n_total = check_evidence_fidelity(raw, packet)
+                n_pass_ev, n_pass = check_conservative_bias(raw)
+                scores = quality_scores(raw)
+
+                row["calls"][cid] = {
+                    "ok":       valid,
+                    "elapsed":  elapsed,
+                    "schema":   valid,
+                    "fidelity": (n_verb, n_total),
+                    "conserv":  (n_pass_ev, n_pass),
+                    "scores":   scores,
+                    "err":      err,
+                    "raw":      raw,
+                }
+                status = "OK " if valid else "ERR"
+                fid = f"{n_verb}/{n_total}" if n_total else "--"
+                con = f"{n_pass_ev}/{n_pass}" if n_pass else "--"
+                sc  = ",".join(str(s) for s in scores) if scores else "--"
+                print(f"  [{status}] {label:<38} {cid[-10:]:<14} "
+                      f"{elapsed:5.1f}s  fidelity={fid}  conserv={con}  "
+                      f"scores=[{sc}]")
+
+            except Exception as e:
+                elapsed = time.time() - t0
+                row["calls"][cid] = {
+                    "ok": False, "elapsed": elapsed,
+                    "err": str(e)[:120], "raw": ""}
+                print(f"  [FAIL] {label:<38} {cid[-10:]:<14} "
+                      f"{elapsed:5.1f}s  {str(e)[:80]}")
+
+        rows.append(row)
+
+    # ── Summary table ─────────────────────────────────────────────────────────
+    print(f"\n{'='*90}")
+    print(f"  BENCHMARK SUMMARY  ({len(calls)} calls)")
+    print(f"{'='*90}")
+    print(f"  {'Model':<38} {'Valid':>5} {'Fidelity':>9} {'Conserv':>8} "
+          f"{'Spread':>7} {'Avg ms':>8}")
+    print(f"  {'-'*85}")
+
+    scored = []
+    for row in rows:
+        valid_calls = [v for v in row["calls"].values() if v.get("ok")]
+        n_valid = len(valid_calls)
+
+        # fidelity: avg across calls
+        fid_nums = [(v["fidelity"][0], v["fidelity"][1])
+                    for v in valid_calls if v.get("fidelity", (0,0))[1] > 0]
+        fid_str = (f"{sum(x[0] for x in fid_nums)}/{sum(x[1] for x in fid_nums)}"
+                   if fid_nums else "--")
+
+        # conservatism: evidence on PASS items
+        con_nums = [(v["conserv"][0], v["conserv"][1])
+                    for v in valid_calls if v.get("conserv", (0,0))[1] > 0]
+        con_str = (f"{sum(x[0] for x in con_nums)}/{sum(x[1] for x in con_nums)}"
+                   if con_nums else "--")
+
+        # score spread: std-dev of scores across calls (higher = more discriminating)
+        all_scores = [s for v in valid_calls for s in v.get("scores", [])]
+        if len(all_scores) >= 2:
+            mean = sum(all_scores) / len(all_scores)
+            spread = (sum((s - mean)**2 for s in all_scores) / len(all_scores)) ** 0.5
+            spread_str = f"{spread:.2f}"
+        else:
+            spread = 0.0
+            spread_str = "--"
+
+        avg_ms = (sum(v["elapsed"] for v in row["calls"].values()) /
+                  len(row["calls"]) * 1000) if row["calls"] else 0
+
+        print(f"  {row['label']:<38} {n_valid}/{len(calls):>3}  "
+              f"{fid_str:>9}  {con_str:>8}  {spread_str:>7}  {avg_ms:>7.0f}ms")
+
+        scored.append((row["label"], n_valid, fid_nums, con_nums, spread, avg_ms))
+
+    # ── Recommendation ────────────────────────────────────────────────────────
+    print(f"\n  COLUMN GUIDE:")
+    print(f"  Valid    = schema-valid outputs / total calls")
+    print(f"  Fidelity = verbatim quotes found in transcript / total quotes cited")
+    print(f"  Conserv  = PASS items with supporting evidence / total PASS items")
+    print(f"  Spread   = score std-dev across calls (higher = more discriminating)")
+    print(f"  Avg ms   = average latency per call")
+    print(f"\n  Best fidelity  = quotes least likely to be hallucinated")
+    print(f"  Best conserv   = least likely to grant unearned compliance passes")
+    print(f"  Best spread    = actually differentiates call quality")
+    print(f"{'='*90}")
+
+    # save raw results
+    out = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                       "benchmark_results.json")
+    with open(out, "w", encoding="utf-8") as f:
+        # strip raw transcripts from saved output to keep file small
+        clean = []
+        for row in rows:
+            r = dict(row)
+            r["calls"] = {cid: {k: v for k, v in cv.items() if k != "raw"}
+                          for cid, cv in row["calls"].items()}
+            clean.append(r)
+        json.dump(clean, f, indent=2)
+    print(f"\n  Full results saved -> {out}")
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--calls", nargs="+", default=DEFAULT_CALLS)
+    ap.add_argument("--providers", nargs="+", default=None,
+                    help="filter to these providers only")
+    args = ap.parse_args()
+    run_benchmark(args.calls, args.providers)
+
+
+if __name__ == "__main__":
+    main()

ml-services/evaluation/consistency.py

@@ -0,0 +1,171 @@
+"""
+Consistency harness for the graph evaluation pipeline.
+
+A scoring system that produces different results on the same input across
+runs is not dependable. This harness runs the SAME call through the graph N
+times and measures agreement:
+
+  - compliance : per-check agreement rate (fraction of runs matching the
+                 modal pass/fail/null verdict)
+  - quality    : per-dimension score min / max / range / mean / stdev
+  - escalation : risk_level + emotion agreement rate
+  - anchoring  : evidence anchor rate per run
+
+Usage:
+  python consistency.py --call_id en_CA_Banking_1586889 --runs 5
+"""
+import os
+import json
+import time
+import argparse
+import statistics
+from collections import Counter
+
+from graph import build_graph
+
+COMPLIANCE_KEYS = [
+    "name_announced", "company_announced", "recording_disclosure",
+    "identity_verified", "resolution_provided", "transfer_next_steps",
+]
+QUALITY_KEYS = [
+    "efficiency", "problem_resolution", "clarity",
+    "professionalism", "empathy", "customer_satisfaction",
+]
+
+
+def agreement(values):
+    """Fraction of values matching the mode. 1.0 = perfect agreement."""
+    if not values:
+        return None
+    most_common = Counter(values).most_common(1)[0][1]
+    return most_common / len(values)
+
+
+def run_sweep(call_id, runs, results_dir):
+    graph = build_graph()
+    evaluations = []
+
+    for i in range(1, runs + 1):
+        print(f"\n--- run {i}/{runs} " + "-" * 50)
+        t0 = time.time()
+        final = graph.invoke({"call_id": call_id, "results_dir": results_dir})
+        dt = time.time() - t0
+        ev = final["evaluation"]
+        evaluations.append(ev)
+        stats = ev.get("_anchor_stats", {})
+        print(f"--- run {i} done in {dt:.1f}s "
+              f"(anchored {stats.get('anchored')}/{stats.get('total')})")
+
+    return evaluations
+
+
+def analyze(evaluations):
+    report = {"n_runs": len(evaluations), "compliance": {}, "quality": {},
+              "escalation": {}, "anchoring": {}}
+
+    # compliance agreement per check
+    for key in COMPLIANCE_KEYS:
+        verdicts = [str(ev["compliance"][key]["passed"]) for ev in evaluations]
+        report["compliance"][key] = {
+            "verdicts": verdicts,
+            "agreement": agreement(verdicts),
+        }
+
+    # quality score variance per dimension
+    for key in QUALITY_KEYS:
+        scores = [ev["quality"][key]["score"] for ev in evaluations
+                  if ev["quality"].get(key)]
+        if not scores:
+            continue
+        report["quality"][key] = {
+            "scores": scores,
+            "min": min(scores),
+            "max": max(scores),
+            "range": max(scores) - min(scores),
+            "mean": round(statistics.mean(scores), 2),
+            "stdev": round(statistics.stdev(scores), 3) if len(scores) > 1 else 0.0,
+        }
+
+    # escalation agreement
+    risks = [ev["escalation"]["risk_level"] for ev in evaluations]
+    emotions = [ev["escalation"]["customer_emotion_text"] for ev in evaluations]
+    report["escalation"] = {
+        "risk_levels": risks,
+        "risk_agreement": agreement(risks),
+        "emotions": emotions,
+        "emotion_agreement": agreement(emotions),
+    }
+
+    # anchor rates
+    rates = []
+    for ev in evaluations:
+        s = ev.get("_anchor_stats", {})
+        if s.get("total"):
+            rates.append(round(s["anchored"] / s["total"], 3))
+    report["anchoring"] = {"rates": rates}
+
+    return report
+
+
+def print_report(report, call_id):
+    n = report["n_runs"]
+    print("\n" + "=" * 68)
+    print(f"  CONSISTENCY REPORT  {call_id}  ({n} runs)")
+    print("=" * 68)
+
+    print("\n  COMPLIANCE (agreement with modal verdict)")
+    worst_c = 1.0
+    for key, r in report["compliance"].items():
+        a = r["agreement"]
+        worst_c = min(worst_c, a)
+        flag = "" if a == 1.0 else f"  <-- UNSTABLE {r['verdicts']}"
+        print(f"    {a * 100:5.1f}%  {key}{flag}")
+
+    print("\n  QUALITY (score spread across runs)")
+    worst_range = 0
+    for key, r in report["quality"].items():
+        worst_range = max(worst_range, r["range"])
+        flag = "" if r["range"] <= 1 else "  <-- UNSTABLE"
+        print(f"    {key:<24} scores={r['scores']}  "
+              f"range={r['range']}  stdev={r['stdev']}{flag}")
+
+    e = report["escalation"]
+    print(f"\n  ESCALATION  risk agreement: {e['risk_agreement'] * 100:.1f}% "
+          f"{e['risk_levels']}")
+    print(f"              emotion agreement: {e['emotion_agreement'] * 100:.1f}% "
+          f"{e['emotions']}")
+
+    rates = report["anchoring"]["rates"]
+    if rates:
+        print(f"\n  ANCHOR RATES per run: {rates}")
+
+    print("\n  VERDICT: ", end="")
+    if worst_c == 1.0 and worst_range <= 1 and e["risk_agreement"] == 1.0:
+        print("stable (all compliance unanimous, quality range <= 1, "
+              "risk unanimous)")
+    else:
+        print("instability detected -- see UNSTABLE flags above")
+    print("=" * 68)
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Run the graph N times on one call, measure agreement")
+    ap.add_argument("--call_id", required=True)
+    ap.add_argument("--runs", type=int, default=5)
+    ap.add_argument("--results_dir", default="results_channels")
+    args = ap.parse_args()
+
+    evaluations = run_sweep(args.call_id, args.runs, args.results_dir)
+    report = analyze(evaluations)
+    print_report(report, args.call_id)
+
+    here = os.path.dirname(os.path.abspath(__file__))
+    out = os.path.join(here, "results", f"{args.call_id}_consistency.json")
+    with open(out, "w", encoding="utf-8") as f:
+        json.dump(report, f, indent=2)
+    print(f"\nSaved -> {out}")
+
+
+if __name__ == "__main__":
+    main()

ml-services/evaluation/env_util.py

@@ -0,0 +1,7 @@
+"""Minimal .env loader (no external dependency).
+
+The actual loading lives in paths.py (which also runs it at import time so
+CAPSTONE_* path overrides in .env take effect before any path constant is
+read). This re-export keeps the existing call sites working unchanged.
+"""
+from paths import load_env  # noqa: F401

ml-services/evaluation/eval_call_data.py

@@ -0,0 +1,140 @@
+"""
+Run the BASELINE evaluator (extract.py's rubric + router) on the frontend's
+call_data.json, then ANCHOR every evidence quote to a real timestamp by locating
+it in the transcript turns (don't trust the LLM's timestamp -- same robustness
+principle as chaptering).
+
+Merges an "evaluation" block into frontend/src/call_data.json so the React
+compliance panel can light up each check at the moment its evidence occurs.
+
+Usage:  python eval_call_data.py
+"""
+import os, json, time
+import paths
+from env_util import load_env
+from extract import SYSTEM, SKELETON, strip_fences
+from router import chat_json_routed
+from rubric import CallEvaluation, RUBRIC_VERSION
+
+load_env()
+CALL_DATA = str(paths.FRONTEND_ROOT / "src" / "call_data.json")
+
+
+def mmss(s):
+    return f"{int(s // 60):02d}:{int(s % 60):02d}"
+
+
+def build_packet(turns, duration, domain="banking"):
+    header = (f"CALL METADATA: domain={domain} | duration={mmss(duration)} "
+              f"| turns={len(turns)}\n\nROLE-MAPPED TRANSCRIPT:\n")
+    body = "\n".join(f"[{t['speaker']} {mmss(t['start'])}] {t['text']}" for t in turns)
+    return header + body
+
+
+def _norm(s):
+    return " ".join(s.lower().split())
+
+
+def anchor(quote, turns):
+    """Locate a quote in the transcript -> (start_sec, turn_index). None if unfound."""
+    if not quote:
+        return None, None
+    q = _norm(quote)
+    for t in turns:
+        if q and q in _norm(t["text"]):
+            return t["start"], t["i"]
+    # fuzzy fallback: first 6 words of the quote
+    frag = " ".join(q.split()[:6])
+    if frag:
+        for t in turns:
+            if frag in _norm(t["text"]):
+                return t["start"], t["i"]
+    return None, None
+
+
+def anchor_evidence(obj, turns, stats):
+    """Recursively attach 'sec' + 'turn' to every evidence object with a quote."""
+    if isinstance(obj, dict):
+        if isinstance(obj.get("quote"), str) and obj["quote"]:
+            sec, ti = anchor(obj["quote"], turns)
+            obj["sec"] = sec
+            obj["turn"] = ti
+            stats["total"] += 1
+            if sec is not None:
+                stats["anchored"] += 1
+        for v in obj.values():
+            anchor_evidence(v, turns, stats)
+    elif isinstance(obj, list):
+        for item in obj:
+            anchor_evidence(item, turns, stats)
+
+
+def main():
+    with open(CALL_DATA, encoding="utf-8") as f:
+        cd = json.load(f)
+    turns = cd["turns"]
+    duration = cd["duration"]
+
+    packet = build_packet(turns, duration)
+    user = f"{packet}\n\n{SKELETON}"
+
+    t0 = time.time()
+    raw, served = chat_json_routed(SYSTEM, user, max_tokens=4000, return_meta=True)
+    dt = time.time() - t0
+
+    data = json.loads(strip_fences(raw))
+    data["rubric_version"] = RUBRIC_VERSION
+    data["metadata"] = {"call_id": cd.get("call", "call_1"), "domain": "banking",
+                        "duration_seconds": duration,
+                        "transcript_model": cd.get("model")}
+    ev = CallEvaluation.model_validate(data)
+    ev_dict = ev.model_dump()
+
+    stats = {"total": 0, "anchored": 0}
+    anchor_evidence(ev_dict, turns, stats)
+    ev_dict["served_by"] = served
+
+    cd["evaluation"] = ev_dict
+    with open(CALL_DATA, "w", encoding="utf-8") as f:
+        json.dump(cd, f, indent=2)
+
+    # ── report ────────────────────────────────────────────────────────────────
+    print(f"\nevaluated by {served} in {dt:.1f}s  |  "
+          f"evidence anchored {stats['anchored']}/{stats['total']}")
+    print("=" * 64)
+    print("  COMPLIANCE")
+    c = ev_dict["compliance"]
+    for key, name in [("name_announced", "Name announced"),
+                      ("company_announced", "Company announced"),
+                      ("recording_disclosure", "Recording disclosure"),
+                      ("identity_verified", "Identity verified"),
+                      ("resolution_provided", "Resolution provided"),
+                      ("transfer_next_steps", "Transfer next-steps")]:
+        item = c[key]
+        mark = {True: "PASS", False: "FAIL", None: "N/A "}[item["passed"]]
+        evd = item.get("evidence")
+        when = f" @{mmss(evd['sec'])}" if evd and evd.get("sec") is not None else ""
+        quote = f'  "{evd["quote"][:48]}"' if evd else (f"  ({item.get('note')})" if item.get("note") else "")
+        print(f"    [{mark}]{when:>7} {name:<22}{quote}")
+    if c.get("identity_method"):
+        print(f"              method: {c['identity_method']}")
+
+    print("\n  QUALITY")
+    q = ev_dict["quality"]
+    for key, name in [("efficiency", "Efficiency"), ("problem_resolution", "Problem resolution"),
+                      ("clarity", "Clarity"), ("professionalism", "Professionalism"),
+                      ("empathy", "Empathy")]:
+        d = q[key]
+        aud = " (+audio)" if d["requires_audio"] else ""
+        print(f"    {d['score']}/5  {name:<20}{aud}")
+
+    e = ev_dict["escalation"]
+    print(f"\n  ESCALATION: {e['risk_level'].upper()} | emotion: {e['customer_emotion_text']}"
+          f" | flags: {', '.join(e['red_flags']) if e['red_flags'] else 'none'}")
+    print(f"\n  SUMMARY: {ev_dict.get('overall_summary')}")
+    print("=" * 64)
+    print(f"\nMerged 'evaluation' into {CALL_DATA}")
+
+
+if __name__ == "__main__":
+    main()

ml-services/evaluation/export_frontend.py

@@ -0,0 +1,215 @@
+"""
+Export every evaluated call into the frontend's per-call format + a merged
+playable audio file, plus a summary index for the dashboard.
+
+For each call_id with a results/{call_id}_graph.json:
+  - turns     rebuilt from the per-channel word-level transcript, using the
+              same word-grouping logic as gen_call_data.py::build_turns
+              (assemble.py's assemble_turns() drops the per-turn `end` field
+              the frontend needs, so it isn't reused here)
+  - chapters  from segment.segment() (one LLM call per call -- chapters were
+              never persisted anywhere), converted from its start_turn/mm:ss
+              shape into the numeric start/end-seconds shape the frontend
+              player expects
+  - evaluation  the existing results/{call_id}_graph.json content, verbatim
+  - audio     agent+customer wav merged into one 2-channel mp3 via ffmpeg
+              (ch0=agent, ch1=customer -- same convention as the original
+              call_1 demo audio)
+
+Writes (served statically, fetched at runtime -- not bundled into the JS build):
+  frontend/public/calls/{call_id}.json   {call, model, duration, turns, chapters, evaluation}
+  frontend/public/calls_index.json       [{call_id, domain, ...summary}, ...]
+  frontend/public/audio/{call_id}.mp3
+
+The demo call (en_CA_Banking_1586889) is copied verbatim from the existing
+call_data.json instead of being regenerated -- it already carries the
+recheck-node bugfix's hand-verified evidence patch.
+
+Usage:  python export_frontend.py
+"""
+import os
+import json
+import shutil
+import subprocess
+
+import paths
+from assemble import load_manifest, DATA
+from segment import segment
+
+EVAL_DIR = os.path.dirname(os.path.abspath(__file__))
+RESULTS_DIR_NAME = "results"
+
+PUBLIC = str(paths.FRONTEND_PUBLIC)
+CALLS_OUT = os.path.join(PUBLIC, "calls")
+AUDIO_OUT = os.path.join(PUBLIC, "audio")
+INDEX_OUT = os.path.join(PUBLIC, "calls_index.json")
+
+DEMO_CALL = "en_CA_Banking_1586889"
+DEMO_CALL_DATA = str(paths.FRONTEND_ROOT / "src" / "call_data.json")
+DEMO_AUDIO_SRC = os.path.join(PUBLIC, "call_1.mp3")
+
+
+def build_turns(result):
+    """Merge agent+customer words into time-ordered speaker turns."""
+    merged = []
+    for speaker, key in (("AGENT", "agent"), ("CUSTOMER", "customer")):
+        for w in result.get(key, []):
+            if w.get("start") is None:
+                continue
+            merged.append((w["start"], w.get("end", w["start"]), speaker, w["word"]))
+    merged.sort(key=lambda x: x[0])
+
+    turns = []
+    cur = None
+    for start, end, sp, word in merged:
+        if cur is None or sp != cur["speaker"]:
+            if cur:
+                turns.append(cur)
+            cur = {"speaker": sp, "text": word, "start": start, "end": end}
+        else:
+            cur["text"] += " " + word
+            cur["end"] = end
+    if cur:
+        turns.append(cur)
+
+    for i, t in enumerate(turns):
+        t["i"] = i
+        t["start"] = round(t["start"], 2)
+        t["end"] = round(t["end"], 2)
+    return turns
+
+
+def numeric_chapters(cc_chapters, turns, duration):
+    """Convert segment.py's start_turn/mm:ss chapters into numeric seconds."""
+    chapters = []
+    for i, ch in enumerate(cc_chapters):
+        start = turns[ch.start_turn]["start"]
+        end = (turns[cc_chapters[i + 1].start_turn]["start"]
+               if i + 1 < len(cc_chapters) else duration)
+        chapters.append({"index": ch.index, "label": ch.label,
+                          "summary": ch.summary, "start": round(start, 2),
+                          "end": round(end, 2), "start_turn": ch.start_turn})
+    return chapters
+
+
+def merge_audio(agent_wav, customer_wav, out_mp3):
+    os.makedirs(os.path.dirname(out_mp3), exist_ok=True)
+    proc = subprocess.run([
+        "ffmpeg", "-y", "-i", agent_wav, "-i", customer_wav,
+        "-filter_complex", "[0:a][1:a]amerge=inputs=2[a]",
+        "-map", "[a]", "-ac", "2", "-b:a", "128k", out_mp3,
+    ], capture_output=True, text=True)
+    if proc.returncode != 0:
+        print(f"    ffmpeg FAILED for {out_mp3}:\n{proc.stderr[-800:]}")
+        return False
+    return True
+
+
+def export_call_artifacts(call_id, meta, out_calls_dir, out_audio_dir):
+    """Build one call's frontend artifacts into the given output dirs.
+
+    Writes {call_id}.json (turns + chapters + evaluation) and the merged
+    2-channel mp3. Reads the graph evaluation and the per-channel transcript
+    already on disk; runs one chapter LLM call. Channel WAVs are expected at
+    the dataset convention DATA/{accent}/{call_id}_{agent,customer}.wav (the
+    orchestrator places fresh-call channels there before calling this).
+    Returns the call dict, which summarize() turns into a dashboard index row.
+    """
+    eval_path = os.path.join(EVAL_DIR, "results", f"{call_id}_graph.json")
+    with open(eval_path, encoding="utf-8") as f:
+        evaluation = json.load(f)
+    duration = evaluation["metadata"]["duration_seconds"]
+
+    result_path = os.path.join(DATA, RESULTS_DIR_NAME, meta["accent"], call_id + ".json")
+    with open(result_path, encoding="utf-8") as f:
+        result = json.load(f)
+    turns = build_turns(result)
+
+    cc, dt, issues = segment(call_id, provider="router", results_dir=RESULTS_DIR_NAME)
+    if issues:
+        print(f"    chapter issues: {issues}")
+    chapters = numeric_chapters(cc.chapters, turns, duration)
+
+    out = {
+        "call": call_id,
+        "model": result.get("model", ""),
+        "duration": round(duration, 2),
+        "turns": turns,
+        "chapters": chapters,
+        "evaluation": evaluation,
+    }
+    os.makedirs(out_calls_dir, exist_ok=True)
+    with open(os.path.join(out_calls_dir, f"{call_id}.json"), "w", encoding="utf-8") as f:
+        json.dump(out, f, indent=2)
+
+    ok = merge_audio(
+        os.path.join(DATA, meta["accent"], f"{call_id}_agent.wav"),
+        os.path.join(DATA, meta["accent"], f"{call_id}_customer.wav"),
+        os.path.join(out_audio_dir, f"{call_id}.mp3"),
+    )
+    print(f"  [{call_id}] {len(turns)} turns, {len(chapters)} chapters, "
+          f"audio {'ok' if ok else 'FAILED'}")
+    return out
+
+
+def export_call(call_id, manifest):
+    return export_call_artifacts(call_id, manifest[call_id], CALLS_OUT, AUDIO_OUT)
+
+
+def summarize(call_id, out):
+    ev = out["evaluation"]
+    c = ev.get("compliance", {})
+    passed = sum(1 for v in c.values() if isinstance(v, dict) and v.get("passed") is True)
+    failed = sum(1 for v in c.values() if isinstance(v, dict) and v.get("passed") is False)
+    wf = ev.get("workflow") or {}
+    steps = wf.get("expected_steps", [])
+    esc = ev.get("escalation", {})
+    meta = ev.get("metadata", {})
+    return {
+        "call_id": call_id,
+        "domain": meta.get("domain"),
+        "accent": meta.get("accent"),
+        "duration": out["duration"],
+        "compliance_passed": passed,
+        "compliance_applicable": passed + failed,
+        "workflow_met": sum(1 for s in steps if s.get("met") is True),
+        "workflow_total": len(steps),
+        "risk_level": esc.get("risk_level"),
+        "subject": wf.get("subject"),
+    }
+
+
+def main():
+    manifest = load_manifest()
+    call_ids = sorted(
+        f[:-len("_graph.json")]
+        for f in os.listdir(os.path.join(EVAL_DIR, "results"))
+        if f.endswith("_graph.json")
+    )
+
+    index = []
+
+    with open(DEMO_CALL_DATA, encoding="utf-8") as f:
+        demo = json.load(f)
+    os.makedirs(CALLS_OUT, exist_ok=True)
+    with open(os.path.join(CALLS_OUT, f"{DEMO_CALL}.json"), "w", encoding="utf-8") as f:
+        json.dump(demo, f, indent=2)
+    os.makedirs(AUDIO_OUT, exist_ok=True)
+    shutil.copy(DEMO_AUDIO_SRC, os.path.join(AUDIO_OUT, f"{DEMO_CALL}.mp3"))
+    index.append(summarize(DEMO_CALL, demo))
+    print(f"[{DEMO_CALL}] copied verbatim (demo call, already patched)")
+
+    for call_id in call_ids:
+        if call_id == DEMO_CALL:
+            continue
+        out = export_call(call_id, manifest)
+        index.append(summarize(call_id, out))
+
+    with open(INDEX_OUT, "w", encoding="utf-8") as f:
+        json.dump(index, f, indent=2)
+    print(f"\nExported {len(index)} calls -> {CALLS_OUT}")
+    print(f"Index -> {INDEX_OUT}")
+
+
+if __name__ == "__main__":
+    main()

ml-services/evaluation/extract.py

@@ -0,0 +1,193 @@
+"""
+Phase 1 -- Monolithic Structured Extraction.
+
+One LLM call turns the role-mapped transcript packet into a validated
+CallEvaluation JSON (compliance booleans + quality scores + escalation), every
+judgment backed by a verbatim quote. Pydantic validates; one retry on failure.
+
+MODALITY DISCIPLINE is enforced in the prompt: the text model judges only what
+text can prove, and flags acoustic items requires_audio rather than guessing.
+
+Usage:
+  python extract.py --call_id en_CA_Banking_1592237 [--provider github] [--model openai/gpt-4o]
+"""
+import os, json, argparse, time
+from pydantic import ValidationError
+
+import paths
+from env_util import load_env
+from assemble import load_manifest, build_packet
+from rubric import CallEvaluation, CallMetadata, RUBRIC_VERSION
+from llm_client import chat_json
+
+load_env()
+DATA = str(paths.NA_TESTSET)
+
+SYSTEM = f"""You are a meticulous call-center QA analyst. You evaluate a single customer-service call against a fixed rubric and output ONLY a JSON object. Rubric version {RUBRIC_VERSION}.
+
+You are given a ROLE-MAPPED transcript: every line is tagged [AGENT mm:ss] or [CUSTOMER mm:ss]. The channel identity is authoritative -- never attribute the agent's words to the customer or vice versa.
+
+EVALUATE THESE ITEMS:
+
+COMPLIANCE (passed = true / false / null). null ONLY when the item does not apply to this call.
+- name_announced: agent stated their own name.
+- company_announced: agent stated the company/bank name.
+- recording_disclosure: agent said the call may be recorded/monitored. If never stated, passed=false (do NOT assume). Use null only if clearly not required.
+- identity_verified: agent verified the customer BEFORE discussing the account (DOB, last 4 digits, account number, customer ID). Set identity_method to the method used.
+- resolution_provided: agent gave a resolution or clear next steps before closing.
+- transfer_next_steps: if the call was transferred, agent explained next steps. passed=null if there was no transfer.
+
+QUALITY (score 1-5 from TEXT signals; list signals_present / signals_absent):
+- efficiency: first-call resolution, no unnecessary holds/transfers.
+- problem_resolution: offered options, explained pros/cons and WHY, respected customer choice, solved the actual issue.
+- clarity: open-ended questions, confirms understanding ("does that make sense?"), recap at end.
+- professionalism: grammar, complete sentences, no slang. (Tone/pace are ACOUSTIC -> set requires_audio=true.)
+- empathy: validation phrases ("I understand your frustration"), uses customer name, acknowledges concern. (Warmth/flat tone are ACOUSTIC -> set requires_audio=true.)
+
+ESCALATION:
+- red_flags: any of manager_requested, competitor_switch, repeat_attempts, issue_too_complex, explicit_dissatisfaction.
+- customer_emotion_text: infer from WORDS only (calm/mild_frustration/frustrated/angry/distressed).
+- risk_level: none / review / escalate.
+
+HARD RULES:
+1. EVIDENCE: every compliance true/false and every quality score must cite a verbatim quote copied EXACTLY from the transcript, with speaker and timestamp. Never invent quotes.
+2. MODALITY: judge ONLY from text. For acoustic qualities (tone, pace, energy, whether the customer "sounds" satisfied) do not guess -- set requires_audio=true and score from text signals alone.
+3. CONSERVATISM: do not mark a compliance item true unless the transcript explicitly shows it.
+4. OUTPUT: a single JSON object, no markdown, no commentary."""
+
+SKELETON = """Return EXACTLY this JSON shape (fill values; keep keys):
+{
+  "compliance": {
+    "name_announced":      {"passed": true, "evidence": {"quote": "...", "speaker": "AGENT", "timestamp": "00:12"}, "note": null},
+    "company_announced":   {"passed": true, "evidence": {"quote": "...", "speaker": "AGENT", "timestamp": "00:12"}, "note": null},
+    "recording_disclosure":{"passed": false, "evidence": null, "note": "never stated"},
+    "identity_verified":   {"passed": true, "evidence": {"quote": "...", "speaker": "AGENT", "timestamp": "00:42"}, "note": null},
+    "identity_method": "date of birth",
+    "resolution_provided": {"passed": true, "evidence": {"quote": "...", "speaker": "AGENT", "timestamp": "10:07"}, "note": null},
+    "transfer_next_steps": {"passed": null, "evidence": null, "note": "no transfer"}
+  },
+  "quality": {
+    "efficiency":         {"score": 4, "signals_present": ["first-call resolution"], "signals_absent": [], "evidence": [{"quote":"...","speaker":"AGENT","timestamp":"08:38"}], "requires_audio": false},
+    "problem_resolution": {"score": 4, "signals_present": ["explained why"], "signals_absent": ["no explicit options A/B/C"], "evidence": [], "requires_audio": false},
+    "clarity":            {"score": 3, "signals_present": [], "signals_absent": ["no recap at end"], "evidence": [], "requires_audio": false},
+    "professionalism":    {"score": 4, "signals_present": ["complete sentences"], "signals_absent": [], "evidence": [], "requires_audio": true},
+    "empathy":            {"score": 2, "signals_present": [], "signals_absent": ["no validation phrase"], "evidence": [], "requires_audio": true}
+  },
+  "escalation": {
+    "red_flags": [],
+    "customer_emotion_text": "calm",
+    "risk_level": "none",
+    "evidence": [],
+    "requires_audio": true
+  },
+  "overall_summary": "2-3 sentence plain-language summary."
+}"""
+
+
+def strip_fences(s):
+    s = s.strip()
+    if s.startswith("```"):
+        s = s.split("```", 2)[1] if "```" in s[3:] else s[3:]
+        if s.startswith("json"):
+            s = s[4:]
+        s = s.rsplit("```", 1)[0]
+    return s.strip()
+
+
+def extract(call_id, provider="github", model=None, results_dir="results_channels"):
+    manifest = load_manifest()
+    meta = manifest[call_id]
+    result_path = os.path.join(DATA, results_dir, meta["accent"], call_id + ".json")
+    with open(result_path, encoding="utf-8") as f:
+        result = json.load(f)
+
+    packet, stats = build_packet(result, meta)
+
+    authoritative_meta = CallMetadata(
+        call_id=call_id, domain=result.get("domain", meta.get("domain", "unknown")),
+        accent=meta.get("accent"), duration_seconds=stats["duration_s"],
+        transcript_model=result.get("model"))
+
+    user = f"{packet}\n\n{SKELETON}"
+    system = SYSTEM
+    last_err = None
+    served = provider
+
+    for attempt in (1, 2):
+        t0 = time.time()
+        if provider == "router":
+            from router import chat_json_routed
+            raw, served = chat_json_routed(system, user, return_meta=True)
+        else:
+            raw = chat_json(provider, system, user, model=model)
+            served = model or provider
+        dt = time.time() - t0
+        try:
+            data = json.loads(strip_fences(raw))
+            data["rubric_version"] = RUBRIC_VERSION
+            data["metadata"] = authoritative_meta.model_dump()
+            evaluation = CallEvaluation.model_validate(data)
+            return evaluation, dt, served
+        except (json.JSONDecodeError, ValidationError) as e:
+            last_err = e
+            print(f"  [attempt {attempt}] validation failed ({type(e).__name__}); retrying...")
+            user = (f"{packet}\n\n{SKELETON}\n\nYour previous output was invalid: "
+                    f"{str(e)[:400]}. Return corrected JSON only.")
+    raise RuntimeError(f"Extraction failed after 2 attempts: {last_err}")
+
+
+def print_report(ev: CallEvaluation, dt, served=None):
+    print("\n" + "=" * 68)
+    tag = f"  [{dt:.1f}s | {served}]" if served else f"  [{dt:.1f}s]"
+    print(f"  EVALUATION  {ev.metadata.call_id}  ({ev.metadata.domain}){tag}")
+    print("=" * 68)
+    print("\n  COMPLIANCE")
+    c = ev.compliance
+    for name, item in [("Name announced", c.name_announced), ("Company announced", c.company_announced),
+                       ("Recording disclosure", c.recording_disclosure), ("Identity verified", c.identity_verified),
+                       ("Resolution provided", c.resolution_provided), ("Transfer next-steps", c.transfer_next_steps)]:
+        mark = {True: "PASS", False: "FAIL", None: "N/A "}[item.passed]
+        ev_str = f'  "{item.evidence.quote[:60]}"' if item.evidence else (f"  ({item.note})" if item.note else "")
+        print(f"    [{mark}] {name:<22}{ev_str}")
+    if c.identity_method:
+        print(f"           method: {c.identity_method}")
+    print("\n  QUALITY (1-5, text-derived)")
+    for name, q in [("Efficiency", ev.quality.efficiency), ("Problem resolution", ev.quality.problem_resolution),
+                    ("Clarity", ev.quality.clarity), ("Professionalism", ev.quality.professionalism),
+                    ("Empathy", ev.quality.empathy)]:
+        aud = " (+audio needed)" if q.requires_audio else ""
+        print(f"    {q.score}/5  {name:<20}{aud}")
+        if q.signals_absent:
+            print(f"          missing: {', '.join(q.signals_absent[:3])}")
+    e = ev.escalation
+    print(f"\n  ESCALATION: {e.risk_level.upper()}  | emotion(text): {e.customer_emotion_text}"
+          f" | flags: {', '.join(e.red_flags) if e.red_flags else 'none'}")
+    print(f"\n  SUMMARY: {ev.overall_summary}")
+    print("=" * 68)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    from llm_client import list_providers
+    ap.add_argument("--call_id", required=True)
+    ap.add_argument("--provider", default="router",
+                    choices=["router"] + list_providers())
+    ap.add_argument("--model", default=None)
+    ap.add_argument("--results_dir", default="results_channels")
+    args = ap.parse_args()
+
+    ev, dt, served = extract(args.call_id, args.provider, args.model, args.results_dir)
+
+    here = os.path.dirname(os.path.abspath(__file__))
+    out_dir = os.path.join(here, "results")
+    os.makedirs(out_dir, exist_ok=True)
+    out = os.path.join(out_dir, f"{args.call_id}.json")
+    with open(out, "w", encoding="utf-8") as f:
+        json.dump(ev.model_dump(), f, indent=2)
+
+    print_report(ev, dt, served)
+    print(f"\nSaved -> {out}")
+
+
+if __name__ == "__main__":
+    main()

ml-services/evaluation/fusion.py

@@ -0,0 +1,256 @@
+"""
+Acoustic-text fusion (v0.4.0) -- combines the audio sentiment model's
+per-sentence output with the text LLM's rubric scores.
+
+Design: DETERMINISTIC POST-HOC FUSION, not prompt injection.
+The LLM judges what text can prove; the acoustic model contributes what text
+cannot see (tone, emotion, escalation intensity); a documented formula
+combines them. This keeps the LLM's variance out of the acoustic signal and
+makes every fused score reproducible and explainable.
+
+Math
+----
+Valence map: Positive=+1, Neutral/Mixed=0, Negative=-1.
+
+Quality dimension fusion (empathy, professionalism <- AGENT channel;
+customer_satisfaction <- CUSTOMER channel, final third weighted 2x):
+    acoustic_score a = 3 + 2 * weighted_mean(valence)        # maps [-1,1] -> [1,5]
+    w_a = W_ACOUSTIC_CAP * coverage                          # coverage = processed/total
+    fused = clamp(round((1 - w_a) * text + w_a * a), 1, 5)
+
+Text stays the primary signal (cap 0.4) because the rubric's behaviours are
+text-defined; acoustics modulate. Zero acoustic data -> w_a = 0 (text only).
+
+Escalation fusion (CUSTOMER channel escalation_score trajectory):
+    late_mean = mean(score over final third), peak = max(score)
+    acoustic tier (v0.4.1, fitted on the 22-call batch -- see threshold
+    comments): escalate if late_mean >= 0.50, review if late_mean >= 0.30,
+    none otherwise. peak is recorded as context but does NOT set a tier.
+    Merge (v0.4.1): tiers that agree merge deterministically ("agreement");
+    tiers that disagree are left to the arbitrate LLM node ("disputed" ->
+    resolved as "llm_arbitration"), because disagreement is where a fixed
+    formula has no information to prefer one channel over the other.
+
+Data layout: data/sentiment/{domain}/{call_id}.json          (model output)
+             data/sentiment/{domain}/{call_id}_segments.json (the EXACT
+             sentence segmentation the model ran against -- seq_ids are only
+             meaningful against their own segmentation)
+"""
+import os
+import json
+
+import paths
+
+SENTIMENT_ROOT = str(paths.SENTIMENT_ROOT)
+
+W_ACOUSTIC_CAP = 0.4          # max acoustic weight at 100% coverage
+LATE_FRACTION = 1 / 3         # "end of call" window for trajectory metrics
+LATE_WEIGHT = 2.0             # extra weight on final-third customer valence
+
+# Acoustic escalation tiers -- fitted on the 22-call batch (2026-07,
+# batch_summary.json, n=21 calls with paired acoustic data):
+#   late_mean: min 0.166, p25 0.219, median 0.227, p75 0.257, p90 0.281,
+#              max 0.406 (single outlier; next highest 0.281).
+#   -> review at 0.30 sits above the calm cluster's p90 with margin; only the
+#      outlier call crosses it. escalate at 0.50 is deliberately outside the
+#      observed range: nothing in this batch warrants an acoustic-only
+#      escalate, and the bar should stay high until data demands lowering it.
+#   peak: median 0.558, p90 0.696 -- even calm calls routinely spike past the
+#      old 0.60 trigger, so peak has no discriminating power as a threshold.
+#      It is kept as recorded context (arbitrator/investigator input) only.
+ESC_THRESHOLDS = {
+    "escalate": {"late_mean": 0.50},
+    "review":   {"late_mean": 0.30},
+}
+
+VALENCE = {"Positive": 1.0, "Negative": -1.0, "Neutral": 0.0, "Mixed": 0.0}
+RISK_ORDER = {"none": 0, "review": 1, "escalate": 2}
+
+# which quality dimension is informed by which channel
+FUSED_DIMS = {
+    "professionalism":       {"channel": "AGENT", "late_weighted": False},
+    "empathy":               {"channel": "AGENT", "late_weighted": False},
+    "customer_satisfaction": {"channel": "CUSTOMER", "late_weighted": True},
+}
+
+
+def load_acoustic(call_id, domain):
+    """
+    Join the sentiment model output with the segmentation it ran against.
+    Returns list of rows {seq_id, pos, speaker, valence, emotion,
+    escalation_score, ok} or None if no acoustic data exists for this call.
+    """
+    d = os.path.join(SENTIMENT_ROOT, domain.lower())
+    sent_path = os.path.join(d, f"{call_id}.json")
+    seg_path = os.path.join(d, f"{call_id}_segments.json")
+    if not (os.path.exists(sent_path) and os.path.exists(seg_path)):
+        return None
+
+    with open(sent_path, encoding="utf-8") as f:
+        sentiment = json.load(f)
+    with open(seg_path, encoding="utf-8") as f:
+        segments = json.load(f)
+
+    speakers = {s["seq_id"]: s["speaker"] for s in segments["sentences"]}
+    n = max(speakers) or 1
+
+    rows = []
+    for s in sentiment["segments"]:
+        sid = s["seq_id"]
+        ok = s.get("processing_status") == "success" and s.get("sentiment")
+        rows.append({
+            "seq_id": sid,
+            "pos": sid / n,                       # 0..1 position in call
+            "speaker": speakers.get(sid),
+            "valence": VALENCE.get(s.get("sentiment"), 0.0) if ok else None,
+            "emotion": s.get("dominant_emotion"),
+            "escalation_score": s.get("escalation_score"),
+            "ok": bool(ok),
+        })
+    return rows
+
+
+def _channel_valence(rows, channel, late_weighted):
+    """(weighted mean valence, coverage) for one speaker channel."""
+    chan = [r for r in rows if r["speaker"] == channel]
+    if not chan:
+        return None, 0.0
+    ok = [r for r in chan if r["ok"]]
+    coverage = len(ok) / len(chan)
+    if not ok:
+        return None, 0.0
+    wsum = vsum = 0.0
+    for r in ok:
+        w = LATE_WEIGHT if (late_weighted and r["pos"] >= 1 - LATE_FRACTION) else 1.0
+        wsum += w
+        vsum += w * r["valence"]
+    return vsum / wsum, coverage
+
+
+def _acoustic_escalation(rows):
+    """Acoustic risk tier from the customer escalation_score trajectory."""
+    cust = [r for r in rows
+            if r["speaker"] == "CUSTOMER" and r["ok"]
+            and r["escalation_score"] is not None]
+    if not cust:
+        return None, None, None
+    late = [r["escalation_score"] for r in cust if r["pos"] >= 1 - LATE_FRACTION]
+    late_mean = sum(late) / len(late) if late else 0.0
+    peak = max(r["escalation_score"] for r in cust)
+
+    for tier in ("escalate", "review"):
+        if late_mean >= ESC_THRESHOLDS[tier]["late_mean"]:
+            return tier, round(late_mean, 3), round(peak, 3)
+    return "none", round(late_mean, 3), round(peak, 3)
+
+
+def trajectory(rows, buckets=48):
+    """
+    Downsample the per-sentence acoustic signal into fixed position buckets
+    for the frontend timeline: [{p, esc, cust_val, agent_val}, ...].
+    p = bucket midpoint (0..1); esc = mean customer escalation_score;
+    *_val = mean valence per channel. Fields are null where a bucket has no
+    processed sentences for that channel.
+    """
+    out = []
+    for b in range(buckets):
+        lo, hi = b / buckets, (b + 1) / buckets
+        cell = [r for r in rows if r["ok"] and lo <= r["pos"] < hi]
+        esc = [r["escalation_score"] for r in cell
+               if r["speaker"] == "CUSTOMER" and r["escalation_score"] is not None]
+        cv = [r["valence"] for r in cell
+              if r["speaker"] == "CUSTOMER" and r["valence"] is not None]
+        av = [r["valence"] for r in cell
+              if r["speaker"] == "AGENT" and r["valence"] is not None]
+        out.append({
+            "p": round((lo + hi) / 2, 4),
+            "esc": round(sum(esc) / len(esc), 3) if esc else None,
+            "cust_val": round(sum(cv) / len(cv), 3) if cv else None,
+            "agent_val": round(sum(av) / len(av), 3) if av else None,
+        })
+    return out
+
+
+def fuse_evaluation(ev, rows):
+    """
+    Mutate an evaluation dict in place: fuse acoustic signal into the three
+    audio-informed quality dimensions and the escalation risk level.
+    Every dimension gets a `hybrid` provenance block (method "text_only"
+    when acoustics did not contribute) so the UI can explain each score.
+    Returns a short summary dict for node_meta.
+    """
+    fused_dims = []
+
+    for name, dim in (ev.get("quality") or {}).items():
+        if not isinstance(dim, dict) or "score" not in dim:
+            continue
+        text_score = dim["score"]
+        cfg = FUSED_DIMS.get(name)
+        vbar, coverage = (_channel_valence(rows, cfg["channel"], cfg["late_weighted"])
+                          if (cfg and rows) else (None, 0.0))
+        if cfg and vbar is not None:
+            a = 3 + 2 * vbar
+            w_a = W_ACOUSTIC_CAP * coverage
+            fused = max(1, min(5, round((1 - w_a) * text_score + w_a * a)))
+            dim["score"] = fused
+            dim["hybrid"] = {
+                "text_score": text_score,
+                "acoustic_score": round(a, 2),
+                "text_weight": round(1 - w_a, 2),
+                "acoustic_weight": round(w_a, 2),
+                "coverage": round(coverage, 2),
+                "channel": cfg["channel"],
+                "method": "weighted_mean",
+            }
+            fused_dims.append(name)
+        else:
+            dim["hybrid"] = {
+                "text_score": text_score,
+                "acoustic_score": None,
+                "text_weight": 1.0,
+                "acoustic_weight": 0.0,
+                "coverage": None,
+                "channel": cfg["channel"] if cfg else None,
+                "method": "text_only",
+            }
+
+    # Escalation merge (v0.4.1): agreement is trivially deterministic;
+    # disagreement carries no information about which channel to trust, so it
+    # is NOT resolved here -- the graph routes disputed calls to the arbitrate
+    # LLM node, which weighs both signals against the transcript. Until (or
+    # unless) arbitration runs, the text tier stands.
+    disputed = False
+    esc = ev.get("escalation") or {}
+    text_risk = esc.get("risk_level")
+    a_risk, late_mean, peak = _acoustic_escalation(rows) if rows else (None, None, None)
+    if text_risk and a_risk is not None:
+        disputed = a_risk != text_risk
+        esc["hybrid"] = {
+            "text_risk": text_risk,
+            "acoustic_risk": a_risk,
+            "late_mean_escalation": late_mean,
+            "peak_escalation": peak,
+            "method": "agreement" if not disputed else "text_only",
+            "arbitration_rationale": None,
+        }
+    elif text_risk:
+        esc["hybrid"] = {
+            "text_risk": text_risk,
+            "acoustic_risk": None,
+            "late_mean_escalation": None,
+            "peak_escalation": None,
+            "method": "text_only",
+            "arbitration_rationale": None,
+        }
+
+    # downsampled acoustic timeline for the frontend Agent-findings lanes
+    if rows:
+        ev["_acoustic"] = {"trajectory": trajectory(rows),
+                           "n_sentences": len(rows)}
+
+    return {
+        "acoustic_available": bool(rows),
+        "fused_dims": fused_dims,
+        "acoustic_risk": a_risk,
+        "disputed": disputed,
+    }

ml-services/evaluation/gen_call_data.py

@@ -0,0 +1,115 @@
+"""
+Build the frontend's call_data.json from transcript_data.json:
+  - groups words into speaker turns (agent/customer)
+  - chapters the call via the LiteLLM router
+  - emits turns + chapters with numeric seconds, ready for the React player
+
+Output: frontend/src/call_data.json
+  { call, duration, turns:[{i,speaker,text,start,end}],
+    chapters:[{index,label,summary,start,end,start_turn}] }
+"""
+import os, json, time
+import paths
+from env_util import load_env
+from segment import SYSTEM, SKELETON, numbered_transcript
+from router import chat_json_routed
+from extract import strip_fences
+
+load_env()
+
+FRONTEND = str(paths.FRONTEND_ROOT)
+TRANSCRIPT = os.path.join(FRONTEND, "transcript_data.json")
+OUT = os.path.join(FRONTEND, "src", "call_data.json")
+
+
+def build_turns(data):
+    """Merge agent+customer words into time-ordered speaker turns."""
+    merged = []
+    for speaker, key in (("AGENT", "agent"), ("CUSTOMER", "customer")):
+        for w in data.get(key, []):
+            if w.get("start") is None:
+                continue
+            merged.append((w["start"], w.get("end", w["start"]),
+                           speaker, w["word"]))
+    merged.sort(key=lambda x: x[0])
+
+    turns = []
+    cur = None
+    for start, end, sp, word in merged:
+        if cur is None or sp != cur["speaker"]:
+            if cur:
+                turns.append(cur)
+            cur = {"speaker": sp, "text": word, "start": start, "end": end}
+        else:
+            cur["text"] += " " + word
+            cur["end"] = end
+    if cur:
+        turns.append(cur)
+
+    for i, t in enumerate(turns):
+        t["i"] = i
+        t["start"] = round(t["start"], 2)
+        t["end"] = round(t["end"], 2)
+    return turns
+
+
+def repair_chapters(seeds, turns, duration):
+    n = len(turns)
+    cleaned, seen = [], set()
+    for s in seeds:
+        try:
+            st = max(0, min(int(s["start_turn"]), n - 1))
+        except (KeyError, ValueError, TypeError):
+            continue
+        if st in seen:
+            continue
+        seen.add(st)
+        cleaned.append({"start_turn": st,
+                        "label": str(s.get("label", "Untitled")).strip(),
+                        "summary": str(s.get("summary", "")).strip()})
+    cleaned.sort(key=lambda x: x["start_turn"])
+    if not cleaned:
+        cleaned = [{"start_turn": 0, "label": "Full Call", "summary": ""}]
+    if cleaned[0]["start_turn"] != 0:
+        cleaned[0]["start_turn"] = 0
+
+    chapters = []
+    for i, c in enumerate(cleaned):
+        st = c["start_turn"]
+        start = turns[st]["start"]
+        end = (turns[cleaned[i + 1]["start_turn"]]["start"]
+               if i + 1 < len(cleaned) else duration)
+        chapters.append({"index": i + 1, "label": c["label"],
+                         "summary": c["summary"], "start": round(start, 2),
+                         "end": round(end, 2), "start_turn": st})
+    return chapters
+
+
+def main():
+    with open(TRANSCRIPT, encoding="utf-8") as f:
+        data = json.load(f)
+
+    turns = build_turns(data)
+    duration = round(max(t["end"] for t in turns), 2)
+    print(f"call={data.get('call')} | {len(turns)} turns | {duration:.0f}s")
+
+    user = f"{numbered_transcript(turns)}\n\n{SKELETON}"
+    t0 = time.time()
+    raw, served = chat_json_routed(SYSTEM, user, max_tokens=1500, return_meta=True)
+    seeds = json.loads(strip_fences(raw)).get("chapters", [])
+    chapters = repair_chapters(seeds, turns, duration)
+    print(f"chaptered by {served} in {time.time()-t0:.1f}s -> {len(chapters)} chapters")
+    for c in chapters:
+        print(f"  [{c['index']}] {c['start']:6.1f}-{c['end']:6.1f}  {c['label']}")
+
+    out = {"call": data.get("call", "call_1"),
+           "model": data.get("model", ""),
+           "duration": duration, "turns": turns, "chapters": chapters}
+    os.makedirs(os.path.dirname(OUT), exist_ok=True)
+    with open(OUT, "w", encoding="utf-8") as f:
+        json.dump(out, f, indent=2)
+    print(f"\nSaved -> {OUT}")
+
+
+if __name__ == "__main__":
+    main()

ml-services/evaluation/graph.py

@@ -0,0 +1,353 @@
+"""
+Graph-based call evaluation pipeline (v0.4.1).
+
+Decomposes the monolithic extract.py into five parallel LLM nodes
+(compliance, quality, escalation, chapters, workflow) orchestrated by
+LangGraph. Each node gets a focused prompt, validates its own output
+fragment, and retries independently on failure. After aggregation, a
+deterministic fuse step blends the audio sentiment model's output into the
+audio-informed scores; when the text and acoustic escalation tiers DISAGREE
+the graph summons an arbitration LLM node (v0.4.1) instead of merging by
+rule. Anchoring + runtime routing (evidence re-runs, conditional
+investigation) close the loop.
+
+Graph shape:
+    assemble → [compliance, quality, escalation, chapters, workflow]
+             → aggregate → fuse → (arbitrate?) → anchor
+             → (re-run | investigate | END)
+
+Usage:
+  python graph.py --call_id en_CA_Banking_1592237
+  python graph.py --call_id en_CA_Banking_1592237 --results_dir results_channels
+"""
+import os
+import json
+import time
+import argparse
+from typing import TypedDict, Annotated
+
+from langgraph.graph import StateGraph, START, END
+
+
+def _merge_meta(a, b):
+    """Reducer: parallel nodes each contribute their own node_meta entry."""
+    return {**(a or {}), **(b or {})}
+
+from nodes import (
+    assemble_node,
+    compliance_node,
+    quality_node,
+    escalation_node,
+    chapter_node,
+    workflow_node,
+    aggregate_node,
+    fuse_node,
+    arbitrate_node,
+    anchor_node,
+    investigate_node,
+)
+from rubric import RUBRIC_VERSION_GRAPH
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# State schema — keys are populated incrementally by each node
+# ─────────────────────────────────────────────────────────────────────────────
+
+class EvalState(TypedDict, total=False):
+    # inputs
+    call_id: str
+    results_dir: str
+
+    # set by assemble
+    packet: str
+    meta: dict
+    turns: list
+
+    # set by parallel LLM nodes
+    compliance: object
+    quality: object
+    escalation: object
+    chapters: dict
+    workflow: dict
+
+    # set by aggregate + anchor
+    evaluation: dict
+
+    # v0.3.0 control fields
+    anchor_attempts: int      # anchor passes so far (loop guard)
+    anchor_feedback: dict     # {section: [unanchored quotes]} from last pass
+
+    # v0.4.1: set by fuse — text vs acoustic escalation tiers disagree
+    fusion_disputed: bool
+
+    # per-node execution metadata (duration, served model, run count);
+    # parallel nodes merge via reducer instead of clobbering each other
+    node_meta: Annotated[dict, _merge_meta]
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Graph construction
+# ─────────────────────────────────────────────────────────────────────────────
+
+# which evaluation section is produced by which LLM node
+SECTION_NODE = {"compliance": "compliance",
+                "quality": "quality",
+                "escalation": "escalation"}
+
+MAX_ANCHOR_PASSES = 2  # first pass + one corrective re-run
+
+
+def route_after_fuse(state):
+    """
+    v0.4.1 routing. Deterministic merging is only defensible when the text
+    and acoustic escalation tiers AGREE (they carry no information to break
+    their own tie). On disagreement, summon the arbitration LLM node.
+    """
+    if state.get("fusion_disputed"):
+        h = state["evaluation"]["escalation"].get("hybrid") or {}
+        print(f"  [router] escalation tiers disagree "
+              f"(text={h.get('text_risk')}, acoustic={h.get('acoustic_risk')})"
+              f" -> arbitration")
+        return "arbitrate"
+    return "anchor"
+
+
+def route_after_anchor(state):
+    """
+    v0.3.0 runtime routing. After anchoring, decide:
+      1. Any section cited quotes that don't exist in the transcript, and we
+         still have a corrective pass left? -> re-run ONLY those LLM nodes
+         with the failed quotes as a revision note.
+      2. Otherwise, does the call carry escalation risk? -> deep-dive
+         investigation node.
+      3. Otherwise -> done.
+    """
+    feedback = state.get("anchor_feedback") or {}
+    if feedback and state.get("anchor_attempts", 0) < MAX_ANCHOR_PASSES:
+        targets = [SECTION_NODE[s] for s in feedback if s in SECTION_NODE]
+        if targets:
+            print(f"  [router] unverified evidence -> re-running: "
+                  f"{', '.join(targets)}")
+            return targets
+
+    risk = state["evaluation"]["escalation"]["risk_level"]
+    if risk != "none":
+        print(f"  [router] risk_level={risk} -> investigation")
+        return "investigate"
+    return END
+
+
+def build_graph():
+    g = StateGraph(EvalState)
+
+    g.add_node("assemble", assemble_node)
+    g.add_node("compliance", compliance_node)
+    g.add_node("quality", quality_node)
+    g.add_node("escalation", escalation_node)
+    g.add_node("chapters", chapter_node)
+    g.add_node("workflow", workflow_node)
+    g.add_node("aggregate", aggregate_node)
+    g.add_node("fuse", fuse_node)
+    g.add_node("arbitrate", arbitrate_node)
+    g.add_node("anchor", anchor_node)
+    g.add_node("investigate", investigate_node)
+
+    # assemble fans out to 5 parallel nodes
+    g.add_edge(START, "assemble")
+    g.add_edge("assemble", "compliance")
+    g.add_edge("assemble", "quality")
+    g.add_edge("assemble", "escalation")
+    g.add_edge("assemble", "chapters")
+    g.add_edge("assemble", "workflow")
+
+    # all 5 converge into aggregate
+    g.add_edge("compliance", "aggregate")
+    g.add_edge("quality", "aggregate")
+    g.add_edge("escalation", "aggregate")
+    g.add_edge("chapters", "aggregate")
+    g.add_edge("workflow", "aggregate")
+
+    # aggregate → fuse (acoustic-text fusion), then RUNTIME routing:
+    #   text/acoustic tiers disagree → arbitrate (LLM weighs both in context;
+    #     agreement merges deterministically and skips this — v0.4.1)
+    #   unverified evidence  → back to the offending LLM node(s) (once)
+    #   escalation risk      → investigation deep-dive
+    #     (fuse/arbitrate run BEFORE routing, so an acoustically-detected
+    #      risk can summon the investigation even when the words look calm)
+    #   clean + calm         → END
+    g.add_edge("aggregate", "fuse")
+    g.add_conditional_edges(
+        "fuse", route_after_fuse, ["arbitrate", "anchor"])
+    g.add_edge("arbitrate", "anchor")
+    g.add_conditional_edges(
+        "anchor", route_after_anchor,
+        ["compliance", "quality", "escalation", "investigate", END])
+    g.add_edge("investigate", END)
+
+    return g.compile()
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Report
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _mmss(s):
+    return f"{int(s // 60):02d}:{int(s % 60):02d}"
+
+
+def print_graph_report(ev, dt):
+    meta = ev.get("metadata", {})
+    print("\n" + "=" * 68)
+    print(f"  GRAPH EVALUATION  {meta.get('call_id', '?')}  "
+          f"({meta.get('domain', '?')})  [{dt:.1f}s total]")
+    print(f"  rubric {ev.get('rubric_version', '?')}")
+    print("=" * 68)
+
+    # compliance
+    print("\n  COMPLIANCE")
+    c = ev.get("compliance", {})
+    for key, name in [("name_announced", "Name announced"),
+                      ("company_announced", "Company announced"),
+                      ("recording_disclosure", "Recording disclosure"),
+                      ("identity_verified", "Identity verified"),
+                      ("resolution_provided", "Resolution provided"),
+                      ("transfer_next_steps", "Transfer next-steps")]:
+        item = c.get(key, {})
+        mark = {True: "PASS", False: "FAIL", None: "N/A "}[item.get("passed")]
+        evd = item.get("evidence")
+        when = (f" @{_mmss(evd['sec'])}" if evd and evd.get("sec") is not None
+                else "")
+        quote = (f'  "{evd["quote"][:50]}"' if evd
+                 else (f"  ({item.get('note')})" if item.get("note") else ""))
+        print(f"    [{mark}]{when:>7} {name:<22}{quote}")
+    if c.get("identity_method"):
+        print(f"              method: {c['identity_method']}")
+
+    # quality
+    print("\n  QUALITY (1-5, hybrid text+acoustic)")
+    q = ev.get("quality", {})
+    dims = [("efficiency", "Efficiency"),
+            ("problem_resolution", "Problem resolution"),
+            ("clarity", "Clarity"),
+            ("professionalism", "Professionalism"),
+            ("empathy", "Empathy"),
+            ("customer_satisfaction", "Customer satisfaction")]
+    for key, name in dims:
+        d = q.get(key)
+        if d is None:
+            continue
+        h = d.get("hybrid") or {}
+        if h.get("method") == "weighted_mean":
+            how = (f" (text {h['text_score']} x {h['text_weight']} + "
+                   f"audio {h['acoustic_score']} x {h['acoustic_weight']}, "
+                   f"{h['channel'].lower()} ch)")
+        else:
+            how = " (text only)"
+        print(f"    {d['score']}/5  {name:<24}{how}")
+        if d.get("signals_absent"):
+            print(f"          missing: {', '.join(d['signals_absent'][:3])}")
+
+    # escalation
+    e = ev.get("escalation", {})
+    flags = e.get("red_flags", [])
+    eh = e.get("hybrid") or {}
+    fused = (f"  [text: {eh['text_risk']} | acoustic: {eh['acoustic_risk']} "
+             f"(late-mean {eh['late_mean_escalation']}, peak {eh['peak_escalation']})"
+             f" via {eh['method']}]"
+             if eh.get("acoustic_risk") is not None else "")
+    print(f"\n  ESCALATION: {e.get('risk_level', '?').upper()}{fused}  "
+          f"| emotion(text): {e.get('customer_emotion_text', '?')}"
+          f" | flags: {', '.join(flags) if flags else 'none'}")
+    if eh.get("arbitration_rationale"):
+        print(f"    arbitrator: {eh['arbitration_rationale']}")
+
+    # workflow (v0.4.0)
+    wf = ev.get("workflow")
+    if wf:
+        print(f"\n  EXPECTED WORKFLOW  (subject: {wf.get('subject', '?')})")
+        for s in wf.get("expected_steps", []):
+            mark = {True: "[x]", False: "[ ]", None: "[-]"}[s.get("met")]
+            evd = s.get("evidence")
+            when = (f" @{_mmss(evd['sec'])}" if evd and evd.get("sec") is not None
+                    else "")
+            print(f"    {mark}{when:>7} {s['step']}")
+
+    # investigation (v0.3.0 — only when risk_level != none)
+    inv = ev.get("investigation")
+    if inv:
+        print(f"\n  INVESTIGATION [{inv.get('priority', '?').upper()} priority]")
+        print(f"    {inv.get('summary', '')}")
+        for f_ in inv.get("contributing_factors", []):
+            print(f"    - {f_}")
+        print(f"    ACTION: {inv.get('recommended_action', '')}")
+
+    # summary
+    summary = ev.get("overall_summary")
+    if summary:
+        print(f"\n  SUMMARY: {summary}")
+
+    # anchor stats
+    astats = ev.get("_anchor_stats", {})
+    if astats:
+        print(f"\n  EVIDENCE: {astats.get('anchored', 0)}/{astats.get('total', 0)} "
+              f"quotes anchored to transcript timestamps")
+
+    print("=" * 68)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# CLI
+# ─────────────────────────────────────────────────────────────────────────────
+
+def evaluate_call(call_id, results_dir="results"):
+    """Run the evaluation graph for one call and return the evaluation dict.
+
+    The `_pipeline` execution trace (node timings, anchor passes, wall clock)
+    is attached for the frontend pipeline strip. No file I/O -- callers (the
+    CLI below, the single-call orchestrator, the worker) decide where it goes.
+    """
+    graph = build_graph()
+    t0 = time.time()
+    final = graph.invoke({
+        "call_id": call_id,
+        "results_dir": results_dir,
+    })
+    dt = time.time() - t0
+
+    ev = final["evaluation"]
+    ev["_pipeline"] = {
+        "nodes": final.get("node_meta", {}),
+        "anchor_passes": final.get("anchor_attempts", 1),
+        "wall_clock": round(dt, 1),
+    }
+    return ev
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Graph-based call evaluation (parallel LLM nodes)")
+    ap.add_argument("--call_id", required=True)
+    ap.add_argument("--results_dir", default="results_channels")
+    args = ap.parse_args()
+
+    print(f"\nEvaluating {args.call_id} via graph pipeline "
+          f"(rubric {RUBRIC_VERSION_GRAPH})...")
+    print("-" * 68)
+
+    ev = evaluate_call(args.call_id, args.results_dir)
+    dt = ev["_pipeline"]["wall_clock"]
+
+    # save
+    here = os.path.dirname(os.path.abspath(__file__))
+    out_dir = os.path.join(here, "results")
+    os.makedirs(out_dir, exist_ok=True)
+    out = os.path.join(out_dir, f"{args.call_id}_graph.json")
+    with open(out, "w", encoding="utf-8") as f:
+        json.dump(ev, f, indent=2)
+
+    print_graph_report(ev, dt)
+    print(f"\nSaved -> {out}")
+
+
+if __name__ == "__main__":
+    main()

ml-services/evaluation/llm_client.py

@@ -0,0 +1,243 @@
+"""
+Provider-agnostic LLM client (OpenAI SDK-compatible for all providers).
+
+Swap provider with a single string -- the rest of the pipeline never changes.
+Used by extract.py via: chat_json(provider, system, user)
+
+Provider notes
+--------------
+github     : GitHub Models free tier -- GPT-4o-mini, zero cost, recommended default
+openai     : OpenAI direct -- costs credits, use for prod/comparison only
+gemini     : Google Gemini via OpenAI-compat endpoint -- free tier, 15 RPM daily cap
+mistral    : Mistral AI -- mistral-small is strong at structured JSON
+cohere     : Cohere -- command-r7b fastest (326ms), good at extraction
+nvidia     : NVIDIA NIM -- llama-3.1-8b, solid and fast
+cerebras   : Cerebras -- zai-glm-4.7 is a reasoning model (needs higher token budget)
+cloudflare : Cloudflare Workers AI -- needs CLOUDFLARE_ID in .env
+huggingface: HF Inference via featherless-ai -- slowest, cold-start ~14s
+sambanova  : SambaNova -- has DeepSeek-V3 + Llama-4, fast on warm runs
+openrouter : OpenRouter -- free models available, quality varies
+deepseek   : DeepSeek -- needs paid balance (402 currently)
+
+JSON mode support
+-----------------
+Providers that natively support response_format=json_object are flagged
+json_mode=True. Others receive a JSON-enforcement suffix in the system prompt
+and the response is extracted via regex fallback if needed.
+"""
+
+from __future__ import annotations
+import os, re, json
+from typing import Optional
+from openai import OpenAI
+from env_util import load_env
+
+load_env()
+
+
+# ── Provider registry ─────────────────────────────────────────────────────────
+# Each entry: base_url, key_env, default_model, json_mode, extra_kwargs
+PROVIDERS: dict[str, dict] = {
+    "github": {
+        "base_url":      "https://models.github.ai/inference",
+        "key_env":       "GITHUB_TOKEN",
+        "default_model": "openai/gpt-4o-mini",
+        "json_mode":     True,
+    },
+    "openai": {
+        "base_url":      None,
+        "key_env":       "OPENAI_API_KEY",
+        "default_model": "gpt-4o-mini",
+        "json_mode":     True,
+    },
+    "gemini": {
+        "base_url":      "https://generativelanguage.googleapis.com/v1beta/openai/",
+        "key_env":       "GEMINI_API_KEY",
+        "default_model": "gemini-2.0-flash",
+        "json_mode":     True,
+    },
+    "mistral": {
+        "base_url":      "https://api.mistral.ai/v1",
+        "key_env":       "MISTRAL_API_KEY",
+        "default_model": "mistral-small-latest",
+        "json_mode":     True,
+    },
+    "cohere": {
+        "base_url":      "https://api.cohere.com/compatibility/v1",
+        "key_env":       "COHERE_API_KEY",
+        "default_model": "command-r7b-12-2024",
+        "json_mode":     False,   # prompt-based JSON
+    },
+    "nvidia": {
+        "base_url":      "https://integrate.api.nvidia.com/v1",
+        "key_env":       "NVIDIA_API_KEY",
+        "default_model": "meta/llama-3.1-8b-instruct",
+        "json_mode":     False,
+    },
+    "cerebras": {
+        "base_url":      "https://api.cerebras.ai/v1",
+        "key_env":       "CEREBRAS_API_KEY",
+        "default_model": "zai-glm-4.7",
+        "json_mode":     False,
+        "reasoning":     True,    # needs higher token budget for chain-of-thought
+    },
+    "cloudflare": {
+        "base_url":      None,    # built dynamically using CLOUDFLARE_ID
+        "key_env":       "CLOUDFLARE_API_TOKEN",
+        "default_model": "@cf/meta/llama-3.1-8b-instruct",
+        "json_mode":     False,
+    },
+    "huggingface": {
+        "base_url":      "https://router.huggingface.co/featherless-ai/v1",
+        "key_env":       "HUGGINGFACE_API_TOKEN",
+        "default_model": "meta-llama/Llama-3.1-8B-Instruct",
+        "json_mode":     False,
+    },
+    "sambanova": {
+        "base_url":      "https://api.sambanova.ai/v1",
+        "key_env":       "SAMBANOVA_API_KEY",
+        "default_model": "Meta-Llama-3.3-70B-Instruct",
+        "json_mode":     False,
+    },
+    "openrouter": {
+        "base_url":      "https://openrouter.ai/api/v1",
+        "key_env":       "OPENROUTER_API_KEY",
+        "default_model": "nvidia/nemotron-3-ultra-550b-a55b:free",
+        "json_mode":     False,
+    },
+    "deepseek": {
+        "base_url":      "https://api.deepseek.com/v1",
+        "key_env":       "DEEPSEEK_API_KEY",
+        "default_model": "deepseek-chat",
+        "json_mode":     True,
+    },
+}
+
+JSON_ENFORCE_SUFFIX = (
+    "\n\nCRITICAL: Your response must be a single valid JSON object. "
+    "No markdown fences, no commentary, no preamble. Start with { and end with }."
+)
+
+
+def get_client(provider: str) -> tuple[OpenAI, dict]:
+    """Return (OpenAI client, provider_config)."""
+    if provider not in PROVIDERS:
+        raise ValueError(f"Unknown provider '{provider}'. Choose from: {list(PROVIDERS)}")
+    cfg = PROVIDERS[provider]
+
+    key_env = cfg.get("key_env")
+    api_key = os.environ.get(key_env, "") if key_env else "local"
+    if key_env and not api_key:
+        raise SystemExit(f"Missing {key_env} in .env for provider '{provider}'")
+
+    base_url = cfg["base_url"]
+    if provider == "cloudflare":
+        account_id = os.environ.get("CLOUDFLARE_ID", "")
+        if not account_id:
+            raise SystemExit("Missing CLOUDFLARE_ID in .env")
+        base_url = f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1"
+
+    kwargs: dict = {"api_key": api_key}
+    if base_url:
+        kwargs["base_url"] = base_url
+
+    return OpenAI(**kwargs), cfg
+
+
+def _extract_json(text: str) -> str:
+    """Strip markdown fences and extract the first {...} block."""
+    text = text.strip()
+    # strip ```json ... ``` or ``` ... ```
+    text = re.sub(r"^```(?:json)?\s*", "", text, flags=re.MULTILINE)
+    text = re.sub(r"\s*```$", "", text, flags=re.MULTILINE)
+    text = text.strip()
+    # find first { ... } spanning the whole string
+    start = text.find("{")
+    if start == -1:
+        return text
+    depth, end = 0, -1
+    for i, ch in enumerate(text[start:], start):
+        if ch == "{":
+            depth += 1
+        elif ch == "}":
+            depth -= 1
+            if depth == 0:
+                end = i
+                break
+    return text[start:end + 1] if end != -1 else text[start:]
+
+
+def chat_json(
+    provider: str,
+    system: str,
+    user: str,
+    model: Optional[str] = None,
+    temperature: float = 0.1,
+    max_tokens: int = 4000,
+) -> str:
+    """
+    Single chat completion that returns a JSON string.
+
+    For providers with native json_mode, passes response_format=json_object.
+    For others, appends a JSON-enforcement instruction and strips fences from
+    the response. Either way the caller gets a raw JSON string to parse.
+    """
+    client, cfg = get_client(provider)
+    chosen_model = model or cfg["default_model"]
+    use_json_mode = cfg.get("json_mode", False)
+
+    # reasoning models need a much larger token budget
+    if cfg.get("reasoning") and max_tokens < 2048:
+        max_tokens = 2048
+
+    sys_msg = system if use_json_mode else system + JSON_ENFORCE_SUFFIX
+
+    kwargs: dict = {
+        "model":       chosen_model,
+        "messages":    [{"role": "system", "content": sys_msg},
+                        {"role": "user",   "content": user}],
+        "temperature": temperature,
+        "max_tokens":  max_tokens,
+    }
+    if use_json_mode:
+        kwargs["response_format"] = {"type": "json_object"}
+
+    resp = client.chat.completions.create(**kwargs)
+    raw = resp.choices[0].message.content or ""
+    return _extract_json(raw) if not use_json_mode else raw.strip()
+
+
+def list_providers() -> list[str]:
+    return list(PROVIDERS.keys())
+
+
+def default_model(provider: str) -> str:
+    return PROVIDERS[provider]["default_model"]
+
+
+# ── Quick smoke test ──────────────────────────────────────────────────────────
+if __name__ == "__main__":
+    import time, argparse
+    ap = argparse.ArgumentParser(description="Ping all providers or a specific one.")
+    ap.add_argument("--provider", default=None, help="test one provider only")
+    ap.add_argument("--prompt", default="Reply with valid JSON: {\"status\": \"ok\", \"msg\": \"hello\"}")
+    args = ap.parse_args()
+
+    targets = [args.provider] if args.provider else list(PROVIDERS.keys())
+    print(f"\n  {'Provider':<14} {'Model':<42} {'ms':>6}  {'Result'}")
+    print("  " + "-" * 90)
+
+    for p in targets:
+        cfg = PROVIDERS[p]
+        mod = cfg["default_model"]
+        try:
+            t0 = time.time()
+            raw = chat_json(p, "You are a helpful assistant.", args.prompt)
+            ms = int((time.time() - t0) * 1000)
+            # try to parse to confirm it's valid JSON
+            parsed = json.loads(raw)
+            print(f"  [OK] {p:<12} {mod:<42} {ms:>6}ms  {str(parsed)[:60]}")
+        except SystemExit as e:
+            print(f"  [--] {p:<12} {mod:<42}  {'missing key -- skipped'}")
+        except Exception as e:
+            print(f"  [XX] {p:<12} {mod:<42}  {str(e)[:70]}")

ml-services/evaluation/nodes.py

@@ -0,0 +1,517 @@
+"""
+Node functions for the graph-based evaluation pipeline.
+
+Each node is a callable that takes EvalState and returns a partial state dict.
+LLM nodes use a shared retry helper; deterministic nodes (assemble, aggregate,
+anchor) do pure data transformation.
+"""
+import os
+import json
+import time
+
+from pydantic import ValidationError
+
+import paths
+from env_util import load_env
+from assemble import load_manifest, build_packet, assemble_turns, estimate_duration
+from pydantic import BaseModel
+from typing import List, Literal
+
+from rubric import (
+    ComplianceChecklist, QualityDimensions, EscalationRisk,
+    InvestigationReport, CallMetadata, CallEvaluation, RUBRIC_VERSION_GRAPH,
+    CallWorkflow, WorkflowStep, Evidence,
+)
+from prompts import (
+    COMPLIANCE_SYSTEM, COMPLIANCE_SKELETON,
+    QUALITY_SYSTEM, QUALITY_SKELETON,
+    ESCALATION_SYSTEM, ESCALATION_SKELETON,
+    INVESTIGATE_SYSTEM, INVESTIGATE_SKELETON,
+    ARBITRATE_SYSTEM, ARBITRATE_SKELETON,
+    SUBJECT_SYSTEM, SUBJECT_SKELETON,
+    WORKFLOW_GEN_SYSTEM, WORKFLOW_GEN_SKELETON,
+    WORKFLOW_CHECK_SYSTEM, WORKFLOW_CHECK_SKELETON,
+    WORKFLOW_RECHECK_SYSTEM, WORKFLOW_RECHECK_SKELETON,
+)
+
+load_env()
+DATA = str(paths.NA_TESTSET)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Shared LLM call + retry helper
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _strip_fences(s):
+    s = s.strip()
+    if s.startswith("```"):
+        s = s.split("```", 2)[1] if "```" in s[3:] else s[3:]
+        if s.startswith("json"):
+            s = s[4:]
+        s = s.rsplit("```", 1)[0]
+    return s.strip()
+
+
+def _feedback_text(quotes):
+    """Build a revision note for a node whose cited quotes failed anchoring."""
+    if not quotes:
+        return None
+    listed = "\n".join(f'- "{q}"' for q in quotes)
+    return ("REVISION NOTE: in a previous evaluation of this call, the "
+            "following cited quotes were NOT found verbatim in the "
+            f"transcript:\n{listed}\n"
+            "Re-evaluate. Every quote must be copied EXACTLY, character for "
+            "character, from a single transcript turn. Do not paraphrase, "
+            "do not merge text from multiple turns.")
+
+
+# provider escalation order for eval nodes: a malformed/invalid 200-OK
+# response is a model-quality failure the transport-level router never sees,
+# so retrying the same provider on the same content fails identically. We
+# escalate the PROVIDER per attempt instead -- a different model's JSON
+# behaviour almost always parses where another's choked.
+_TIER_SEQUENCE = ("qa-primary", "qa-fallback", "qa-safety")
+
+
+def _llm_eval_with_retry(system, skeleton, packet, validator, max_attempts=3,
+                         feedback=None):
+    """
+    Call the LLM router, parse JSON, validate with `validator`, retry on
+    failure with error feedback AND provider escalation. Returns
+    (validated_model, dt, served_model). `feedback` is an optional revision
+    note (e.g. failed anchor quotes) appended to the user prompt.
+    """
+    from router import chat_json_routed
+
+    user = f"{packet}\n\n{skeleton}"
+    if feedback:
+        user += f"\n\n{feedback}"
+    last_err = None
+
+    for attempt in range(1, max_attempts + 1):
+        tier = _TIER_SEQUENCE[min(attempt - 1, len(_TIER_SEQUENCE) - 1)]
+        t0 = time.time()
+        raw, served = chat_json_routed(system, user, return_meta=True, tier=tier)
+        dt = time.time() - t0
+        try:
+            data = json.loads(_strip_fences(raw))
+            result = validator(data)
+            return result, dt, served
+        except (json.JSONDecodeError, ValidationError, ValueError) as e:
+            last_err = e
+            print(f"  [{system[:40]}...] attempt {attempt} ({tier}) failed "
+                  f"({type(e).__name__}); escalating provider...")
+            user = (f"{packet}\n\n{skeleton}"
+                    + (f"\n\n{feedback}" if feedback else "")
+                    + f"\n\nYour previous output was invalid: "
+                    f"{str(e)[:400]}. Return corrected JSON only.")
+
+    raise RuntimeError(f"Node failed after {max_attempts} attempts: {last_err}")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Deterministic nodes
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _runs(state, node):
+    """How many times `node` has executed in this invocation (1-based)."""
+    return ((state.get("node_meta") or {}).get(node, {}).get("runs", 0)) + 1
+
+
+def assemble_node(state):
+    """Load transcript, build packet and turns for downstream nodes."""
+    t0 = time.time()
+    call_id = state["call_id"]
+    results_dir = state.get("results_dir", "results_channels")
+    manifest = load_manifest()
+    meta = manifest[call_id]
+
+    result_path = os.path.join(DATA, results_dir, meta["accent"],
+                               call_id + ".json")
+    with open(result_path, encoding="utf-8") as f:
+        result = json.load(f)
+
+    packet, stats = build_packet(result, meta)
+    turns = assemble_turns(result)
+
+    cm = CallMetadata(
+        call_id=call_id,
+        domain=result.get("domain", meta.get("domain", "unknown")),
+        accent=meta.get("accent"),
+        duration_seconds=stats["duration_s"],
+        transcript_model=result.get("model"),
+    )
+
+    return {
+        "packet": packet,
+        "meta": cm.model_dump(),
+        "turns": turns,
+        "node_meta": {"assemble": {"duration": round(time.time() - t0, 2)}},
+    }
+
+
+def aggregate_node(state):
+    """Merge parallel node outputs into a single CallEvaluation."""
+    ev = CallEvaluation(
+        rubric_version=RUBRIC_VERSION_GRAPH,
+        metadata=CallMetadata(**state["meta"]),
+        compliance=state["compliance"],
+        quality=state["quality"],
+        escalation=state["escalation"],
+        workflow=state.get("workflow"),
+        overall_summary=None,
+    )
+    return {"evaluation": ev.model_dump()}
+
+
+def _collect_unanchored(node, out):
+    """Recursively collect evidence quotes that anchoring could not locate."""
+    if isinstance(node, dict):
+        if "quote" in node and "speaker" in node and node.get("sec") is None:
+            out.append(node["quote"])
+        else:
+            for v in node.values():
+                _collect_unanchored(v, out)
+    elif isinstance(node, list):
+        for v in node:
+            _collect_unanchored(v, out)
+
+
+def anchor_node(state):
+    """
+    Attach real transcript timestamps to every evidence quote.
+    Also reports WHICH sections cited quotes that could not be located, so
+    the graph can route those sections back for re-evaluation (v0.3.0).
+    """
+    from eval_call_data import anchor_evidence
+
+    ev_dict = state["evaluation"]
+    if not isinstance(ev_dict, dict):
+        ev_dict = ev_dict.model_dump()
+
+    turns_indexed = [{"i": i, **t} for i, t in enumerate(state["turns"])]
+    stats = {"total": 0, "anchored": 0}
+    anchor_evidence(ev_dict, turns_indexed, stats)
+    ev_dict["_anchor_stats"] = stats
+
+    # per-section unanchored quotes -> feedback for the re-anchor loop
+    feedback = {}
+    for section in ("compliance", "quality", "escalation"):
+        missing = []
+        _collect_unanchored(ev_dict.get(section, {}), missing)
+        if missing:
+            feedback[section] = missing
+
+    attempts = state.get("anchor_attempts", 0) + 1
+    if feedback:
+        n = sum(len(v) for v in feedback.values())
+        print(f"  [anchor] pass {attempts}: {n} quote(s) not found verbatim "
+              f"in sections: {', '.join(feedback)}")
+    else:
+        print(f"  [anchor] pass {attempts}: all quotes anchored")
+
+    return {
+        "evaluation": ev_dict,
+        "anchor_attempts": attempts,
+        "anchor_feedback": feedback,
+    }
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# LLM nodes — each validates its own Pydantic fragment
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _section_feedback(state, section):
+    """Revision note if this section's quotes failed anchoring last pass."""
+    fb = (state.get("anchor_feedback") or {}).get(section)
+    if fb:
+        print(f"  [{section}] re-running with {len(fb)} failed-quote note(s)")
+    return _feedback_text(fb)
+
+
+def compliance_node(state):
+    """Evaluate 6 compliance checks."""
+    result, dt, served = _llm_eval_with_retry(
+        system=COMPLIANCE_SYSTEM,
+        skeleton=COMPLIANCE_SKELETON,
+        packet=state["packet"],
+        validator=lambda d: ComplianceChecklist.model_validate(d),
+        feedback=_section_feedback(state, "compliance"),
+    )
+    print(f"  [compliance] done in {dt:.1f}s via {served}")
+    return {"compliance": result,
+            "node_meta": {"compliance": {"duration": round(dt, 1),
+                                         "served_by": served,
+                                         "runs": _runs(state, "compliance")}}}
+
+
+def quality_node(state):
+    """Score 6 quality dimensions with enriched signal guidance."""
+    result, dt, served = _llm_eval_with_retry(
+        system=QUALITY_SYSTEM,
+        skeleton=QUALITY_SKELETON,
+        packet=state["packet"],
+        validator=lambda d: QualityDimensions.model_validate(d),
+        feedback=_section_feedback(state, "quality"),
+    )
+    print(f"  [quality] done in {dt:.1f}s via {served}")
+    return {"quality": result,
+            "node_meta": {"quality": {"duration": round(dt, 1),
+                                      "served_by": served,
+                                      "runs": _runs(state, "quality")}}}
+
+
+def escalation_node(state):
+    """Assess escalation risk: red flags, emotion, risk level."""
+    result, dt, served = _llm_eval_with_retry(
+        system=ESCALATION_SYSTEM,
+        skeleton=ESCALATION_SKELETON,
+        packet=state["packet"],
+        validator=lambda d: EscalationRisk.model_validate(d),
+        feedback=_section_feedback(state, "escalation"),
+    )
+    print(f"  [escalation] done in {dt:.1f}s via {served}")
+    return {"escalation": result,
+            "node_meta": {"escalation": {"duration": round(dt, 1),
+                                         "served_by": served,
+                                         "runs": _runs(state, "escalation")}}}
+
+
+def investigate_node(state):
+    """
+    Conditional deep-dive (v0.3.0): runs ONLY when the anchored evaluation
+    carries risk_level != "none". Produces a manager-ready incident report
+    and attaches it to the evaluation.
+    """
+    ev = state["evaluation"]
+    esc = ev["escalation"]
+    context = (
+        f"{state['packet']}\n\n"
+        f"FIRST-PASS ESCALATION SCREENING RESULT:\n"
+        f"{json.dumps({k: esc[k] for k in ('red_flags', 'customer_emotion_text', 'risk_level')}, indent=2)}"
+    )
+    report, dt, served = _llm_eval_with_retry(
+        system=INVESTIGATE_SYSTEM,
+        skeleton=INVESTIGATE_SKELETON,
+        packet=context,
+        validator=lambda d: InvestigationReport.model_validate(d),
+    )
+    print(f"  [investigate] done in {dt:.1f}s via {served} "
+          f"(priority: {report.priority})")
+
+    # anchor the report's own evidence quotes too
+    from eval_call_data import anchor_evidence
+    report_dict = report.model_dump()
+    turns_indexed = [{"i": i, **t} for i, t in enumerate(state["turns"])]
+    anchor_evidence(report_dict, turns_indexed, {"total": 0, "anchored": 0})
+
+    ev["investigation"] = report_dict
+    return {"evaluation": ev,
+            "node_meta": {"investigate": {"duration": round(dt, 1),
+                                          "served_by": served,
+                                          "priority": report.priority}}}
+
+
+# ── workflow track (v0.4.0) — three isolated LLM phases in one node ─────────
+
+class _Subject(BaseModel):
+    subject: str
+
+
+class _StepsDraft(BaseModel):
+    expected_steps: List[WorkflowStep]
+
+
+def workflow_node(state):
+    """
+    Subject-derived expected workflow (v0.4.0). Three phases with strict
+    prompt isolation:
+      a. subject   — reads the transcript, states the REQUEST in one sentence
+      b. expected  — sees ONLY domain + subject (never the transcript), lists
+                     the boxes a competent agent should check for such a call
+      c. check     — audits the transcript against that independent checklist
+    The isolation in (b) is the point: expectations are formed before knowing
+    what actually happened, so the audit cannot rationalize the observed call.
+    """
+    domain = state["meta"]["domain"]
+
+    subj, dt_a, served_a = _llm_eval_with_retry(
+        system=SUBJECT_SYSTEM, skeleton=SUBJECT_SKELETON,
+        packet=state["packet"],
+        validator=lambda d: _Subject.model_validate(d))
+    print(f'  [workflow] subject in {dt_a:.1f}s: "{subj.subject}"')
+
+    # phase b: NO transcript — only domain + subject
+    gen_packet = f"DOMAIN: {domain}\nCALL SUBJECT: {subj.subject}"
+    draft, dt_b, served_b = _llm_eval_with_retry(
+        system=WORKFLOW_GEN_SYSTEM, skeleton=WORKFLOW_GEN_SKELETON,
+        packet=gen_packet,
+        validator=lambda d: _StepsDraft.model_validate(d))
+    print(f"  [workflow] {len(draft.expected_steps)} expected steps in {dt_b:.1f}s")
+
+    checklist = json.dumps(
+        {"expected_steps": [{"step": s.step, "rationale": s.rationale}
+                            for s in draft.expected_steps]}, indent=2)
+    check_packet = (f"EXPECTED WORKFLOW (written without seeing this call):\n"
+                    f"{checklist}\n\nTRANSCRIPT:\n{state['packet']}")
+    # completeness is part of validity: a truncated audit (fewer entries than
+    # the draft) would otherwise be silently padded with met=null at the join
+    def _check_validator(d):
+        r = _StepsDraft.model_validate(d)
+        if len(r.expected_steps) != len(draft.expected_steps):
+            raise ValueError(
+                f"audit returned {len(r.expected_steps)} entries for "
+                f"{len(draft.expected_steps)} steps; return one entry per "
+                f"given step, same order")
+        return r
+
+    checked, dt_c, served_c = _llm_eval_with_retry(
+        system=WORKFLOW_CHECK_SYSTEM, skeleton=WORKFLOW_CHECK_SKELETON,
+        packet=check_packet, validator=_check_validator)
+
+    # join by index against the DRAFT's step text — the draft is the contract;
+    # the checker only contributes met/evidence. Missing entries stay unchecked.
+    steps = []
+    for i, ds in enumerate(draft.expected_steps):
+        cs = checked.expected_steps[i] if i < len(checked.expected_steps) else None
+        steps.append(WorkflowStep(
+            step=ds.step, rationale=ds.rationale,
+            met=cs.met if cs else None,
+            evidence=cs.evidence if cs else None))
+
+    # single-item recheck pass: fires ONLY on met=false steps, isolated from
+    # the other steps so it can never perturb something the main pass already
+    # got right. Catches the specific failure mode where the main pass wants
+    # a direct question and misses that the value was established via a
+    # later read-back/confirmation instead.
+    def _recheck_validator(d):
+        if "met" not in d:
+            raise ValueError("missing met")
+        return d
+
+    dt_d = 0.0
+    for s in steps:
+        if s.met is not False:
+            continue
+        recheck_packet = (
+            f"CHECKLIST ITEM: {s.step}\n"
+            f"RATIONALE: {s.rationale}\n"
+            f"FIRST-PASS VERDICT: met=false\n\n"
+            f"TRANSCRIPT:\n{state['packet']}")
+        try:
+            rc, dt_r, _ = _llm_eval_with_retry(
+                system=WORKFLOW_RECHECK_SYSTEM, skeleton=WORKFLOW_RECHECK_SKELETON,
+                packet=recheck_packet, validator=_recheck_validator)
+        except RuntimeError:
+            continue
+        dt_d += dt_r
+        if rc.get("met") is True and rc.get("evidence"):
+            s.met = True
+            s.evidence = Evidence.model_validate(rc["evidence"])
+
+    met = sum(1 for s in steps if s.met is True)
+    missed = sum(1 for s in steps if s.met is False)
+    print(f"  [workflow] audit in {dt_c:.1f}s: {met} met, {missed} missed, "
+          f"{len(steps) - met - missed} n/a of {len(steps)}"
+          + (f"  (+{dt_d:.1f}s recheck)" if dt_d else ""))
+
+    dt = dt_a + dt_b + dt_c + dt_d
+    return {"workflow": CallWorkflow(subject=subj.subject,
+                                     expected_steps=steps).model_dump(),
+            "node_meta": {"workflow": {"duration": round(dt, 1),
+                                       "served_by": served_c,
+                                       "phases": {"subject": round(dt_a, 1),
+                                                  "expected": round(dt_b, 1),
+                                                  "check": round(dt_c, 1)},
+                                       "runs": _runs(state, "workflow")}}}
+
+
+def fuse_node(state):
+    """
+    Deterministic acoustic-text fusion (v0.4.0). Loads the audio sentiment
+    model's output (if any) and fuses it into the three audio-informed
+    quality dimensions + the escalation risk level, attaching provenance.
+    Runs between aggregate and anchor so the fused risk drives routing.
+    """
+    from fusion import load_acoustic, fuse_evaluation
+    t0 = time.time()
+    ev = state["evaluation"]
+    rows = load_acoustic(state["call_id"], state["meta"]["domain"])
+    summary = fuse_evaluation(ev, rows)
+    if summary["acoustic_available"]:
+        print(f"  [fuse] acoustic data found: fused "
+              f"{', '.join(summary['fused_dims']) or 'nothing'}; "
+              f"acoustic risk = {summary['acoustic_risk']}")
+    else:
+        print("  [fuse] no acoustic data for this call -> text-only scores")
+    return {"evaluation": ev,
+            "fusion_disputed": summary["disputed"],
+            "node_meta": {"fuse": {"duration": round(time.time() - t0, 2),
+                                   **summary}}}
+
+
+class _Arbitration(BaseModel):
+    risk_level: Literal["none", "review", "escalate"]
+    rationale: str
+
+
+def arbitrate_node(state):
+    """
+    Conditional arbitration (v0.4.1): runs ONLY when the text tier and the
+    acoustic tier disagree about escalation risk. Deterministic merging is
+    fine when the channels agree; when they disagree, a fixed rule has no
+    basis to prefer one -- an LLM weighs both signals against the transcript
+    (was the issue resolved? does vocal escalation RISE late in the call?).
+    """
+    ev = state["evaluation"]
+    esc = ev["escalation"]
+    h = esc.get("hybrid") or {}
+    traj = (ev.get("_acoustic") or {}).get("trajectory") or []
+    # compact trajectory line: customer escalation score per position bucket
+    esc_series = " ".join(f"{t['p']:.2f}:{t['esc']:.2f}"
+                          for t in traj if t.get("esc") is not None)
+
+    context = (
+        f"{state['packet']}\n\n"
+        f"TEXT ASSESSMENT:\n"
+        f"{json.dumps({k: esc[k] for k in ('red_flags', 'customer_emotion_text', 'risk_level')}, indent=2)}\n\n"
+        f"ACOUSTIC ASSESSMENT (audio model, customer channel):\n"
+        f"  tier: {h.get('acoustic_risk')}\n"
+        f"  mean escalation over final third: {h.get('late_mean_escalation')}\n"
+        f"  peak escalation: {h.get('peak_escalation')}\n"
+        f"  trajectory (position:score, 0=start 1=end): {esc_series or 'n/a'}"
+    )
+
+    verdict, dt, served = _llm_eval_with_retry(
+        system=ARBITRATE_SYSTEM, skeleton=ARBITRATE_SKELETON,
+        packet=context,
+        validator=lambda d: _Arbitration.model_validate(d))
+
+    print(f"  [arbitrate] text={h.get('text_risk')} vs "
+          f"acoustic={h.get('acoustic_risk')} -> {verdict.risk_level} "
+          f"in {dt:.1f}s via {served}")
+
+    esc["risk_level"] = verdict.risk_level
+    esc["hybrid"] = {**h, "method": "llm_arbitration",
+                     "arbitration_rationale": verdict.rationale}
+    return {"evaluation": ev,
+            "node_meta": {"arbitrate": {"duration": round(dt, 1),
+                                        "served_by": served,
+                                        "verdict": verdict.risk_level}}}
+
+
+def chapter_node(state):
+    """Segment the call into coherent chapters (reuses segment.py)."""
+    from segment import segment
+    cc, dt, issues = segment(
+        state["call_id"],
+        provider="router",
+        results_dir=state.get("results_dir", "results_channels"),
+    )
+    if issues:
+        print(f"  [chapters] {len(issues)} invariant issue(s): {issues}")
+    print(f"  [chapters] {cc.n_chapters} chapters in {dt:.1f}s")
+    return {"chapters": cc.model_dump(),
+            "node_meta": {"chapters": {"duration": round(dt, 1),
+                                       "served_by": "router",
+                                       "runs": _runs(state, "chapters")}}}

ml-services/evaluation/paths.py

@@ -0,0 +1,56 @@
+"""Central path configuration for the ml-services pipeline.
+
+Every location the pipeline reads or writes derives from two roots:
+
+  REPO_ROOT  -- the repository checkout, derived from this file's location.
+  DATA_ROOT  -- the data directory; env-overridable (CAPSTONE_DATA_ROOT) so a
+                hosted container can point it at a writable scratch volume.
+                Defaults to {REPO_ROOT}/data, the local dev layout.
+
+This module also loads .env (repo root) into os.environ at import time,
+via setdefault -- real environment variables always win over .env entries.
+Doing the load here (rather than in env_util) guarantees CAPSTONE_* keys in
+.env are honored regardless of module import order; env_util.load_env stays
+as a re-export for existing callers.
+"""
+import os
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+ENV_FILE = Path(os.environ.get("CAPSTONE_ENV_FILE", REPO_ROOT / ".env"))
+
+
+def load_env(path=None):
+    """Load KEY=VALUE lines from a .env file into os.environ (setdefault)."""
+    path = Path(path) if path else ENV_FILE
+    vals = {}
+    if path.exists():
+        for line in path.read_text(encoding="utf-8").splitlines():
+            line = line.strip()
+            if not line or line.startswith("#") or "=" not in line:
+                continue
+            k, v = line.split("=", 1)
+            vals[k.strip()] = v.strip()
+            os.environ.setdefault(k.strip(), v.strip())
+    return vals
+
+
+load_env()
+
+DATA_ROOT = Path(os.environ.get("CAPSTONE_DATA_ROOT", str(REPO_ROOT / "data")))
+
+NA_TESTSET = DATA_ROOT / "na_testset"
+MANIFEST = NA_TESTSET / "manifest.json"
+# Registry for calls ingested at runtime (fresh uploads); merged over MANIFEST
+# by assemble.load_manifest so every downstream consumer sees them.
+RUNTIME_MANIFEST = NA_TESTSET / "manifest_runtime.json"
+
+SENTIMENT_ROOT = DATA_ROOT / "sentiment"
+SENTENCE_SEG_ROOT = DATA_ROOT / "sentence_segments"
+
+EVAL_RESULTS = Path(os.environ.get(
+    "CAPSTONE_EVAL_RESULTS", str(Path(__file__).resolve().parent / "results")))
+
+FRONTEND_ROOT = REPO_ROOT / "frontend"
+FRONTEND_PUBLIC = Path(os.environ.get(
+    "CAPSTONE_FRONTEND_PUBLIC", str(FRONTEND_ROOT / "public")))

ml-services/evaluation/prompts.py

@@ -0,0 +1,477 @@
+"""
+Focused system prompts for the graph-based evaluation pipeline.
+
+Each prompt covers ONE evaluation domain (compliance / quality / escalation)
+so the LLM concentrates on a narrower task with richer signal guidance.
+The monolithic prompt in extract.py is preserved for backward compatibility.
+"""
+from rubric import RUBRIC_VERSION_GRAPH
+
+# ─────────────────────────────────────────────────────────────────────────────
+# COMPLIANCE — 6 binary checks, strict evidence requirement
+# ─────────────────────────────────────────────────────────────────────────────
+
+COMPLIANCE_SYSTEM = f"""\
+You are a call-center compliance auditor. You evaluate ONE customer-service \
+call transcript against 6 mandatory compliance checks and output ONLY a JSON \
+object. Rubric version {RUBRIC_VERSION_GRAPH}.
+
+You are given a ROLE-MAPPED transcript: every line is tagged \
+[AGENT mm:ss] or [CUSTOMER mm:ss]. Channel identity is authoritative.
+
+CHECKS (passed = true / false / null). null ONLY when the item does not \
+apply to this call.
+
+1. name_announced
+   Agent stated their own name at the start of the call.
+
+2. company_announced
+   Agent stated the company or bank name.
+
+3. recording_disclosure
+   Agent said the call may be recorded or monitored.
+   If never stated, passed=false (do NOT assume it happened off-mic).
+   Use null only if clearly not required for this call type.
+
+4. identity_verified
+   Agent verified the customer's identity BEFORE discussing account details.
+   Accepted methods: last 4 digits of account, full account number,
+   customer ID, date of birth, security question.
+   Set identity_method to the method used (e.g. "date of birth").
+
+5. resolution_provided
+   Agent gave a clear resolution or explicit next steps before closing.
+
+6. transfer_next_steps
+   If the call was transferred, agent explained what happens next and
+   where the customer is being transferred to.
+   passed=null if there was no transfer in this call.
+
+RULES:
+1. EVIDENCE: every true or false MUST cite a verbatim quote copied EXACTLY \
+from the transcript, with speaker and timestamp. Never invent or paraphrase.
+2. CONSERVATISM: do not mark true unless the transcript explicitly shows it. \
+If ambiguous, mark false and explain in the note field.
+3. OUTPUT: a single JSON object, no markdown fences, no commentary."""
+
+COMPLIANCE_SKELETON = """\
+Return EXACTLY this JSON shape (fill values, keep all keys):
+{
+  "name_announced":       {"passed": true,  "evidence": {"quote": "...", "speaker": "AGENT", "timestamp": "00:12"}, "note": null},
+  "company_announced":    {"passed": true,  "evidence": {"quote": "...", "speaker": "AGENT", "timestamp": "00:12"}, "note": null},
+  "recording_disclosure": {"passed": false, "evidence": null, "note": "never stated"},
+  "identity_verified":    {"passed": true,  "evidence": {"quote": "...", "speaker": "AGENT", "timestamp": "00:42"}, "note": null},
+  "identity_method": "date of birth",
+  "resolution_provided":  {"passed": true,  "evidence": {"quote": "...", "speaker": "AGENT", "timestamp": "10:07"}, "note": null},
+  "transfer_next_steps":  {"passed": null,  "evidence": null, "note": "no transfer"}
+}"""
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# QUALITY — 6 scored dimensions with enriched signal lists
+# ─────────────────────────────────────────────────────────────────────────────
+
+QUALITY_SYSTEM = f"""\
+You are a call-center quality analyst. You score 6 quality dimensions from \
+a customer-service call transcript and output ONLY a JSON object. \
+Rubric version {RUBRIC_VERSION_GRAPH}.
+
+You are given a ROLE-MAPPED transcript: every line is tagged \
+[AGENT mm:ss] or [CUSTOMER mm:ss]. Channel identity is authoritative.
+
+For each dimension:
+- Score 1-5 from TEXT signals only (1=poor, 5=excellent).
+- List specific signals_present and signals_absent from the rubric below.
+- Cite at least one verbatim evidence quote per dimension.
+- Set requires_audio=true for dimensions that also need acoustic analysis.
+
+DIMENSIONS AND SIGNALS:
+
+1. EFFICIENCY
+   - First-call resolution (FCR) — issue resolved without follow-up?
+   - Appropriate call duration for the complexity of the issue?
+   - No unnecessary holds or transfers?
+   Score 5 if FCR with no wasted time. Score 1 if multiple holds, \
+unnecessary transfers, or issue left unresolved.
+
+2. PROBLEM RESOLUTION
+   - Offered multiple options (A, B, C) when applicable?
+   - Explained pros and cons of each option?
+   - Waited for customer preference before proceeding?
+   - Had a backup plan if first option didn't work?
+   - Respected customer's choice even if agent preferred another?
+   - Solution fits the customer's specific situation?
+   - Explained WHY (not just what)?
+   - Was the customer's actual issue solved by the end?
+   - Offered alternatives if the issue was unsolvable?
+   Score 5 if issue solved with clear options and explanation. \
+Score 1 if issue ignored or dismissed with no alternatives.
+
+3. CLARITY
+   - Asked open-ended diagnostic questions ("When did this start?", \
+"What exactly happened?")?
+   - Checked understanding ("Does that make sense?")?
+   - Asked clarifying questions before jumping to a solution?
+   - Confirmed understanding mid-call?
+   - Provided a recap or summary at the end?
+   Score 5 if thorough discovery and clear recap. \
+Score 1 if agent assumed the problem and skipped confirmation.
+
+4. PROFESSIONALISM
+   TEXT signals (score from these):
+   - Proper grammar and complete sentences throughout?
+   - No slang or informal language?
+   - Knew the answer or actively found it?
+   - Provided accurate information?
+   - Explained WHY, not just what to do?
+   - Listened fully without interrupting?
+   - Willing to repeat or rephrase when asked?
+   AUDIO signals (set requires_audio=true):
+   - Clear pace, confident voice, consistent tone — cannot judge from text.
+   Score from text signals only. Set requires_audio=true.
+
+5. EMPATHY
+   HIGH-empathy signals (text):
+   - Validation phrases: "I totally understand your frustration", \
+"That must be stressful", "I'm sorry you dealt with this", \
+"I hear you", "I want to help you"
+   - Uses customer's name naturally
+   - Pauses to let customer finish speaking (no interruptions)
+   - Acknowledges the concern before moving to solutions
+   - Matches the emotional register of the customer
+   LOW-empathy signals (text):
+   - Dismissive: "That's not my problem", "It's not that big a deal", \
+"Yeah, rates go up. Nothing I can do."
+   - Interrupts the customer constantly
+   - Transfers without explaining why
+   - No validation or regret expressed
+   - Flat, scripted responses with no personal engagement
+   AUDIO signals (set requires_audio=true):
+   - Flat/robotic tone, energy/interest, warmth — cannot judge from text.
+   Score from text signals only. Set requires_audio=true.
+
+6. CUSTOMER SATISFACTION
+   - Problem was identified clearly?
+   - Issue was actually addressed (not deflected)?
+   - Clear resolution was provided?
+   - Agent didn't dismiss or minimize the customer's concern?
+   - Agent followed through on promises made during the call?
+   - No escalation threat from the customer?
+   - First-call resolution achieved?
+   - Customer didn't mention switching to a competitor?
+   AUDIO signals (set requires_audio=true):
+   - Customer sounds satisfied at end, not frustrated — cannot judge from text.
+   Score from text signals only. Set requires_audio=true.
+
+MODALITY RULE: Judge ONLY from text. For acoustic qualities (tone, pace, \
+energy, whether the customer "sounds" satisfied) do not guess — set \
+requires_audio=true and score from text signals alone.
+
+OUTPUT: a single JSON object, no markdown fences, no commentary."""
+
+QUALITY_SKELETON = """\
+Return EXACTLY this JSON shape (fill values, keep all keys):
+{
+  "efficiency":              {"score": 4, "signals_present": ["first-call resolution"], "signals_absent": [], "evidence": [{"quote": "...", "speaker": "AGENT", "timestamp": "08:38"}], "requires_audio": false},
+  "problem_resolution":      {"score": 4, "signals_present": ["explained why"], "signals_absent": ["no explicit options"], "evidence": [{"quote": "...", "speaker": "AGENT", "timestamp": "05:20"}], "requires_audio": false},
+  "clarity":                 {"score": 3, "signals_present": ["open-ended questions"], "signals_absent": ["no recap at end"], "evidence": [{"quote": "...", "speaker": "AGENT", "timestamp": "02:15"}], "requires_audio": false},
+  "professionalism":         {"score": 4, "signals_present": ["complete sentences", "accurate information"], "signals_absent": [], "evidence": [{"quote": "...", "speaker": "AGENT", "timestamp": "01:30"}], "requires_audio": true},
+  "empathy":                 {"score": 3, "signals_present": ["uses customer name"], "signals_absent": ["no validation phrase"], "evidence": [{"quote": "...", "speaker": "AGENT", "timestamp": "03:45"}], "requires_audio": true},
+  "customer_satisfaction":   {"score": 4, "signals_present": ["issue addressed", "clear resolution"], "signals_absent": [], "evidence": [{"quote": "...", "speaker": "CUSTOMER", "timestamp": "09:50"}], "requires_audio": true}
+}"""
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# ESCALATION — red flags, customer emotion, risk level
+# ─────────────────────────────────────────────────────────────────────────────
+
+ESCALATION_SYSTEM = f"""\
+You assess escalation risk in a customer-service call transcript and output \
+ONLY a JSON object. Rubric version {RUBRIC_VERSION_GRAPH}.
+
+You are given a ROLE-MAPPED transcript: every line is tagged \
+[AGENT mm:ss] or [CUSTOMER mm:ss]. Channel identity is authoritative.
+
+RED FLAGS (include ALL that apply from this list):
+- manager_requested: customer explicitly asks for a manager or supervisor.
+  Example: "I want to speak to a manager", "Can I talk to your supervisor?"
+- competitor_switch: customer threatens to leave for a competitor.
+  Example: "I'm switching to [other company]", "I'll take my business elsewhere"
+- repeat_attempts: customer indicates this is not their first call about the issue.
+  Example: "I've been calling a couple of times", "I already called about this"
+- issue_too_complex: agent is visibly struggling — keeps putting the customer \
+on hold, cannot find an answer, escalates internally, or admits they don't know.
+- explicit_dissatisfaction: customer makes strong negative statements beyond \
+normal frustration. Example: "This is unacceptable", "I've never had such \
+terrible service"
+
+CUSTOMER EMOTION (infer from WORDS only, not tone):
+  calm — no negative language, cooperative
+  mild_frustration — minor complaints but still engaged
+  frustrated — repeated complaints, impatience, but no threats
+  angry — hostile language, demands, threats
+  distressed — emotional distress, desperation, vulnerability
+
+RISK LEVEL:
+  escalate — immediate escalation needed. Triggers: manager_requested, \
+angry or distressed emotion, or 2+ red flags present.
+  review — a manager should review this call. Triggers: 1 red flag, \
+or frustrated emotion with unresolved issue.
+  none — no escalation needed. Customer is calm or mildly frustrated \
+and the issue was handled.
+
+RULES:
+1. EVIDENCE: cite a verbatim quote for every red flag detected. \
+Never invent quotes.
+2. Set requires_audio=true — emotion intensity needs acoustic confirmation \
+before driving real action.
+3. OUTPUT: a single JSON object, no markdown fences, no commentary."""
+
+ESCALATION_SKELETON = """\
+Return EXACTLY this JSON shape (fill values, keep all keys):
+{
+  "red_flags": [],
+  "customer_emotion_text": "calm",
+  "risk_level": "none",
+  "evidence": [{"quote": "...", "speaker": "CUSTOMER", "timestamp": "06:30"}],
+  "requires_audio": true
+}"""
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# INVESTIGATION — conditional deep-dive, runs only when risk_level != "none"
+# ─────────────────────────────────────────────────────────────────────────────
+
+INVESTIGATE_SYSTEM = f"""\
+You are an escalation review analyst. A first-pass screening flagged this \
+customer-service call as an escalation risk. Your job is to produce a concise \
+incident report a manager can act on, and output ONLY a JSON object. \
+Rubric version {RUBRIC_VERSION_GRAPH}.
+
+You are given the ROLE-MAPPED transcript plus the screening result \
+(red flags, customer emotion, risk level).
+
+REPORT REQUIREMENTS:
+1. summary — 2-3 sentences: what happened, why it is a risk, current state \
+at call end. Written for a manager who has NOT heard the call.
+2. contributing_factors — the specific agent behaviours and customer \
+circumstances that led here. Name both sides where applicable.
+3. recommended_action — ONE concrete next step (e.g. "callback within 24h \
+with fee reversal authority", "coach agent on hold etiquette").
+4. priority — high (escalate risk, angry/distressed customer), \
+medium (review risk with unresolved issue), low (review risk, issue resolved).
+
+RULES:
+1. EVIDENCE: cite the verbatim quotes that justify the report. \
+Never invent quotes.
+2. Be specific. "Improve communication" is useless; \
+"agent left customer on hold 3 times without time estimates" is useful.
+3. OUTPUT: a single JSON object, no markdown fences, no commentary."""
+
+INVESTIGATE_SKELETON = """\
+Return EXACTLY this JSON shape (fill values, keep all keys):
+{
+  "summary": "...",
+  "contributing_factors": ["...", "..."],
+  "recommended_action": "...",
+  "priority": "medium",
+  "evidence": [{"quote": "...", "speaker": "CUSTOMER", "timestamp": "06:30"}]
+}"""
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# WORKFLOW (v0.4.0) — three ISOLATED phases. Phase b never sees the
+# transcript, so the expected checklist cannot be shaped by what actually
+# happened on the call.
+# ─────────────────────────────────────────────────────────────────────────────
+
+SUBJECT_SYSTEM = """\
+You are a call-intake classifier. Read the transcript and state the SUBJECT \
+of the call: ONE sentence (max 20 words) describing what the customer \
+contacted about and what they wanted.
+
+RULES:
+1. State the REQUEST only. Do NOT mention how the call went, whether it was \
+resolved, or anything the agent did. A reader must not be able to tell the \
+outcome from your sentence.
+2. NO call-specific figures: no dollar amounts, dates, account numbers, or \
+personal names. Say "set up a recurring transfer between accounts", never \
+"transfer $400 on the 15th". Your sentence seeds an expected checklist -- \
+leaked specifics would make its boxes trivially satisfiable.
+3. Be concrete about the TYPE of request: "set up an automatic monthly \
+transfer between checking and savings" beats "a banking question".
+4. OUTPUT: a single JSON object, no markdown fences, no commentary."""
+
+SUBJECT_SKELETON = """\
+Return EXACTLY this JSON shape:
+{
+  "subject": "..."
+}"""
+
+
+WORKFLOW_GEN_SYSTEM = """\
+You are a contact-center QA workflow designer. You are given ONLY the \
+call domain and a one-sentence subject. You have NOT seen the transcript \
+and must not assume anything about how the call actually went.
+
+List the steps a competent agent would be EXPECTED to perform on a call \
+with this subject: 4-8 steps, in the order they should occur. Mix the \
+universal procedure (greeting, identity verification, recap) with steps \
+SPECIFIC to this subject (e.g. for a transfer setup: confirm amount, \
+confirm date, state cancellation terms).
+
+RULES:
+1. Each step is a short imperative phrase plus a one-line rationale.
+2. Steps must be OBSERVABLE in a transcript (no "be friendly" — instead \
+"greet the customer and offer help").
+3. OUTPUT: a single JSON object, no markdown fences, no commentary."""
+
+WORKFLOW_GEN_SKELETON = """\
+Return EXACTLY this JSON shape:
+{
+  "expected_steps": [
+    {"step": "...", "rationale": "..."},
+    {"step": "...", "rationale": "..."}
+  ]
+}"""
+
+
+WORKFLOW_CHECK_SYSTEM = """\
+You are a call QA auditor. You are given an expected-workflow checklist \
+(written by someone who knew only the call's subject, not its content) and \
+the ROLE-MAPPED transcript. Audit the call against the checklist.
+
+For EACH step, in the same order, decide:
+  met=true  — the agent performed it (cite a verbatim quote as evidence)
+  met=false — it should have happened and did not
+  met=null  — cannot be determined from text, or genuinely not applicable
+
+RULES:
+1. Keep each "step" string EXACTLY as given. Do not add, remove, or reorder \
+steps.
+2. EVIDENCE: quotes must be copied verbatim from a single transcript turn. \
+Never invent or paraphrase quotes. Omit evidence when met is false or null.
+3. Judge conservatively toward true: partial or implied performance without \
+clear text evidence is met=null, not met=true.
+4. Do NOT hide behind null. You have the FULL transcript: if a step plainly \
+never happened anywhere in it, that is met=false -- a determination, not an \
+unknown. Reserve met=null for steps the text genuinely cannot decide \
+(audio-only behaviour, transcript cut off mid-call) or steps made \
+inapplicable by how the call unfolded. An audit that returns mostly null has \
+failed at its job.
+5. A step listing multiple items with "or" / "any" (e.g. "explain any \
+fees, minimum balance requirements, or transfer limits") is disjunctive: \
+met=true if AT LEAST ONE listed item was substantively covered: cite that \
+item's quote as evidence. met=false only if NONE of the listed items were \
+covered anywhere in the call.
+6. OUTPUT: a single JSON object, no markdown fences, no commentary."""
+
+WORKFLOW_CHECK_SKELETON = """\
+Return EXACTLY this JSON shape (one entry per given step, same order):
+{
+  "expected_steps": [
+    {"step": "...", "rationale": "...", "met": true, "evidence": {"quote": "...", "speaker": "AGENT", "timestamp": "00:15"}},
+    {"step": "...", "rationale": "...", "met": false, "evidence": null}
+  ]
+}"""
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# WORKFLOW RECHECK (v0.4.2) — single-item second pass, isolated per step.
+# Fires ONLY on steps the main check marked met=false. Because it never sees
+# the other 7 steps, it cannot perturb anything the main pass already got
+# right -- unlike editing WORKFLOW_CHECK_SYSTEM itself, whose added rules
+# were shown (by repeated A/B testing) to bleed into unrelated steps' verdicts
+# even when the added rule's content had nothing to do with them.
+# ─────────────────────────────────────────────────────────────────────────────
+
+WORKFLOW_RECHECK_SYSTEM = """\
+You are a call QA auditor doing a SECOND-PASS check on ONE checklist item \
+that a first-pass audit marked as missed (met=false). Your only job is to \
+double-check that single verdict against the full transcript -- you are not \
+scoring any other item.
+
+A first-pass auditor sometimes marks a step false because it looked for a \
+DIRECT QUESTION and didn't find one, while missing that the same information \
+was actually established another way later in the call (e.g. the agent \
+read back or confirmed the specific value instead of asking for it \
+up front). Your job is to catch that specific failure mode -- and only that \
+failure mode. Do not relitigate the whole call.
+
+Decide:
+  met=true  — the step's specific requirement is actually satisfied \
+somewhere in the transcript, even if not phrased as a direct question \
+(a read-back, recap, or confirmation stating the concrete value counts).
+  met=false — confirm the original verdict: it was never satisfied.
+
+EVIDENCE RULES (strict -- this is what most often goes wrong on second pass):
+- The quote must be copied verbatim from a single transcript turn.
+- The quote must contain the ACTUAL, SPECIFIC content the step asks for \
+(e.g. if the step asks for account numbers, the quote must contain the \
+literal numbers -- not just the words "checking account" or "savings \
+account" with no number attached). A quote that is merely on-topic but \
+lacks the concrete value does NOT satisfy the step: keep met=false.
+- If you cannot find a quote meeting that bar, you must return met=false \
+with evidence=null. Do not lower the bar to find something to cite.
+- The quote must serve the exact purpose stated in the step's RATIONALE, \
+not merely share similar words with it. In particular: account numbers or \
+other details exchanged to CARRY OUT a transaction (e.g. to set up a \
+transfer) are not, by themselves, evidence that the customer's IDENTITY \
+was verified -- and vice versa. If the quote's real purpose in context was \
+a different checklist item, it does not count here: met=false.
+
+OUTPUT: a single JSON object, no markdown fences, no commentary."""
+
+WORKFLOW_RECHECK_SKELETON = """\
+Return EXACTLY this JSON shape:
+{
+  "met": true,
+  "evidence": {"quote": "...", "speaker": "AGENT", "timestamp": "00:15"}
+}
+or if still unmet:
+{
+  "met": false,
+  "evidence": null
+}"""
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# ARBITRATION (v0.4.1) — conditional node, runs ONLY when the text LLM's
+# escalation tier and the audio model's acoustic tier disagree. Batch evidence
+# (22 calls): when either channel flags risk, they almost never agree (1/10) —
+# so disagreement cannot be resolved by a fixed rule; it needs the context.
+# ─────────────────────────────────────────────────────────────────────────────
+
+ARBITRATE_SYSTEM = """\
+You are an escalation arbitrator. Two independent assessments of the same \
+customer-service call DISAGREE about its escalation risk:
+  - a TEXT assessment (an LLM reading the transcript: red flags, wording, \
+outcome)
+  - an ACOUSTIC assessment (an audio model scoring the customer's vocal \
+escalation per sentence: it hears tone, not words)
+
+You are given both assessments and the transcript. Decide the final \
+risk_level: none, review, or escalate.
+
+HOW TO WEIGH THE CHANNELS:
+1. Neither channel outranks the other by default. Text knows WHAT was said \
+and whether the issue was resolved; audio knows HOW the customer sounded, \
+which words can mask (polite phrasing over rising agitation, or animated \
+speech that reads angry on paper but is merely energetic).
+2. A high acoustic signal with a genuinely resolved, thanked-and-closed call \
+usually means vocal energy, not risk. A calm transcript with a high LATE \
+acoustic signal (end of call) deserves suspicion: the customer may have \
+given up rather than calmed down.
+3. Anchor your decision in the trajectory: escalation that RISES toward the \
+end of the call matters more than an isolated mid-call spike.
+4. rationale: 1-2 sentences a manager can read, naming which channel you \
+sided with and the deciding observation.
+
+OUTPUT: a single JSON object, no markdown fences, no commentary."""
+
+ARBITRATE_SKELETON = """\
+Return EXACTLY this JSON shape:
+{
+  "risk_level": "review",
+  "rationale": "..."
+}"""

ml-services/evaluation/router.py

@@ -0,0 +1,143 @@
+"""
+LiteLLM-based priority router for QA extraction.
+
+Routing policy is DERIVED FROM benchmark.py results, not guessed:
+  - primary  = mistral/mistral-small-latest
+       only model that DISCRIMINATED scores between calls ([5,4,3,5,2] vs
+       [4,4,3,4,2]); high evidence fidelity; conservative.
+  - fallback = sambanova/Meta-Llama-3.3-70B-Instruct
+       fastest (2.6s), best raw fidelity (75%), perfectly conservative.
+  - safety   = github/gpt-4o-mini
+       always-on free baseline, reliable JSON.
+
+DeepSeek-V3.1 was EXCLUDED: 50% evidence fidelity = fabricates quotes, the worst
+failure mode for a compliance system.
+
+LiteLLM Router gives us retries, fallback chaining, and per-deployment cooldowns
+for free. On a 429/timeout/error the call transparently drops to the next tier.
+
+Public API mirrors llm_client.chat_json so extract.py can swap to routing with a
+one-line change:  chat_json_routed(system, user) -> json string
+"""
+import os, logging
+from env_util import load_env
+load_env()
+
+import litellm
+litellm.suppress_debug_info = True
+litellm.set_verbose = False
+logging.getLogger("LiteLLM").setLevel(logging.ERROR)
+logging.getLogger("litellm").setLevel(logging.ERROR)
+
+from litellm import Router
+from llm_client import JSON_ENFORCE_SUFFIX, _extract_json
+
+
+# ── Routing policy (priority order) ───────────────────────────────────────────
+def _model_list():
+    return [
+        {"model_name": "qa-primary",
+         "litellm_params": {"model": "mistral/mistral-small-latest",
+                            "api_key": os.environ.get("MISTRAL_API_KEY")}},
+        {"model_name": "qa-fallback",
+         "litellm_params": {"model": "sambanova/Meta-Llama-3.3-70B-Instruct",
+                            "api_key": os.environ.get("SAMBANOVA_API_KEY")}},
+        {"model_name": "qa-safety",
+         "litellm_params": {"model": "github/gpt-4o-mini",
+                            "api_key": os.environ.get("GITHUB_TOKEN")}},
+    ]
+
+# strict priority: primary -> fallback -> safety
+FALLBACKS = [{"qa-primary": ["qa-fallback", "qa-safety"]}]
+
+_router = None
+
+
+def get_router():
+    global _router
+    if _router is None:
+        _router = Router(
+            model_list=_model_list(),
+            fallbacks=FALLBACKS,
+            num_retries=2,          # retries on the same tier before falling back
+            timeout=90,             # per-request timeout (s)
+            cooldown_time=60,       # park a failing deployment for 60s
+            retry_after=2,
+        )
+    return _router
+
+
+def chat_json_routed(system, user, temperature=0.1, max_tokens=4000,
+                     return_meta=False, tier="qa-primary"):
+    """
+    Routed JSON completion: tries `tier`, transparently falls back on API
+    failure. Returns the JSON string (extracted from any markdown fences). With
+    return_meta=True, returns (json_str, served_model_string).
+
+    `tier` selects the starting deployment (qa-primary/qa-fallback/qa-safety).
+    Callers escalate deliberately when a 200-OK response is unparseable JSON --
+    a model-quality failure the transport-level router never sees.
+    """
+    router = get_router()
+    messages = [
+        {"role": "system", "content": system + JSON_ENFORCE_SUFFIX},
+        {"role": "user",   "content": user},
+    ]
+    resp = router.completion(
+        model=tier,
+        messages=messages,
+        temperature=temperature,
+        max_tokens=max_tokens,
+    )
+    raw = resp.choices[0].message.content or ""
+    out = _extract_json(raw)
+    if return_meta:
+        return out, getattr(resp, "model", "?")
+    return out
+
+
+# ── Self-test ─────────────────────────────────────────────────────────────────
+if __name__ == "__main__":
+    import json, time, argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--test-fallback", action="store_true",
+                    help="break the primary to prove fallback fires")
+    args = ap.parse_args()
+
+    SYS = "You are a helpful assistant."
+    USR = 'Return JSON: {"status": "ok", "n": 42}'
+
+    if not args.test_fallback:
+        print("\n[1] Normal routed call (should serve from PRIMARY = mistral):")
+        t0 = time.time()
+        out, served = chat_json_routed(SYS, USR, return_meta=True)
+        print(f"    served by : {served}")
+        print(f"    latency   : {int((time.time()-t0)*1000)}ms")
+        print(f"    parsed    : {json.loads(out)}")
+    else:
+        print("\n[2] Fallback test: primary deliberately broken ->")
+        broken = Router(
+            model_list=[
+                {"model_name": "qa-primary",
+                 "litellm_params": {"model": "mistral/mistral-small-latest",
+                                    "api_key": "sk-DELIBERATELY-BROKEN"}},
+                {"model_name": "qa-fallback",
+                 "litellm_params": {"model": "sambanova/Meta-Llama-3.3-70B-Instruct",
+                                    "api_key": os.environ.get("SAMBANOVA_API_KEY")}},
+                {"model_name": "qa-safety",
+                 "litellm_params": {"model": "github/gpt-4o-mini",
+                                    "api_key": os.environ.get("GITHUB_TOKEN")}},
+            ],
+            fallbacks=[{"qa-primary": ["qa-fallback", "qa-safety"]}],
+            num_retries=1, timeout=90,
+        )
+        t0 = time.time()
+        resp = broken.completion(
+            model="qa-primary",
+            messages=[{"role": "system", "content": SYS + JSON_ENFORCE_SUFFIX},
+                      {"role": "user", "content": USR}],
+            max_tokens=30)
+        served = getattr(resp, "model", "?")
+        print(f"    primary failed -> served by FALLBACK: {served}")
+        print(f"    latency   : {int((time.time()-t0)*1000)}ms")
+        print(f"    parsed    : {json.loads(_extract_json(resp.choices[0].message.content))}")

ml-services/evaluation/rubric.py

@@ -0,0 +1,265 @@
+"""
+Formal, versioned translation of the domain expert's QA rubric into a
+machine-readable schema (Pydantic v2). This is the OUTPUT CONTRACT the LLM
+evaluation layer must satisfy.
+
+Design principle -- MODALITY SPLIT
+----------------------------------
+The expert's rubric mixes two kinds of judgment:
+  * TEXT-derivable   : can be decided from the transcript alone.
+  * AUDIO/paralinguistic: tone, pace, energy, "sounds satisfied" -- these
+                          require the acoustic model (WP3) and must NOT be
+                          guessed from text. They are represented here but
+                          flagged `requires_audio=True` and left null by the
+                          text LLM, to be filled by audio-text fusion (WBS 3.4).
+
+Each leaf judgment carries EVIDENCE (a verbatim quote + speaker + timestamp) so
+every score is provable -- no ungrounded compliance passes. This directly
+supports the "Verbatim Evidence Extractor" track later.
+
+RUBRIC_VERSION is bumped whenever criteria change so labelled data stays traceable.
+"""
+from __future__ import annotations
+from typing import Optional, List, Literal
+from pydantic import BaseModel, Field
+
+RUBRIC_VERSION = "0.1.0"          # legacy monolithic path (extract.py)
+RUBRIC_VERSION_GRAPH = "0.4.1"    # graph path: 0.2.0 added customer_satisfaction,
+                                  # 0.3.0 added evidence re-anchor loop +
+                                  # escalation investigation node,
+                                  # 0.4.0 added acoustic-text fusion (hybrid
+                                  # scores) + expected-workflow track,
+                                  # 0.4.1 refit fusion thresholds on the
+                                  # 22-call batch + arbitrate node on
+                                  # text/acoustic disagreement
+
+Speaker = Literal["AGENT", "CUSTOMER"]
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Shared evidence object -- makes every judgment provable
+# ─────────────────────────────────────────────────────────────────────────────
+class Evidence(BaseModel):
+    quote: str = Field(description="Verbatim text copied from the transcript (no paraphrasing).")
+    speaker: Speaker
+    timestamp: Optional[str] = Field(default=None, description="mm:ss of the turn, if available.")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# 1. COMPLIANCE  (all TEXT-derivable; binary pass/fail with N/A)
+#    passed = True | False | None(=not applicable, e.g. recording disclosure
+#    "when needed", or transfer steps when there was no transfer)
+# ─────────────────────────────────────────────────────────────────────────────
+class ComplianceItem(BaseModel):
+    passed: Optional[bool] = Field(description="True=met, False=violated, None=not applicable to this call.")
+    evidence: Optional[Evidence] = Field(default=None, description="Required when passed is True or False.")
+    note: Optional[str] = None
+
+
+class ComplianceChecklist(BaseModel):
+    # "Announce their name"
+    name_announced: ComplianceItem
+    # "Announce company name"
+    company_announced: ComplianceItem
+    # "State 'this call may be recorded' (when needed)"  -> None if not needed
+    recording_disclosure: ComplianceItem
+    # "Confirm customer identity before discussing account" (last 4 / acct # / customer ID / DOB)
+    identity_verified: ComplianceItem
+    identity_method: Optional[str] = Field(
+        default=None, description="e.g. 'last 4 digits', 'account number', 'date of birth', 'customer ID'.")
+    # "Provide resolution / next steps before closing"
+    resolution_provided: ComplianceItem
+    # "If call transferred, provide information of next steps" -> None if no transfer
+    transfer_next_steps: ComplianceItem
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# 2. QUALITY  (1-5 each). Each dimension lists the concrete signals the rubric
+#    names, split into text vs audio. The text LLM fills text signals + a score;
+#    `requires_audio` flags dimensions whose FULL judgment also needs WP3.
+# ─────────────────────────────────────────────────────���───────────────────────
+class HybridScore(BaseModel):
+    """Provenance for a fused score (v0.4.0). method='text_only' means the
+    acoustic model contributed nothing (no data / dimension is text-defined)."""
+    text_score: int = Field(ge=1, le=5, description="LLM score from text signals alone.")
+    acoustic_score: Optional[float] = Field(default=None, description="Acoustic subscore mapped to 1-5.")
+    text_weight: float
+    acoustic_weight: float
+    coverage: Optional[float] = Field(
+        default=None, description="Fraction of channel sentences the audio model processed.")
+    channel: Optional[str] = Field(default=None, description="Speaker channel the acoustic signal came from.")
+    method: Literal["weighted_mean", "text_only"] = "text_only"
+
+
+class QualityDimension(BaseModel):
+    score: int = Field(ge=1, le=5, description="1=poor, 5=excellent. Text LLM score; "
+                       "overwritten with the fused score when acoustic data exists (v0.4.0).")
+    signals_present: List[str] = Field(default_factory=list, description="Rubric behaviours observed.")
+    signals_absent: List[str] = Field(default_factory=list, description="Expected behaviours not observed.")
+    evidence: List[Evidence] = Field(default_factory=list)
+    requires_audio: bool = Field(
+        default=False, description="True if a complete judgment also needs acoustic signals (WP3).")
+    hybrid: Optional[HybridScore] = Field(
+        default=None, description="How this score was computed (v0.4.0+). Null in older evaluations.")
+
+
+class QualityDimensions(BaseModel):
+    # a. Efficiency: FCR, appropriate duration, unnecessary holds/transfers (text-derivable;
+    #    duration is metadata)
+    efficiency: QualityDimension
+    # c. Problem Resolution: options A/B/C, pros/cons, explains WHY, respects choice, issue solved
+    problem_resolution: QualityDimension
+    # b. Clarity: open-ended questions, "does that make sense?", confirms understanding, recap
+    clarity: QualityDimension
+    # d. Professionalism: grammar/complete sentences/no slang (TEXT) + pace/confident voice (AUDIO)
+    professionalism: QualityDimension   # requires_audio likely True
+    # e. Empathy: validation phrases + name use (TEXT) + flat/robotic tone, matches emotion (AUDIO)
+    empathy: QualityDimension           # requires_audio likely True
+    # f. Customer Satisfaction: problem addressed, clear resolution, followed through (TEXT)
+    #    + "sounds satisfied", not frustrated at end (AUDIO). Added in v0.2.0.
+    customer_satisfaction: Optional[QualityDimension] = Field(
+        default=None,
+        description="Customer outcome signals. Added in v0.2.0; null in v0.1.0 evaluations."
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# 3. ESCALATION RISK
+#    Red flags are mostly TEXT (phrases). Customer emotion is text-inferred here
+#    but should be CONFIRMED by the audio model before it drives an action.
+# ─────────────────────────────────────────────────────────────────────────────
+RedFlag = Literal[
+    "manager_requested",        # "I want to speak to a manager"
+    "competitor_switch",        # "I'm switching to [competitor]"
+    "repeat_attempts",          # "I've called a couple of times"
+    "issue_too_complex",        # agent struggling / repeated holds
+    "explicit_dissatisfaction", # other strong negative statements
+]
+
+
+class EscalationFusion(BaseModel):
+    """Provenance for the fused risk level.
+    v0.4.0 fused via ordinal max (more severe tier wins, "ordinal_max").
+    v0.4.1: tiers that AGREE merge deterministically ("agreement"); tiers
+    that DISAGREE are resolved by an arbitration LLM node that weighs both
+    signals in context ("llm_arbitration") -- disagreement is exactly where
+    mechanical merging has nothing to stand on (batch evidence: of 10 calls
+    where either channel flagged risk, both flagged on only 1)."""
+    text_risk: Literal["none", "review", "escalate"]
+    acoustic_risk: Optional[Literal["none", "review", "escalate"]] = None
+    late_mean_escalation: Optional[float] = Field(
+        default=None, description="Mean customer escalation_score over the final third of the call.")
+    peak_escalation: Optional[float] = Field(
+        default=None, description="Max customer escalation_score anywhere in the call. Context only "
+                                  "since v0.4.1 -- not a tier trigger (batch median 0.56 gave it no "
+                                  "discriminating power).")
+    method: Literal["agreement", "llm_arbitration", "ordinal_max", "text_only"] = "text_only"
+    arbitration_rationale: Optional[str] = Field(
+        default=None, description="Arbitrator's reasoning when method='llm_arbitration'.")
+
+
+class EscalationRisk(BaseModel):
+    red_flags: List[RedFlag] = Field(default_factory=list)
+    customer_emotion_text: Literal["calm", "mild_frustration", "frustrated", "angry", "distressed"] = Field(
+        description="Emotion inferred from TEXT only; audio model confirms before action.")
+    risk_level: Literal["none", "review", "escalate"] = Field(
+        description="none / review (manager may review) / escalate (immediate). "
+                    "Fused with the acoustic tier when audio data exists (v0.4.0).")
+    evidence: List[Evidence] = Field(default_factory=list)
+    requires_audio: bool = Field(
+        default=True, description="Emotion intensity needs acoustic confirmation.")
+    hybrid: Optional[EscalationFusion] = Field(
+        default=None, description="How risk_level was computed (v0.4.0+). Null in older evaluations.")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# 4. INVESTIGATION (v0.3.0) -- produced ONLY when risk_level != "none".
+#    A conditional deep-dive that turns escalation red flags into a
+#    manager-ready incident report.
+# ─────────────────────────────────────────────────────────────────────────────
+class InvestigationReport(BaseModel):
+    summary: str = Field(description="2-3 sentence incident summary for a manager.")
+    contributing_factors: List[str] = Field(
+        default_factory=list, description="What led to the escalation risk (agent + customer side).")
+    recommended_action: str = Field(description="Concrete next step for the reviewing manager.")
+    priority: Literal["low", "medium", "high"] = Field(
+        description="Review urgency. escalate risk -> high, review risk -> low/medium.")
+    evidence: List[Evidence] = Field(default_factory=list)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# 5. EXPECTED WORKFLOW (v0.4.0) -- three isolated LLM phases:
+#    a. subject: one sentence stating what the customer contacted about
+#       (the REQUEST only, never the outcome).
+#    b. expected steps: generated from domain + subject ONLY -- the model has
+#       not seen the transcript, so expectations cannot be contaminated by
+#       what actually happened on the call.
+#    c. check: audits the transcript against that independent checklist.
+# ─────────────────────────────────────────────────────────────────────────────
+class WorkflowStep(BaseModel):
+    step: str = Field(description="Short imperative phrase, e.g. 'Verify customer identity'.")
+    rationale: Optional[str] = Field(default=None, description="Why this step is expected for this call type.")
+    met: Optional[bool] = Field(
+        default=None, description="True=performed, False=missed, None=not determinable/applicable. "
+                                  "Filled by the check phase; null before checking.")
+    evidence: Optional[Evidence] = None
+
+
+class CallWorkflow(BaseModel):
+    subject: str = Field(description="One-sentence gist of what the customer wanted (request, not outcome).")
+    expected_steps: List[WorkflowStep] = Field(
+        description="4-8 boxes a competent agent should check for a call with this subject.")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Top-level evaluation payload
+# ─────────────────────────────────────────────────────────────────────────────
+class CallMetadata(BaseModel):
+    call_id: str
+    domain: str                       # injected as "Call Type" lens
+    accent: Optional[str] = None
+    duration_seconds: Optional[float] = None
+    transcript_model: Optional[str] = None
+
+
+class CallEvaluation(BaseModel):
+    rubric_version: str = RUBRIC_VERSION
+    metadata: CallMetadata
+    compliance: ComplianceChecklist
+    quality: QualityDimensions
+    escalation: EscalationRisk
+    investigation: Optional[InvestigationReport] = Field(
+        default=None,
+        description="Deep-dive incident report. Only populated when risk_level != 'none' (v0.3.0+).")
+    workflow: Optional[CallWorkflow] = Field(
+        default=None,
+        description="Subject-derived expected workflow + audit results (v0.4.0+).")
+    overall_summary: Optional[str] = Field(
+        default=None, description="2-3 sentence plain-language summary for the dashboard.")
+
+
+# Documentation map: which rubric areas are text vs audio vs metadata.
+# Used by the prompt builder (Phase 1) and the report.
+MODALITY_MAP = {
+    "compliance.name_announced":        "text",
+    "compliance.company_announced":     "text",
+    "compliance.recording_disclosure":  "text",
+    "compliance.identity_verified":     "text",
+    "compliance.resolution_provided":   "text",
+    "compliance.transfer_next_steps":   "text",
+    "quality.efficiency":               "text+metadata",  # duration from metadata
+    "quality.problem_resolution":       "text",
+    "quality.clarity":                  "text",
+    "quality.professionalism":          "text+audio",     # grammar=text, tone/pace=audio
+    "quality.empathy":                  "text+audio",     # phrases=text, warmth/tone=audio
+    "escalation.red_flags":             "text",
+    "escalation.customer_emotion":      "text+audio",      # words=text, intensity=audio
+    "quality.customer_satisfaction":    "text+audio",      # outcome=text, "sounds satisfied"=audio
+}
+
+
+if __name__ == "__main__":
+    # emit the JSON schema so we can eyeball the contract
+    import json
+    print(f"RUBRIC_VERSION = {RUBRIC_VERSION}\n")
+    print(json.dumps(CallEvaluation.model_json_schema(), indent=2)[:1500], "...")

ml-services/evaluation/run_graph_batch.py

@@ -0,0 +1,143 @@
+"""
+Batch-run the graph evaluation over every call that has audio sentiment data.
+
+Purpose (v0.4.0 validation):
+  1. Ground the fusion constants — collect text-vs-acoustic score pairs and
+     escalation trajectories across calls so thresholds can be fitted to the
+     observed distributions instead of asserted.
+  2. Probe the workflow track for leakage/leniency — if expected-step pass
+     rates are ~100% on every call including badly-handled ones, the checklist
+     generation or the audit is compromised.
+
+Usage:  python run_graph_batch.py [--results_dir results] [--only banking]
+Output: results/{call_id}_graph.json per call + results/batch_summary.json
+"""
+import os
+import json
+import time
+import argparse
+import traceback
+
+import paths
+from graph import build_graph
+from rubric import RUBRIC_VERSION_GRAPH
+
+SENTIMENT_ROOT = str(paths.SENTIMENT_ROOT)
+
+
+def calls_with_sentiment(only=None):
+    out = []
+    for domain in sorted(os.listdir(SENTIMENT_ROOT)):
+        if only and domain != only:
+            continue
+        d = os.path.join(SENTIMENT_ROOT, domain)
+        for f in sorted(os.listdir(d)):
+            if f.endswith(".json") and not f.endswith("_segments.json"):
+                out.append(f[:-5])
+    return out
+
+
+def summarize(ev):
+    """Extract the batch-level facts we want to analyze."""
+    q = ev.get("quality", {})
+    dims = {}
+    for name, d in q.items():
+        if isinstance(d, dict) and "score" in d:
+            h = d.get("hybrid") or {}
+            dims[name] = {"fused": d["score"],
+                          "text": h.get("text_score", d["score"]),
+                          "acoustic": h.get("acoustic_score"),
+                          "coverage": h.get("coverage")}
+    e = ev.get("escalation", {})
+    eh = e.get("hybrid") or {}
+    wf = ev.get("workflow") or {}
+    steps = wf.get("expected_steps", [])
+    return {
+        "quality": dims,
+        "risk": e.get("risk_level"),
+        "risk_text": eh.get("text_risk"),
+        "risk_acoustic": eh.get("acoustic_risk"),
+        "esc_late_mean": eh.get("late_mean_escalation"),
+        "esc_peak": eh.get("peak_escalation"),
+        "esc_method": eh.get("method"),
+        "emotion": e.get("customer_emotion_text"),
+        "red_flags": e.get("red_flags", []),
+        "wf_subject": wf.get("subject"),
+        "wf_total": len(steps),
+        "wf_met": sum(1 for s in steps if s.get("met") is True),
+        "wf_missed": sum(1 for s in steps if s.get("met") is False),
+        "anchor": ev.get("_anchor_stats"),
+        "investigated": ev.get("investigation") is not None,
+        "wall_clock": (ev.get("_pipeline") or {}).get("wall_clock"),
+    }
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--results_dir", default="results",
+                    help="transcription dir under na_testset (default: results "
+                         "— the flattened backend format covers all 22 calls)")
+    ap.add_argument("--only", default=None, help="restrict to one domain")
+    ap.add_argument("--skip_existing", action="store_true")
+    args = ap.parse_args()
+
+    calls = calls_with_sentiment(args.only)
+    here = os.path.dirname(os.path.abspath(__file__))
+    out_dir = os.path.join(here, "results")
+    os.makedirs(out_dir, exist_ok=True)
+
+    graph = build_graph()
+    summary, failures = {}, {}
+    t_start = time.time()
+
+    print(f"Batch: {len(calls)} calls, rubric {RUBRIC_VERSION_GRAPH}, "
+          f"results_dir={args.results_dir}\n" + "=" * 68)
+
+    for i, call_id in enumerate(calls, 1):
+        out_path = os.path.join(out_dir, f"{call_id}_graph.json")
+        if args.skip_existing and os.path.exists(out_path):
+            with open(out_path, encoding="utf-8") as f:
+                prev = json.load(f)
+            if prev.get("rubric_version") == RUBRIC_VERSION_GRAPH:
+                summary[call_id] = summarize(prev)
+                print(f"[{i:2}/{len(calls)}] {call_id} — cached")
+                continue
+
+        print(f"[{i:2}/{len(calls)}] {call_id}")
+        t0 = time.time()
+        try:
+            final = graph.invoke({"call_id": call_id,
+                                  "results_dir": args.results_dir})
+            ev = final["evaluation"]
+            ev["_pipeline"] = {
+                "nodes": final.get("node_meta", {}),
+                "anchor_passes": final.get("anchor_attempts", 1),
+                "wall_clock": round(time.time() - t0, 1),
+            }
+            with open(out_path, "w", encoding="utf-8") as f:
+                json.dump(ev, f, indent=2)
+            summary[call_id] = summarize(ev)
+            s = summary[call_id]
+            print(f"    ok in {s['wall_clock']}s | risk={s['risk']} "
+                  f"(text={s['risk_text']}, ac={s['risk_acoustic']}) | "
+                  f"wf {s['wf_met']}/{s['wf_total']} met, {s['wf_missed']} missed")
+        except Exception as e:
+            failures[call_id] = f"{type(e).__name__}: {e}"
+            print(f"    FAILED: {failures[call_id]}")
+            traceback.print_exc(limit=2)
+
+    with open(os.path.join(out_dir, "batch_summary.json"), "w",
+              encoding="utf-8") as f:
+        json.dump({"rubric_version": RUBRIC_VERSION_GRAPH,
+                   "results_dir": args.results_dir,
+                   "wall_clock_total": round(time.time() - t_start, 1),
+                   "calls": summary, "failures": failures}, f, indent=2)
+
+    print("=" * 68)
+    print(f"Done: {len(summary)} ok, {len(failures)} failed, "
+          f"{round(time.time() - t_start, 1)}s total")
+    print(f"Summary -> {os.path.join(out_dir, 'batch_summary.json')}")
+
+
+if __name__ == "__main__":
+    main()

ml-services/evaluation/segment.py

@@ -0,0 +1,234 @@
+"""
+Call chaptering / segmentation layer.
+
+Turns the role-mapped transcript into coherent CHAPTERS (phases of the call),
+like automatic video chapters: greeting, identity verification, problem
+statement, investigation, resolution, closing, etc.
+
+DESIGN -- why this is robust
+----------------------------
+A naive approach asks the model for mm:ss boundaries, which it can hallucinate
+(timestamps past the call end, overlapping spans, gaps). Instead we:
+
+  1. NUMBER every turn and show the model the numbered transcript.
+  2. Ask only for the START TURN INDEX of each chapter (+ label + summary).
+  3. DERIVE all timestamps from the real turns, and REPAIR the result in code:
+     - clamp indices into range, dedupe, sort
+     - force chapter 1 to start at turn 0
+     - each chapter ends exactly where the next begins (contiguous, no gaps)
+     - the last chapter ends at the call duration
+
+So the output is always valid by construction -- the model only chooses *where*
+topics shift, never the raw numbers.
+
+Uses the LiteLLM router by default (Mistral primary). Override with --provider.
+
+Usage:
+  python segment.py --call_id en_CA_Banking_1592237
+  python segment.py --call_id en_CA_Banking_1592237 --provider github
+"""
+import os, json, time, argparse
+from typing import List
+from pydantic import BaseModel, Field, ValidationError
+
+import paths
+from env_util import load_env
+from assemble import load_manifest, assemble_turns, estimate_duration, mmss
+from llm_client import chat_json
+from extract import strip_fences
+
+load_env()
+DATA = str(paths.NA_TESTSET)
+
+
+# ── Schema ────────────────────────────────────────────────────────────────────
+class Chapter(BaseModel):
+    index: int
+    label: str
+    start_turn: int
+    start_time: str          # mm:ss (derived)
+    end_time: str            # mm:ss (derived)
+    summary: str
+
+
+class CallChapters(BaseModel):
+    call_id: str
+    domain: str
+    duration: str
+    n_turns: int
+    n_chapters: int
+    served_by: str
+    chapters: List[Chapter]
+
+
+# ── Prompt ────────────────────────────────────────────────────────────────────
+SYSTEM = """You segment a customer-service phone call into coherent CHAPTERS -- the natural phases of the call, like automatic video chapters.
+
+Typical phases (use only those that actually occur, in the order they occur):
+greeting/introduction, identity verification, problem statement, investigation/discussion, options/explanation, resolution & next steps, upsell/offer, closing.
+
+RULES:
+- Identify between 3 and 8 chapters. Fewer for short calls, more for long ones.
+- Each chapter is a run of CONSECUTIVE turns covering ONE phase. No overlaps, no gaps.
+- Base boundaries on REAL topic shifts in the transcript, not fixed sizes.
+- The first chapter MUST start at turn 0.
+- For each chapter output: start_turn (the turn index where the phase BEGINS),
+  a short label (2-4 words), and a one-sentence summary of what happens in it.
+- Output ONLY JSON. No markdown, no commentary."""
+
+SKELETON = """Return EXACTLY this shape:
+{"chapters":[
+  {"start_turn":0,"label":"Greeting & Introduction","summary":"Agent greets the caller and identifies themselves and the company."},
+  {"start_turn":4,"label":"Identity Verification","summary":"Agent verifies the customer's identity before discussing the account."}
+]}"""
+
+
+def numbered_transcript(turns) -> str:
+    lines = []
+    for i, t in enumerate(turns):
+        lines.append(f"[{i}] [{t['speaker']} {mmss(t['start'])}] {t['text']}")
+    return "\n".join(lines)
+
+
+def _call(provider, system, user):
+    """Return (raw_json_str, served_model)."""
+    if provider == "router":
+        from router import chat_json_routed
+        return chat_json_routed(system, user, max_tokens=1500, return_meta=True)
+    raw = chat_json(provider, system, user, max_tokens=1500)
+    return raw, provider
+
+
+def repair_chapters(seeds, turns, duration_str) -> List[Chapter]:
+    """Turn raw model seeds into valid, contiguous, gap-free chapters."""
+    n = len(turns)
+    cleaned, seen = [], set()
+    for s in seeds:
+        try:
+            st = int(s["start_turn"])
+        except (KeyError, ValueError, TypeError):
+            continue
+        st = max(0, min(st, n - 1))
+        if st in seen:
+            continue
+        seen.add(st)
+        cleaned.append({"start_turn": st,
+                        "label": str(s.get("label", "Untitled")).strip(),
+                        "summary": str(s.get("summary", "")).strip()})
+
+    cleaned.sort(key=lambda x: x["start_turn"])
+    if not cleaned:
+        cleaned = [{"start_turn": 0, "label": "Full Call",
+                    "summary": "Entire call."}]
+    # force coverage from the start
+    if cleaned[0]["start_turn"] != 0:
+        cleaned[0]["start_turn"] = 0
+
+    chapters = []
+    for i, c in enumerate(cleaned):
+        st = c["start_turn"]
+        start_time = mmss(turns[st]["start"])
+        if i + 1 < len(cleaned):
+            end_time = mmss(turns[cleaned[i + 1]["start_turn"]]["start"])
+        else:
+            end_time = duration_str
+        chapters.append(Chapter(index=i + 1, label=c["label"], start_turn=st,
+                                start_time=start_time, end_time=end_time,
+                                summary=c["summary"]))
+    return chapters
+
+
+def segment(call_id, provider="router", results_dir="results_channels"):
+    manifest = load_manifest()
+    meta = manifest[call_id]
+    result_path = os.path.join(DATA, results_dir, meta["accent"], call_id + ".json")
+    with open(result_path, encoding="utf-8") as f:
+        result = json.load(f)
+
+    turns = assemble_turns(result)
+    duration_str = mmss(estimate_duration(result))
+    user = f"{numbered_transcript(turns)}\n\n{SKELETON}"
+
+    t0 = time.time()
+    raw, served = _call(provider, SYSTEM, user)
+    dt = time.time() - t0
+
+    data = json.loads(strip_fences(raw))
+    seeds = data.get("chapters", [])
+    chapters = repair_chapters(seeds, turns, duration_str)
+
+    cc = CallChapters(
+        call_id=call_id, domain=result.get("domain", meta.get("domain", "?")),
+        duration=duration_str, n_turns=len(turns), n_chapters=len(chapters),
+        served_by=served, chapters=chapters)
+    issues = verify_invariants(cc, turns)
+    return cc, dt, issues
+
+
+def verify_invariants(cc: CallChapters, turns) -> List[str]:
+    """Return a list of invariant violations (empty = correct)."""
+    issues = []
+    chs = cc.chapters
+    if not chs:
+        return ["no chapters produced"]
+    if chs[0].start_turn != 0:
+        issues.append("chapter 1 does not start at turn 0")
+    if chs[0].start_time != mmss(turns[0]["start"]):
+        issues.append("first chapter start != call start")
+    if chs[-1].end_time != cc.duration:
+        issues.append("last chapter end != call duration (coverage gap)")
+    for a, b in zip(chs, chs[1:]):
+        if a.end_time != b.start_time:
+            issues.append(f"gap/overlap between ch{a.index} and ch{b.index}")
+        if b.start_turn <= a.start_turn:
+            issues.append(f"non-increasing start_turn at ch{b.index}")
+    if not (1 <= cc.n_chapters <= 12):
+        issues.append(f"chapter count {cc.n_chapters} outside sane range")
+    return issues
+
+
+def print_timeline(cc: CallChapters, dt, issues=None):
+    print("\n" + "=" * 70)
+    print(f"  CALL TIMELINE  {cc.call_id}  ({cc.domain})")
+    print(f"  {cc.duration} | {cc.n_turns} turns | {cc.n_chapters} chapters "
+          f"| {dt:.1f}s | {cc.served_by}")
+    print("=" * 70)
+    for ch in cc.chapters:
+        print(f"\n  {ch.start_time}-{ch.end_time}   [{ch.index}] {ch.label}")
+        print(f"                  {ch.summary}")
+    print("\n  " + "-" * 66)
+    if issues is None:
+        print("  invariants: (not checked)")
+    elif issues:
+        print(f"  invariants: {len(issues)} VIOLATION(S):")
+        for x in issues:
+            print(f"     - {x}")
+    else:
+        print("  invariants: OK (contiguous, gap-free, full coverage)")
+    print("=" * 70)
+
+
+def main():
+    from llm_client import list_providers
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--call_id", required=True)
+    ap.add_argument("--provider", default="router",
+                    choices=["router"] + list_providers())
+    ap.add_argument("--results_dir", default="results_channels")
+    args = ap.parse_args()
+
+    cc, dt, issues = segment(args.call_id, args.provider, args.results_dir)
+
+    here = os.path.dirname(os.path.abspath(__file__))
+    out_dir = os.path.join(here, "chapters")
+    os.makedirs(out_dir, exist_ok=True)
+    out = os.path.join(out_dir, f"{args.call_id}.json")
+    with open(out, "w", encoding="utf-8") as f:
+        json.dump(cc.model_dump(), f, indent=2)
+
+    print_timeline(cc, dt, issues)
+    print(f"\nSaved -> {out}")
+
+
+if __name__ == "__main__":
+    main()

ml-services/evaluation/sentence_segments.py

@@ -0,0 +1,165 @@
+"""
+Build sentence-level segments from word-level faster-whisper output.
+
+Sentence boundaries are defined by:
+  1. Sentence-ending punctuation (. ? !) on a word — excluding known abbreviations
+  2. Silence gaps >= GAP_SPLIT_S between consecutive words (speaker turn break)
+
+Fragments shorter than MIN_DURATION_S are merged into the preceding sentence
+(only when the gap between them is small) so wav2vec2 always receives a clip
+long enough to infer emotion.
+
+Input:  frontend/transcript_data.json  (word-level agent + customer arrays)
+Output: frontend/src/sentence_segments.json
+
+Schema per sentence:
+  {
+    "id":       "AGENT_001",      # <SPEAKER>_<zero-padded index>
+    "seq_id":   1,                # global chronological index across both speakers
+    "speaker":  "AGENT",
+    "start":    13.49,
+    "end":      20.43,
+    "duration": 6.94,
+    "text":     "Oh, good afternoon. ..."
+  }
+"""
+import json
+import os
+import re
+
+MIN_DURATION_S = 1.5   # merge fragments shorter than this into the previous sentence
+GAP_SPLIT_S    = 2.0   # silence gap that always forces a new sentence regardless of punctuation
+MAX_MERGE_GAP  = 1.0   # only merge a short fragment if the gap from previous sentence is <= this
+
+TRANSCRIPT = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    "..", "..", "frontend", "transcript_data.json"
+)
+OUTPUT = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    "..", "..", "frontend", "src", "sentence_segments.json"
+)
+
+# abbreviations whose trailing period must NOT trigger a sentence split
+ABBREVIATIONS = {
+    "mr", "mrs", "ms", "dr", "prof", "sr", "jr", "vs", "st",
+    "ave", "blvd", "dept", "est", "approx", "etc", "e.g", "i.e",
+    "jan", "feb", "mar", "apr", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
+}
+
+SENTENCE_END = re.compile(r"[.?!]$")
+
+
+def is_sentence_end(word: str) -> bool:
+    """True if this word ends a sentence (not an abbreviation period)."""
+    if not SENTENCE_END.search(word):
+        return False
+    # strip trailing punctuation to get the base token
+    base = re.sub(r"[.?!,;:]+$", "", word).lower()
+    if base in ABBREVIATIONS:
+        return False
+    # single uppercase letter followed by period → initial, not sentence end
+    if re.match(r"^[a-z]$", base):
+        return False
+    return True
+
+
+def words_to_sentences(words: list[dict], speaker: str) -> list[dict]:
+    """
+    Split words into sentences using punctuation + gap heuristics,
+    then merge short trailing fragments into their predecessor.
+    """
+    if not words:
+        return []
+
+    # ── phase 1: split into raw sentence buckets ─────────────────────────────
+    raw_sentences = []
+    bucket = [words[0]]
+
+    for prev_w, curr_w in zip(words, words[1:]):
+        gap = curr_w["start"] - prev_w["end"]
+        if is_sentence_end(prev_w["word"]) or gap >= GAP_SPLIT_S:
+            raw_sentences.append(bucket)
+            bucket = [curr_w]
+        else:
+            bucket.append(curr_w)
+
+    if bucket:
+        raw_sentences.append(bucket)
+
+    # ── phase 2: merge short fragments into predecessor ───────────────────────
+    merged = []
+    for sent_words in raw_sentences:
+        start = sent_words[0]["start"]
+        end   = sent_words[-1]["end"]
+        dur   = end - start
+        text  = " ".join(w["word"] for w in sent_words)
+
+        # only merge fragments that lack terminal punctuation (trailing word scraps,
+        # not real sentences that just happen to be short)
+        last_word_has_terminal = SENTENCE_END.search(sent_words[-1]["word"])
+        if merged and dur < MIN_DURATION_S and not last_word_has_terminal:
+            gap_from_prev = start - merged[-1]["end"]
+            if gap_from_prev <= MAX_MERGE_GAP:
+                prev = merged[-1]
+                prev["text"]     = prev["text"] + " " + text
+                prev["end"]      = end
+                prev["duration"] = round(prev["end"] - prev["start"], 3)
+                continue
+
+        merged.append({
+            "speaker":  speaker,
+            "start":    round(start, 3),
+            "end":      round(end, 3),
+            "duration": round(dur, 3),
+            "text":     text,
+        })
+
+    for idx, s in enumerate(merged):
+        s["id"] = f"{speaker}_{idx + 1:03d}"
+
+    return merged
+
+
+def interleave(agent_sents: list[dict], customer_sents: list[dict]) -> list[dict]:
+    """Merge both speaker lists sorted by start time, add global seq_id."""
+    combined = sorted(agent_sents + customer_sents, key=lambda s: s["start"])
+    for i, s in enumerate(combined):
+        s["seq_id"] = i + 1
+    return combined
+
+
+def main():
+    with open(TRANSCRIPT, encoding="utf-8") as f:
+        data = json.load(f)
+
+    agent_sents    = words_to_sentences(data["agent"],    "AGENT")
+    customer_sents = words_to_sentences(data["customer"], "CUSTOMER")
+    all_sents      = interleave(agent_sents, customer_sents)
+
+    output = {
+        "call":               data.get("call", "call_1"),
+        "model":              data.get("model"),
+        "total":              len(all_sents),
+        "agent_sentences":    len(agent_sents),
+        "customer_sentences": len(customer_sents),
+        "sentences":          all_sents,
+    }
+
+    os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
+    with open(OUTPUT, "w", encoding="utf-8") as f:
+        json.dump(output, f, indent=2)
+
+    print(f"call_1  |  {len(agent_sents)} agent + {len(customer_sents)} customer"
+          f" = {len(all_sents)} sentences total")
+    print(f"saved -> {os.path.abspath(OUTPUT)}")
+
+    print("\nfirst 8 sentences:")
+    for s in all_sents[:8]:
+        print(f"  [{s['id']:>12}] seq={s['seq_id']:>3}  "
+              f"{s['start']:>7.2f}s – {s['end']:>7.2f}s  "
+              f"({s['duration']:.2f}s)  \"{s['text'][:65]}\"")
+
+
+if __name__ == "__main__":
+    main()

ml-services/pipeline/acoustic.py

@@ -0,0 +1,139 @@
+"""
+Acoustic sentiment stage (gated).
+
+Runs Clara's wav2vec2 call-center emotion model over each sentence segment and
+emits the SIMPLE per-seq_id schema that `fusion.load_acoustic()` joins against
+the segmentation:
+
+  {call_id, model_version, segments:[{seq_id, sentiment, dominant_emotion,
+                                      escalation_score, processing_status}, ...]}
+
+Fidelity note (correctness-critical): fusion's escalation tiers (late_mean >=
+0.30 review, >= 0.50 escalate) were FITTED on the 22-call batch, which was
+produced by `run_timestamped_sentiment.py` calling `analyze_audio(clip)` PER
+SENTENCE with the default `build_timeline=True`. We reproduce that call exactly
+(default build_timeline) so a fresh call's escalation_score lands on the same
+distribution the thresholds assume. Changing build_timeline or the per-sentence
+granularity would silently bias the tiers.
+
+Weights come from the public HF repo (cached by huggingface_hub), overridable
+with ACOUSTIC_MODEL_DIR. Any unavailability raises AcousticUnavailable; the
+orchestrator catches it and lets the graph fuse text-only.
+"""
+import os
+import tempfile
+from pathlib import Path
+
+DEFAULT_MODEL_REPO = "clarayoussef/wav2vec2-callcenter-emotion-v5"
+MIN_SEG_S = 1.0   # matches run_timestamped_sentiment.py --min-segment-seconds
+
+
+class AcousticUnavailable(RuntimeError):
+    """Model weights, deps, or predictor could not be loaded."""
+
+
+def _enum(v):
+    """Enum -> its string value; pass through plain strings/None."""
+    return v.value if hasattr(v, "value") else v
+
+
+def resolve_model_dir():
+    """Local dir override (ACOUSTIC_MODEL_DIR), else download the hub repo."""
+    local = os.environ.get("ACOUSTIC_MODEL_DIR")
+    if local:
+        if not Path(local).exists():
+            raise AcousticUnavailable(f"ACOUSTIC_MODEL_DIR not found: {local}")
+        return local
+    repo = os.environ.get("ACOUSTIC_MODEL_REPO", DEFAULT_MODEL_REPO)
+    try:
+        from huggingface_hub import snapshot_download
+        # HF_TOKEN in the environment is honored automatically if the repo
+        # is ever made private; the default repo is public.
+        return snapshot_download(repo_id=repo)
+    except Exception as e:
+        raise AcousticUnavailable(f"could not fetch acoustic model {repo}: {e}")
+
+
+_PREDICTOR = None
+
+
+def _get_predictor():
+    """Lazy singleton EmotionPredictor (loads weights once per process)."""
+    global _PREDICTOR
+    if _PREDICTOR is None:
+        try:
+            from src.inference.emotion_predictor import EmotionPredictor
+        except Exception as e:                       # missing torch/transformers/src
+            raise AcousticUnavailable(f"emotion predictor import failed: {e}")
+        try:
+            _PREDICTOR = EmotionPredictor(model_dir=resolve_model_dir())
+        except AcousticUnavailable:
+            raise
+        except Exception as e:
+            raise AcousticUnavailable(f"emotion predictor init failed: {e}")
+    return _PREDICTOR
+
+
+def analyze_call(call_id, segmentation, agent_wav, customer_wav):
+    """
+    Per-sentence acoustic sentiment for one call.
+
+    `segmentation` is the `segment_transcript()` dict; each sentence carries
+    seq_id, speaker (AGENT/CUSTOMER), start, end (seconds). Slices the matching
+    speaker channel per sentence and runs the emotion model. Returns the simple
+    fusion-ready dict described in the module docstring.
+    """
+    import soundfile as sf
+    predictor = _get_predictor()
+
+    # preload each channel once; slice in-memory per sentence
+    channels = {}
+    for speaker, wav in (("AGENT", agent_wav), ("CUSTOMER", customer_wav)):
+        data, sr = sf.read(str(wav), dtype="float32")
+        if data.ndim > 1:
+            data = data.mean(axis=1)
+        channels[speaker] = (data, sr)
+
+    rows = []
+    for s in segmentation["sentences"]:
+        seq_id = s["seq_id"]
+        speaker = s.get("speaker")
+        start, end = float(s["start"]), float(s["end"])
+
+        if end - start < MIN_SEG_S or speaker not in channels:
+            rows.append({"seq_id": seq_id, "processing_status": "skipped_too_short"})
+            continue
+
+        data, sr = channels[speaker]
+        clip = data[int(start * sr): int(end * sr)]
+        if clip.size == 0:
+            rows.append({"seq_id": seq_id, "processing_status": "skipped_too_short"})
+            continue
+
+        tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
+        tmp.close()
+        try:
+            sf.write(tmp.name, clip, sr)
+            # default build_timeline=True -- see module fidelity note
+            res = predictor.analyze_audio(Path(tmp.name), call_id=call_id)
+            rows.append({
+                "seq_id": seq_id,
+                "sentiment": _enum(res.overall_audio_sentiment),
+                "dominant_emotion": _enum(res.dominant_emotion),
+                "escalation_score": round(float(res.audio_escalation_score), 4),
+                "processing_status": res.processing_status or "success",
+            })
+        except Exception as e:
+            rows.append({"seq_id": seq_id, "processing_status": f"error: {e}"[:120]})
+        finally:
+            try:
+                os.unlink(tmp.name)
+            except OSError:
+                pass
+
+    # record the repo id (stable, machine-independent) -- NOT
+    # config._name_or_path, which resolves to the local HF cache dir and would
+    # leak an absolute user path into an uploaded artifact
+    model_version = os.environ.get("ACOUSTIC_MODEL_REPO", DEFAULT_MODEL_REPO)
+
+    return {"call_id": call_id, "model_version": model_version, "segments": rows}

ml-services/pipeline/audio_io.py

@@ -0,0 +1,82 @@
+"""
+Audio ingest helpers built on ffmpeg/ffprobe.
+
+A fresh call arrives either as one stereo file (agent = ch0, customer = ch1,
+the apptek convention) or as two mono files already split by channel. The
+pipeline downstream of here always works on two mono 16 kHz WAVs (one per
+speaker), so this module normalizes both shapes to that.
+"""
+import json
+import subprocess
+from pathlib import Path
+
+SAMPLE_RATE = 16000   # wav2vec2 + faster-whisper canonical rate
+
+
+class AudioError(RuntimeError):
+    pass
+
+
+def _run(cmd):
+    proc = subprocess.run(cmd, capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise AudioError(f"{cmd[0]} failed:\n{proc.stderr[-800:]}")
+    return proc
+
+
+def probe_channels(path):
+    """Number of audio channels in the first audio stream (via ffprobe)."""
+    proc = _run([
+        "ffprobe", "-v", "error", "-select_streams", "a:0",
+        "-show_entries", "stream=channels", "-of", "json", str(path),
+    ])
+    info = json.loads(proc.stdout)
+    streams = info.get("streams") or []
+    if not streams:
+        raise AudioError(f"no audio stream in {path}")
+    return int(streams[0]["channels"])
+
+
+def to_mono_16k(src, out):
+    """Downmix/resample any input to a single-channel 16 kHz WAV."""
+    out = Path(out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    _run(["ffmpeg", "-y", "-i", str(src),
+          "-ac", "1", "-ar", str(SAMPLE_RATE), str(out)])
+    return str(out)
+
+
+def split_stereo(src, out_agent, out_customer):
+    """
+    Split a 2-channel file into two mono 16 kHz WAVs: ch0 -> agent, ch1 ->
+    customer (apptek stereo attribution). Returns (agent_path, customer_path).
+    """
+    out_agent, out_customer = Path(out_agent), Path(out_customer)
+    for p in (out_agent, out_customer):
+        p.parent.mkdir(parents=True, exist_ok=True)
+    _run([
+        "ffmpeg", "-y", "-i", str(src),
+        "-filter_complex", "[0:a]channelsplit=channel_layout=stereo[l][r]",
+        "-map", "[l]", "-ac", "1", "-ar", str(SAMPLE_RATE), str(out_agent),
+        "-map", "[r]", "-ac", "1", "-ar", str(SAMPLE_RATE), str(out_customer),
+    ])
+    return str(out_agent), str(out_customer)
+
+
+def prepare_channels(primary, out_agent, out_customer, secondary=None):
+    """
+    Normalize an upload to (agent_wav, customer_wav) mono 16 kHz.
+
+      - two files given            -> primary=agent, secondary=customer (resampled)
+      - one 2-channel file         -> channel-split
+      - one 1-channel file, no 2nd -> ambiguous, caller must reject (raises)
+    """
+    if secondary is not None:
+        return (to_mono_16k(primary, out_agent),
+                to_mono_16k(secondary, out_customer))
+    ch = probe_channels(primary)
+    if ch >= 2:
+        return split_stereo(primary, out_agent, out_customer)
+    raise AudioError(
+        "single mono file has no separable speaker channels; upload a stereo "
+        "file or two per-channel files")

ml-services/pipeline/orchestrator.py

@@ -0,0 +1,210 @@
+"""
+Single-call orchestrator: fresh call in -> full dashboard artifacts out.
+
+Runs the same chain the offline CLIs run, but for ONE call, in-process, with a
+progress callback so a web worker can report stage transitions:
+
+  transcribing -> registering -> segmenting -> (acoustic) -> evaluating -> exporting
+
+Every stage reuses the existing modules (no logic forked): faster-whisper via
+transcribe_channels, segmentation via batch_sentence_segments.segment_transcript,
+the LangGraph evaluation via graph.evaluate_call, and the frontend export via
+export_frontend.export_call_artifacts. The acoustic stage is gated (ENABLE_ACOUSTIC
+or the enable_acoustic arg) and degrades to text-only fusion on any failure.
+
+Artifacts land in the same places the batch flow uses, plus the flat
+frontend/public/{sentence_segments,sentiment}/ files CallDetail fetches.
+"""
+import os
+import sys
+import json
+import shutil
+from pathlib import Path
+
+# make the flat evaluation/scripts modules and the src package importable
+# regardless of the caller's cwd (mirrors run_batch.py's sys.path shim)
+_ML = Path(__file__).resolve().parents[1]                 # ml-services/
+_HERE = Path(__file__).resolve().parent                   # ml-services/pipeline/
+for _p in (_HERE, _ML / "evaluation", _ML / "scripts", _ML):
+    if str(_p) not in sys.path:
+        sys.path.insert(0, str(_p))
+
+import paths                                               # noqa: E402
+from assemble import load_manifest                         # noqa: E402
+from batch_sentence_segments import segment_transcript     # noqa: E402
+from graph import evaluate_call                            # noqa: E402
+from export_frontend import export_call_artifacts, summarize  # noqa: E402
+from prompts_domain import prompt_for                      # noqa: E402
+
+RESULTS_DIR_NAME = "results"       # graph reads na_testset/results/{accent}/{id}.json
+EVAL_RESULTS = _ML / "evaluation" / "results"
+
+
+def _noop(*_a, **_k):
+    pass
+
+
+_WHISPER = None
+
+
+def _whisper_model():
+    """Lazy singleton faster-whisper model (loaded once per process)."""
+    global _WHISPER
+    if _WHISPER is None:
+        from faster_whisper import WhisperModel
+        name = os.environ.get("WHISPER_MODEL", "small.en")
+        _WHISPER = WhisperModel(name, device="cpu", compute_type="int8")
+    return _WHISPER
+
+
+def _acoustic_enabled(explicit):
+    if explicit is not None:
+        return bool(explicit)
+    return os.environ.get("ENABLE_ACOUSTIC", "0").strip().lower() not in ("", "0", "false", "no")
+
+
+def _canonical_wavs(call_id, accent, agent_wav, customer_wav):
+    """Ensure per-channel WAVs sit at the dataset convention path the export +
+    manifest consumers expect: NA_TESTSET/{accent}/{call_id}_{role}.wav.
+    Copies only if the source is elsewhere (fresh uploads); manifest calls are
+    already in place."""
+    dst_dir = paths.NA_TESTSET / accent
+    dst_dir.mkdir(parents=True, exist_ok=True)
+    out = {}
+    for role, src in (("agent", agent_wav), ("customer", customer_wav)):
+        dst = dst_dir / f"{call_id}_{role}.wav"
+        if os.path.abspath(src) != os.path.abspath(str(dst)):
+            shutil.copyfile(src, dst)
+        out[role] = str(dst)
+    return out["agent"], out["customer"]
+
+
+def _register(call_id, accent, domain):
+    """Add/replace this call in the runtime manifest so load_manifest() (and
+    thus every eval/export consumer) finds it. Lean entry -- the hosted path
+    reads only accent/domain/call_id; reference transcripts aren't needed."""
+    rm = paths.RUNTIME_MANIFEST
+    entries = json.loads(rm.read_text(encoding="utf-8")) if rm.exists() else []
+    entries = [e for e in entries if e.get("call_id") != call_id]
+    entries.append({
+        "call_id": call_id, "accent": accent, "domain": domain,
+        "agent_wav": f"{accent}/{call_id}_agent.wav",
+        "customer_wav": f"{accent}/{call_id}_customer.wav",
+    })
+    rm.parent.mkdir(parents=True, exist_ok=True)
+    rm.write_text(json.dumps(entries, indent=2), encoding="utf-8")
+
+
+def _write(path, obj):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(obj, indent=2), encoding="utf-8")
+
+
+def _publish_index(call_id, summary):
+    """Upsert one row into frontend/public/calls_index.json (dashboard list)."""
+    idx = paths.FRONTEND_PUBLIC / "calls_index.json"
+    index = json.loads(idx.read_text(encoding="utf-8")) if idx.exists() else []
+    index = [e for e in index if e.get("call_id") != call_id]
+    index.append(summary)
+    _write(idx, index)
+
+
+def process_call(spec, *, progress=_noop, enable_acoustic=None):
+    """
+    Run the full pipeline for one call.
+
+    spec: {call_id, domain, accent, agent_wav, customer_wav} (wav paths absolute).
+    progress: called with a stage name string at each transition.
+    Returns an artifacts dict describing what was written (paths + index row).
+    """
+    call_id = spec["call_id"]
+    accent = spec["accent"]
+    domain = spec["domain"]
+    do_acoustic = _acoustic_enabled(enable_acoustic)
+
+    agent_wav, customer_wav = _canonical_wavs(
+        call_id, accent, spec["agent_wav"], spec["customer_wav"])
+
+    artifacts = {"call_id": call_id, "domain": domain, "accent": accent,
+                 "acoustic": False, "paths": {}}
+
+    # 1. transcribe -----------------------------------------------------------
+    # idempotent: an existing non-empty transcript is reused, so a job retried
+    # after a mid-pipeline crash skips the expensive re-transcription (fresh
+    # uploads carry unique call_ids, so this never wrongly reuses another call)
+    progress("transcribing")
+    transcript_path = paths.NA_TESTSET / RESULTS_DIR_NAME / accent / f"{call_id}.json"
+    if transcript_path.exists() and transcript_path.stat().st_size > 0:
+        print(f"  [transcribe] reusing existing transcript {transcript_path.name}")
+        result = json.loads(transcript_path.read_text(encoding="utf-8"))
+    else:
+        from transcribe_channels import transcribe_call
+        model = _whisper_model()
+        agent_words, customer_words, _segs, _dur = transcribe_call(
+            model, agent_wav, customer_wav,
+            decode={"initial_prompt": prompt_for(domain)})
+        result = {
+            "model": f"faster-whisper {os.environ.get('WHISPER_MODEL', 'small.en')}/int8 + per-channel",
+            "call_id": call_id, "accent": accent, "domain": domain,
+            "agent": agent_words, "customer": customer_words,
+        }
+        _write(transcript_path, result)
+    artifacts["paths"]["transcript"] = str(transcript_path)
+
+    # 2. register -------------------------------------------------------------
+    progress("registering")
+    _register(call_id, accent, domain)
+
+    # 3. segment --------------------------------------------------------------
+    progress("segmenting")
+    seg = segment_transcript(result)
+    _write(paths.SENTENCE_SEG_ROOT / domain.lower() / f"{call_id}.json", seg)
+    # flat, frontend-shaped copy CallDetail fetches (it reads only .sentences)
+    _write(paths.FRONTEND_PUBLIC / "sentence_segments" / f"{call_id}.json", {
+        "call": call_id, "model": seg.get("model", ""),
+        "total": seg["total"], "agent_sentences": seg["agent_sentences"],
+        "customer_sentences": seg["customer_sentences"], "sentences": seg["sentences"],
+    })
+    artifacts["paths"]["sentence_segments"] = str(
+        paths.FRONTEND_PUBLIC / "sentence_segments" / f"{call_id}.json")
+
+    # 4. acoustic (gated) -----------------------------------------------------
+    if do_acoustic:
+        progress("acoustic")
+        try:
+            import acoustic
+            sentiment = acoustic.analyze_call(call_id, seg, agent_wav, customer_wav)
+            sdir = paths.SENTIMENT_ROOT / domain.lower()
+            _write(sdir / f"{call_id}.json", sentiment)          # fusion reads here
+            _write(sdir / f"{call_id}_segments.json", seg)       # fusion join
+            _write(paths.FRONTEND_PUBLIC / "sentiment" / f"{call_id}.json", sentiment)
+            artifacts["acoustic"] = True
+            artifacts["paths"]["sentiment"] = str(
+                paths.FRONTEND_PUBLIC / "sentiment" / f"{call_id}.json")
+        except Exception as e:
+            print(f"  [acoustic] unavailable -> text-only fusion: "
+                  f"{type(e).__name__}: {e}")
+    else:
+        print("  [acoustic] disabled (ENABLE_ACOUSTIC=0) -> text-only fusion")
+
+    # 5. evaluate -------------------------------------------------------------
+    progress("evaluating")
+    ev = evaluate_call(call_id, results_dir=RESULTS_DIR_NAME)
+    _write(EVAL_RESULTS / f"{call_id}_graph.json", ev)
+    artifacts["paths"]["evaluation"] = str(EVAL_RESULTS / f"{call_id}_graph.json")
+
+    # 6. export ---------------------------------------------------------------
+    progress("exporting")
+    meta = load_manifest()[call_id]
+    calls_out = str(paths.FRONTEND_PUBLIC / "calls")
+    audio_out = str(paths.FRONTEND_PUBLIC / "audio")
+    out = export_call_artifacts(call_id, meta, calls_out, audio_out)
+    summary = summarize(call_id, out)
+    _publish_index(call_id, summary)
+    artifacts["paths"]["call_json"] = os.path.join(calls_out, f"{call_id}.json")
+    artifacts["paths"]["audio"] = os.path.join(audio_out, f"{call_id}.mp3")
+    artifacts["index_summary"] = summary
+
+    progress("done")
+    return artifacts

ml-services/pipeline/prompts_domain.py

@@ -0,0 +1,34 @@
+"""
+Whisper decode hints per call domain.
+
+Shared by the offline batch transcriber (scripts/run_batch.py) and the
+single-call orchestrator (pipeline/orchestrator.py) so both feed identical
+initial_prompt context to faster-whisper. Domains without an explicit entry
+fall back to FALLBACK_PROMPT -- the prompt is only a soft decoding bias, so a
+generic hint is safe for the domains outside the original three.
+"""
+
+DOMAIN_PROMPTS = {
+    "banking": (
+        "Banking and customer service call between an agent and a customer. "
+        "Topics include accounts, transfers, payments, balances, credit cards, "
+        "loans, account numbers, and online banking."
+    ),
+    "health": (
+        "Healthcare customer service call between an agent and a patient or caller. "
+        "Topics include appointments, medical records, prescriptions, insurance "
+        "coverage, billing, referrals, and health plan benefits."
+    ),
+    "telecom": (
+        "Telecommunications customer service call between an agent and a customer. "
+        "Topics include mobile plans, internet service, data usage, billing, "
+        "account management, and technical support."
+    ),
+}
+
+FALLBACK_PROMPT = "Customer service call between an agent and a customer."
+
+
+def prompt_for(domain):
+    """Decode hint for a domain (case-insensitive), FALLBACK_PROMPT if unknown."""
+    return DOMAIN_PROMPTS.get((domain or "").lower(), FALLBACK_PROMPT)

ml-services/pipeline/run_one.py

@@ -0,0 +1,66 @@
+"""
+CLI: run the single-call orchestrator for one call_id from the manifest.
+
+  python run_one.py --call_id en_CA_Aviation_1586888            # env ENABLE_ACOUSTIC decides
+  python run_one.py --call_id en_CA_Aviation_1586888 --no-acoustic
+  python run_one.py --call_id en_CA_Aviation_1586888 --acoustic
+
+Builds the CallSpec from the manifest (base + runtime) and streams stage
+timings. The heavy lifting lives in orchestrator.process_call.
+"""
+import sys
+import json
+import time
+import argparse
+from pathlib import Path
+
+_ML = Path(__file__).resolve().parents[1]
+for _p in (Path(__file__).resolve().parent, _ML / "evaluation", _ML / "scripts", _ML):
+    if str(_p) not in sys.path:
+        sys.path.insert(0, str(_p))
+
+import paths                       # noqa: E402
+from assemble import load_manifest  # noqa: E402
+from orchestrator import process_call  # noqa: E402
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--call_id", required=True)
+    g = ap.add_mutually_exclusive_group()
+    g.add_argument("--acoustic", dest="acoustic", action="store_true",
+                   help="force the acoustic stage on")
+    g.add_argument("--no-acoustic", dest="acoustic", action="store_false",
+                   help="force the acoustic stage off")
+    ap.set_defaults(acoustic=None)   # None -> ENABLE_ACOUSTIC env decides
+    args = ap.parse_args()
+
+    manifest = load_manifest()
+    if args.call_id not in manifest:
+        raise SystemExit(f"call_id '{args.call_id}' not in manifest")
+    m = manifest[args.call_id]
+    spec = {
+        "call_id": args.call_id,
+        "domain": m["domain"],
+        "accent": m["accent"],
+        "agent_wav": str(paths.NA_TESTSET / m["agent_wav"].replace("\\", "/")),
+        "customer_wav": str(paths.NA_TESTSET / m["customer_wav"].replace("\\", "/")),
+    }
+
+    t0 = time.time()
+    print(f"Processing {args.call_id} "
+          f"(domain={m['domain']}, accent={m['accent']}, acoustic={args.acoustic})")
+    print("-" * 68)
+
+    def progress(stage):
+        print(f"  [{time.time() - t0:6.1f}s] {stage}")
+
+    artifacts = process_call(spec, progress=progress, enable_acoustic=args.acoustic)
+
+    print("-" * 68)
+    print(f"Done in {time.time() - t0:.1f}s")
+    print(json.dumps(artifacts, indent=2))
+
+
+if __name__ == "__main__":
+    main()

ml-services/scripts/audio_preprocess.py

@@ -0,0 +1,132 @@
+"""
+Phase 2 audio preprocessing: high-pass filter + loudness normalization.
+
+Applied PER CHANNEL, before transcription. Both operations are zero-phase or
+gain-only, so word-level timestamps are unaffected (critical -- we worked hard
+for accurate timestamps and won't shift them here).
+
+High-pass filter
+----------------
+Zero-phase Butterworth high-pass (sosfiltfilt -> no group delay -> no timestamp
+shift). Removes DC offset, sub-bass rumble, and mains hum below the cutoff.
+
+NOTE: spectral analysis of this dataset showed ~0% energy below 80 Hz -- these
+recordings were already high-passed upstream. So on THIS data the filter is
+near-neutral. It is kept because (a) it is correct, robust practice, (b) it is
+cheap insurance for any future recording that does contain rumble, and (c) it
+removes any residual DC before the loudness measurement.
+
+Loudness normalization
+----------------------
+The real lever. Active-RMS normalization: estimate the speech level from
+speech frames only (frames within `rel_db` of the loudest frame, so the estimate
+is immune to the channel's silence fraction), then apply a single gain to hit
+`target_dbfs`. Quiet channels (one was at -37 dBFS active RMS) get boosted toward
+the level Whisper's VAD and encoder expect; already-loud channels barely move.
+
+Safety: gain is capped (`max_gain_db`) so a near-silent channel is not blown up,
+and the output peak is limited to `peak_ceiling` to prevent clipping.
+
+We deliberately do NOT use pyloudnorm/LUFS (extra dependency, and gated LUFS is
+overkill for mono speech) nor spectral denoising (Whisper is trained on noisy
+audio; aggressive denoising tends to hurt -- that is a separate, measured
+experiment, not a default).
+"""
+
+import numpy as np
+from scipy import signal
+
+SR = 16000
+
+
+def highpass(audio, sr=SR, cutoff=80.0, order=2):
+    """Zero-phase Butterworth high-pass. No timestamp shift."""
+    if len(audio) < 32:
+        return audio
+    sos = signal.butter(order, cutoff, btype="high", fs=sr, output="sos")
+    return signal.sosfiltfilt(sos, audio).astype(np.float32)
+
+
+def active_rms(audio, sr=SR, frame_ms=20, rel_db=25.0):
+    """
+    Speech-level estimate: RMS over frames within `rel_db` of the loudest frame.
+    Relative threshold -> robust to how much of the channel is silence.
+    """
+    fl = int(frame_ms / 1000 * sr)
+    if len(audio) < fl:
+        return float(np.sqrt(np.mean(audio ** 2) + 1e-12))
+    n = len(audio) // fl
+    fr = audio[: n * fl].reshape(n, fl)
+    fe = np.sqrt((fr ** 2).mean(axis=1) + 1e-12)
+    thr = fe.max() * (10 ** (-rel_db / 20))
+    speech = fe >= thr
+    if not speech.any():
+        return float(np.sqrt(np.mean(audio ** 2) + 1e-12))
+    return float(np.sqrt((fr[speech] ** 2).mean() + 1e-12))
+
+
+def soft_limit(x, knee=0.85, ceiling=0.98):
+    """
+    Soft knee limiter. Samples below `knee` pass through linearly (the bulk of
+    speech); samples above are tanh-compressed toward `ceiling`. This saturates
+    only the rare transient peaks instead of scaling the whole signal down --
+    so a quiet channel with one click/pop can still reach target loudness.
+    """
+    ax = np.abs(x)
+    over = ax > knee
+    if not over.any():
+        return x
+    out = x.astype(np.float32).copy()
+    s = np.sign(x[over])
+    out[over] = s * (knee + (ceiling - knee) * np.tanh((ax[over] - knee) / (ceiling - knee)))
+    return out
+
+
+def loudness_normalize(audio, sr=SR, target_dbfs=-20.0, max_gain_db=25.0):
+    """Scale to target active-RMS level (gain-capped), then soft-limit peaks."""
+    lvl = active_rms(audio, sr)
+    if lvl <= 1e-9:
+        return audio
+    gain_db = min(target_dbfs - 20 * np.log10(lvl), max_gain_db)
+    out = audio * (10 ** (gain_db / 20))
+    return soft_limit(out).astype(np.float32)
+
+
+def preprocess(audio, sr=SR, do_highpass=True, do_loudness=True,
+               hp_cutoff=80.0, target_dbfs=-20.0, loudness_gate_db=-26.0):
+    """
+    High-pass then conditional loudness normalization.
+
+    Loudness normalization is only applied when the channel's active RMS is
+    below `loudness_gate_db` (default -26 dBFS). Channels already at reasonable
+    levels are left untouched, avoiding transcription regressions caused by
+    over-amplifying carefully-spoken digits or quiet call openings.
+
+    Order matters: high-pass first to remove DC/rumble before measuring level.
+    """
+    if do_highpass:
+        audio = highpass(audio, sr, hp_cutoff)
+    if do_loudness:
+        lvl = 20 * np.log10(active_rms(audio, sr) + 1e-12)
+        if lvl < loudness_gate_db:
+            audio = loudness_normalize(audio, sr, target_dbfs)
+    return audio
+
+
+if __name__ == "__main__":
+    import os, json, soundfile as sf
+    DATA = r"d:\Desktop\ai-ml-capstone\data\na_testset"
+    with open(os.path.join(DATA, "manifest.json")) as f:
+        man = {m["call_id"]: m for m in json.load(f)}
+    print("Self-test (active RMS dBFS, before -> after preprocess):")
+    for cid in ["en_US_General_Health_1587175", "en_CA_Banking_1588683", "en_CA_Banking_1586889"]:
+        m = man[cid]
+        for role, key in [("agent", "agent_wav"), ("customer", "customer_wav")]:
+            a, sr = sf.read(os.path.join(DATA, m[key]), dtype="float32")
+            if a.ndim > 1:
+                a = a.mean(axis=1)
+            before = 20 * np.log10(active_rms(a, sr) + 1e-12)
+            out = preprocess(a, sr)
+            after = 20 * np.log10(active_rms(out, sr) + 1e-12)
+            peak_after = 20 * np.log10(float(np.abs(out).max()) + 1e-12)
+            print(f"  {cid:<30} {role:<9} {before:6.1f} -> {after:6.1f} dBFS  (peak {peak_after:5.1f})")

ml-services/scripts/build_na_testset.py

@@ -0,0 +1,134 @@
+"""
+Build a North-American-accent test set from apptek_callcenter_dialogues.
+
+Downloads the parquet index, filters to the requested accents (all domains),
+pairs channel1 (agent) + channel2 (customer) into calls, downloads the per-channel
+WAVs, and writes a manifest.json the batch runner / evaluator consume.
+
+Resumable: skips WAVs already on disk.
+
+Usage:
+  python build_na_testset.py --accents en-CA en-US_General
+  python build_na_testset.py --accents en-CA en-US_General --limit-per-accent 3   # pilot
+"""
+
+import os
+import re
+import json
+import time
+import argparse
+import urllib.request
+from collections import defaultdict
+
+import pyarrow.parquet as pq
+
+REPO       = "apptek-com/apptek_callcenter_dialogues"
+PARQUET_URL = (
+    "https://huggingface.co/datasets/apptek-com/apptek_callcenter_dialogues"
+    "/resolve/refs%2Fconvert%2Fparquet/default/test/0000.parquet"
+)
+OUT_DIR    = r"d:\Desktop\ai-ml-capstone\data\na_testset"
+INDEX_PARQUET = os.path.join(OUT_DIR, "_index.parquet")
+MANIFEST   = os.path.join(OUT_DIR, "manifest.json")
+UA         = {"User-Agent": "Mozilla/5.0"}
+
+
+def fetch(url, dest, retries=3):
+    for attempt in range(1, retries + 1):
+        try:
+            req = urllib.request.Request(url, headers=UA)
+            with urllib.request.urlopen(req, timeout=120) as r, open(dest, "wb") as f:
+                f.write(r.read())
+            return True
+        except Exception as e:
+            print(f"      attempt {attempt}/{retries} failed: {e}")
+            time.sleep(2 * attempt)
+    return False
+
+
+def path_to_url(hf_path):
+    # hf://datasets/<repo>@<rev>/test/<accent>/audio/<file>.wav
+    m = re.search(r"@([0-9a-f]+)/(.*)$", hf_path)
+    rev, path_in_repo = m.group(1), m.group(2)
+    return f"https://huggingface.co/datasets/{REPO}/resolve/{rev}/{path_in_repo}"
+
+
+def call_id_of(hf_path):
+    fname = hf_path.split("/")[-1]                 # en_CA_Agriculture_1586885_channel1.wav
+    return re.sub(r"_channel[12]\.wav$", "", fname)  # en_CA_Agriculture_1586885
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--accents", nargs="+", default=["en-CA", "en-US_General"])
+    ap.add_argument("--limit-per-accent", type=int, default=None,
+                    help="cap calls per accent (for a pilot run)")
+    args = ap.parse_args()
+
+    os.makedirs(OUT_DIR, exist_ok=True)
+
+    # 1. Index
+    if not os.path.exists(INDEX_PARQUET):
+        print("Downloading parquet index...")
+        if not fetch(PARQUET_URL, INDEX_PARQUET):
+            raise SystemExit("Failed to download parquet index")
+    df = pq.read_table(INDEX_PARQUET).to_pandas()
+    df["path"] = df["audio"].apply(lambda a: a["path"])
+
+    # 2. Filter + pair channels into calls
+    manifest = []
+    for accent in args.accents:
+        sub = df[df["accent"] == accent]
+        calls = defaultdict(dict)
+        for _, row in sub.iterrows():
+            cid = call_id_of(row["path"])
+            ch  = "agent" if row["path"].endswith("channel1.wav") else "customer"
+            calls[cid][ch] = row
+
+        complete = {cid: v for cid, v in calls.items() if "agent" in v and "customer" in v}
+        cids = sorted(complete)
+        if args.limit_per_accent:
+            cids = cids[: args.limit_per_accent]
+        print(f"\n{accent}: {len(complete)} complete calls"
+              f"{f' (using first {len(cids)})' if args.limit_per_accent else ''}")
+
+        adir = os.path.join(OUT_DIR, accent)
+        os.makedirs(adir, exist_ok=True)
+
+        for i, cid in enumerate(cids, 1):
+            a_row, c_row = complete[cid]["agent"], complete[cid]["customer"]
+            a_wav = os.path.join(adir, f"{cid}_agent.wav")
+            c_wav = os.path.join(adir, f"{cid}_customer.wav")
+
+            for wav, row in [(a_wav, a_row), (c_wav, c_row)]:
+                if os.path.exists(wav) and os.path.getsize(wav) > 0:
+                    continue
+                print(f"  [{i}/{len(cids)}] {os.path.basename(wav)}")
+                if not fetch(path_to_url(row["path"]), wav):
+                    print(f"      SKIP (download failed)")
+
+            if os.path.exists(a_wav) and os.path.exists(c_wav):
+                manifest.append({
+                    "call_id": cid,
+                    "accent":  accent,
+                    "domain":  a_row["domain"],
+                    "agent_wav":    os.path.relpath(a_wav, OUT_DIR),
+                    "customer_wav": os.path.relpath(c_wav, OUT_DIR),
+                    "agent_gender":    a_row["gender"],
+                    "customer_gender": c_row["gender"],
+                    "agent_transcript":    a_row["text"],
+                    "customer_transcript": c_row["text"],
+                })
+
+    with open(MANIFEST, "w", encoding="utf-8") as f:
+        json.dump(manifest, f, indent=2)
+
+    by_accent = defaultdict(int)
+    for m in manifest:
+        by_accent[m["accent"]] += 1
+    print(f"\nManifest written: {MANIFEST}")
+    print(f"Total calls: {len(manifest)}  " + "  ".join(f"{k}={v}" for k, v in by_accent.items()))
+
+
+if __name__ == "__main__":
+    main()

ml-services/scripts/compare_models.py

@@ -0,0 +1,151 @@
+"""
+Standalone model comparison over the 12-call probe. NO API calls, NO transcription
+-- pure evaluation of whatever result dirs exist on disk. Run anytime.
+
+Reads (when present):
+  results_channels         small.en  (words: agent/customer lists)
+  results_channels_medium  medium.en (words)
+  results_channels_large   large-v3  (words)
+  results_api_vad          whisper-1 API + VAD trim (agent_text/customer_text)
+  results_channels_llm     small.en + LLM cleanup   (agent_text/customer_text)
+
+Prints: per-call table, OVERALL, per-tier averages, and the banking-only cut
+(banking is the project's target domain). Flags whether each model clears the
+93% goal with cushion.
+"""
+import os, json, jiwer
+from collections import defaultdict
+from eval_common import normalise
+
+DATA      = r"d:\Desktop\ai-ml-capstone\data\na_testset"
+MANIFEST  = os.path.join(DATA, "manifest.json")
+PROBE_SET = os.path.join(DATA, "probe_set.json")
+
+MODELS = [
+    ("small",  "results_channels",        "words"),
+    ("medium", "results_channels_medium", "words"),
+    ("large",  "results_channels_large",  "words"),
+    ("API",    "results_api_vad",         "text"),
+    ("LLM",    "results_channels_llm",    "text"),
+]
+
+
+def acc(ref, hyp):
+    r, h = normalise(ref), normalise(hyp)
+    n = len(r.split())
+    return (1 - jiwer.wer(r, h)) * 100 if n else 100.0, n
+
+
+def load_call(model_dir, fmt, accent, cid):
+    """Return (agent_str, customer_str) or None if not present."""
+    path = os.path.join(DATA, model_dir, accent, cid + ".json")
+    if not os.path.exists(path) or os.path.getsize(path) == 0:
+        return None
+    with open(path, encoding="utf-8") as f:
+        d = json.load(f)
+    if fmt == "words":
+        return (" ".join(w["word"] for w in d["agent"]),
+                " ".join(w["word"] for w in d["customer"]))
+    return d["agent_text"], d["customer_text"]
+
+
+def main():
+    with open(MANIFEST, encoding="utf-8") as f:
+        manifest = {m["call_id"]: m for m in json.load(f)}
+    with open(PROBE_SET, encoding="utf-8") as f:
+        probe = json.load(f)["calls"]
+
+    # which models actually have data
+    present = []
+    for name, d, fmt in MODELS:
+        if os.path.isdir(os.path.join(DATA, d)):
+            present.append((name, d, fmt))
+
+    # per-call overall accuracy per model
+    rows = []                     # (cid, tier, domain, {model: (acc, words, n_done)})
+    agg  = {n: [0.0, 0, 0] for n, _, _ in present}   # [acc*words, words, n_calls]
+    tier_agg = defaultdict(lambda: {n: [0.0, 0] for n, _, _ in present})
+    bank_agg = {n: [0.0, 0] for n, _, _ in present}
+
+    for p in probe:
+        cid, accent, tier = p["call_id"], p["accent"], p["tier"]
+        m = manifest[cid]
+        domain = m["domain"]
+        cell = {}
+        for name, d, fmt in present:
+            loaded = load_call(d, fmt, accent, cid)
+            if loaded is None:
+                cell[name] = None
+                continue
+            ag, cu = loaded
+            aa, na = acc(m["agent_transcript"], ag)
+            ac, nc = acc(m["customer_transcript"], cu)
+            o = (aa*na + ac*nc) / (na+nc)
+            cell[name] = (o, na+nc)
+            agg[name][0] += o*(na+nc); agg[name][1] += na+nc; agg[name][2] += 1
+            tier_agg[tier][name][0] += o; tier_agg[tier][name][1] += 1
+            if domain.lower() == "banking":
+                bank_agg[name][0] += o; bank_agg[name][1] += 1
+        rows.append((cid, tier, domain, cell))
+
+    names = [n for n, _, _ in present]
+    hdr = "  {:<32} {:<13}".format("call_id", "tier") + "".join(f"{n:>8}" for n in names)
+    print(hdr)
+    print("  " + "-" * (len(hdr)-2))
+    for cid, tier, domain, cell in rows:
+        line = "  {:<32} {:<13}".format(cid, tier)
+        for n in names:
+            v = cell[n]
+            line += f"{v[0]:>7.1f}%" if v else f"{'--':>8}"
+        print(line)
+    print("  " + "-" * (len(hdr)-2))
+
+    # overall
+    line = "  {:<32} {:<13}".format("OVERALL", "")
+    overall = {}
+    for n in names:
+        if agg[n][1]:
+            o = agg[n][0]/agg[n][1]; overall[n] = (o, agg[n][2])
+            line += f"{o:>7.1f}%"
+        else:
+            line += f"{'--':>8}"
+    print(line)
+    # n calls done per model
+    line = "  {:<32} {:<13}".format("(calls scored)", "")
+    for n in names:
+        line += f"{agg[n][2]:>7}/12" if agg[n][2] else f"{'--':>8}"
+    print(line.replace("/12  ", "/12"))
+
+    # per-tier
+    print("\n  Per-tier average accuracy:")
+    tiers = ["catastrophic", "bad_banking", "mid_banking", "good_banking", "good_other"]
+    print("  {:<14}".format("tier") + "".join(f"{n:>8}" for n in names))
+    for t in tiers:
+        if t not in tier_agg: continue
+        line = "  {:<14}".format(t)
+        for n in names:
+            s, c = tier_agg[t][n]
+            line += f"{s/c:>7.1f}%" if c else f"{'--':>8}"
+        print(line)
+
+    # banking cut
+    print("\n  BANKING-ONLY (target domain):")
+    line = "  {:<14}".format("banking avg")
+    for n in names:
+        s, c = bank_agg[n]
+        line += f"{s/c:>7.1f}%" if c else f"{'--':>8}"
+    print(line)
+
+    # goal check
+    print("\n  Goal = >=93.0% overall with cushion:")
+    for n in names:
+        if n in overall and overall[n][1] == 12:
+            o = overall[n][0]
+            tag = "CLEARS +cushion" if o >= 93.5 else "clears (thin)" if o >= 93.0 else "short"
+            print(f"    {n:<8} {o:5.1f}%  ({12} calls)  -> {tag}")
+        elif n in overall:
+            print(f"    {n:<8} {overall[n][0]:5.1f}%  ({overall[n][1]}/12 calls -- partial)")
+
+
+if __name__ == "__main__":
+    main()

ml-services/scripts/compare_probe.py

@@ -0,0 +1,137 @@
+"""
+A/B comparison on the frozen probe set. Compares two result dirs without
+re-transcribing. Default: per-channel (A) vs per-channel+preprocess (B).
+The 'base' column is the static mono-mix baseline_acc recorded in probe_set.json.
+
+Also reports:
+  - no_speech_prob diagnostics per dir (mean, #high-ns segments, total words) to
+    show whether preprocessing reduced Whisper's silence-hallucination signal
+  - a confidence-filter experiment on B (drop words with segment ns > threshold)
+
+Usage:
+  python compare_probe.py                               # chan vs chan+pp
+  python compare_probe.py --a-dir results --a-label mix --b-dir results_channels --b-label chan
+"""
+
+import os
+import json
+import argparse
+import jiwer
+from eval_common import normalise
+
+DATA_DIR  = r"d:\Desktop\ai-ml-capstone\data\na_testset"
+MANIFEST  = os.path.join(DATA_DIR, "manifest.json")
+PROBE_SET = os.path.join(DATA_DIR, "probe_set.json")
+
+
+def load_result(root, accent, cid):
+    p = os.path.join(DATA_DIR, root, accent, cid + ".json")
+    if not os.path.exists(p):
+        return None
+    with open(p, encoding="utf-8") as f:
+        return json.load(f)
+
+
+def text_of(words, ns_max=None):
+    if ns_max is None:
+        return " ".join(w["word"] for w in words)
+    return " ".join(w["word"] for w in words if w.get("ns", 0.0) <= ns_max)
+
+
+def score_call(ref_a, ref_c, hyp_a, hyp_c):
+    ra, rc = normalise(ref_a), normalise(ref_c)
+    ha, hc = normalise(hyp_a), normalise(hyp_c)
+    na, nc = len(ra.split()), len(rc.split())
+    if na + nc == 0:
+        return None, 0
+    wa = jiwer.wer(ra, ha) if na else 0.0
+    wc = jiwer.wer(rc, hc) if nc else 0.0
+    return (wa * na + wc * nc) / (na + nc), na + nc
+
+
+def acc(wer):
+    return None if wer is None else (1 - wer) * 100
+
+
+def ns_diag(result):
+    """(#high-ns segments, total segments, total hyp words) from captured segments."""
+    if not result or "segments" not in result:
+        return None
+    segs = result["segments"].get("agent", []) + result["segments"].get("customer", [])
+    hi = sum(1 for s in segs if s.get("no_speech_prob", 0) > 0.6)
+    words = len(result.get("agent", [])) + len(result.get("customer", []))
+    return hi, len(segs), words
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--a-dir", default="results_channels"); ap.add_argument("--a-label", default="chan")
+    ap.add_argument("--b-dir", default="results_channels_pp"); ap.add_argument("--b-label", default="chan+pp")
+    ap.add_argument("--ns-thresholds", nargs="*", type=float, default=[0.6, 0.8])
+    args = ap.parse_args()
+
+    with open(MANIFEST, encoding="utf-8") as f:
+        manifest = {m["call_id"]: m for m in json.load(f)}
+    with open(PROBE_SET, encoding="utf-8") as f:
+        probe = json.load(f)["calls"]
+
+    print(f"{'tier':<13} {'call_id':<32} {'base':>6} {args.a_label:>7} {args.b_label:>8} {'delta':>7}")
+    print("-" * 80)
+
+    agg = {"a": [0.0, 0], "b": [0.0, 0]}
+    agg_ns = {t: [0.0, 0] for t in args.ns_thresholds}
+    nsA = [0, 0, 0]; nsB = [0, 0, 0]   # hi, segs, words
+    missing = []
+
+    for p in probe:
+        cid, accent = p["call_id"], p["accent"]
+        m = manifest[cid]
+        ra_, rc_ = m["agent_transcript"], m["customer_transcript"]
+        A = load_result(args.a_dir, accent, cid)
+        B = load_result(args.b_dir, accent, cid)
+        if A is None or B is None:
+            missing.append((cid, args.a_dir if A is None else args.b_dir))
+            continue
+
+        a_wer, n = score_call(ra_, rc_, text_of(A["agent"]), text_of(A["customer"]))
+        b_wer, _ = score_call(ra_, rc_, text_of(B["agent"]), text_of(B["customer"]))
+        agg["a"][0] += a_wer * n; agg["a"][1] += n
+        agg["b"][0] += b_wer * n; agg["b"][1] += n
+
+        for t in args.ns_thresholds:
+            w, _ = score_call(ra_, rc_, text_of(B["agent"], t), text_of(B["customer"], t))
+            agg_ns[t][0] += w * n; agg_ns[t][1] += n
+
+        for store, res in [(nsA, A), (nsB, B)]:
+            d = ns_diag(res)
+            if d:
+                store[0] += d[0]; store[1] += d[1]; store[2] += d[2]
+
+        delta = acc(b_wer) - acc(a_wer)
+        flag = "  <<" if delta >= 3 else ("  !!" if delta <= -3 else "")
+        print(f"{p['tier']:<13} {cid:<32} {p['baseline_acc']*100:>5.1f}% "
+              f"{acc(a_wer):>6.1f}% {acc(b_wer):>7.1f}% {delta:>+6.1f}{flag}")
+
+    print("-" * 80)
+    a_acc = acc(agg["a"][0] / agg["a"][1])
+    b_acc = acc(agg["b"][0] / agg["b"][1])
+    print(f"{'OVERALL (word-weighted)':<46} {a_acc:>6.1f}% {b_acc:>7.1f}% {b_acc - a_acc:>+6.1f}")
+
+    print(f"\nno_speech_prob diagnostics (hallucination signal):")
+    print(f"  {args.a_label:<10} high-ns segs: {nsA[0]:>4} / {nsA[1]:<5} segments | total hyp words: {nsA[2]}")
+    print(f"  {args.b_label:<10} high-ns segs: {nsB[0]:>4} / {nsB[1]:<5} segments | total hyp words: {nsB[2]}")
+
+    print(f"\nConfidence filter on {args.b_label} (drop words where segment no_speech_prob > t):")
+    print(f"  no filter        : {b_acc:5.1f}%")
+    for t in args.ns_thresholds:
+        a = acc(agg_ns[t][0] / agg_ns[t][1])
+        print(f"  ns <= {t:<4}       : {a:5.1f}%  ({a - b_acc:+.1f})")
+
+    if missing:
+        print("\nMissing results:")
+        for cid, which in missing:
+            print(f"  {cid}  ({which})")
+
+
+if __name__ == "__main__":
+    main()