feat: aI analysis servisleri routes detectors ve test suites eklendi

Dockerfile

@@ -40,15 +40,9 @@ RUN pip install --no-cache-dir torch torchvision torchaudio --index-url https://
     && pip install --no-cache-dir -r requirements.txt \
     && pip install --no-cache-dir "huggingface-hub>=0.20.0"
 
-# Uygulama kodu
-COPY app ./app
-COPY startup.py .
-
-# Models dizini — startup.py çalışınca doldurulacak
-RUN mkdir -p /app/models
-
 # Hugging Face Spaces için non-root kullanıcı (güvenlik)
 RUN useradd -m -u 1000 user \
+    && mkdir -p /app/models \
     && chown -R user:user /app
 USER user
 
@@ -57,12 +51,22 @@ ENV HOME=/home/user \
     AURIS_MODELS_REPO=Rthur2003/auris-models \
     MODELS_DIR=/app/models
 
+# Model ağırlıkları imaja gömülür: Space uykudan her uyandığında ~415 MB
+# AURIS modelini ve wav2vec2-base'i yeniden indirmek yerine hazır bulur.
+# Kod değişince bu katmanlar önbellekten gelir (yalnızca startup.py değişirse yeniden iner).
+COPY --chown=user:user startup.py .
+RUN python startup.py \
+    && python -c "from transformers import Wav2Vec2Model; Wav2Vec2Model.from_pretrained('facebook/wav2vec2-base')"
+
+# Uygulama kodu
+COPY --chown=user:user app ./app
+
 # Hugging Face Spaces varsayılan port: 7860
 EXPOSE 7860
 
-# Health check — model download ~2-3 dk sürebilir, start-period uzun
-HEALTHCHECK --interval=30s --timeout=10s --start-period=180s --retries=5 \
+# Health check — modeller imajda, açılış kısa
+HEALTHCHECK --interval=30s --timeout=10s --start-period=90s --retries=5 \
     CMD curl -f http://localhost:7860/api/health || exit 1
 
-# Başlat: önce modelleri indir, sonra API'yi ayağa kaldır
+# startup.py dosyalar mevcutsa hiçbir şey indirmez; eksik kalan olursa tamamlar.
 CMD ["sh", "-c", "python startup.py && uvicorn app.main:app --host 0.0.0.0 --port 7860"]

app/routes/analyze.py

@@ -1,29 +1,41 @@
 """General analysis route for CrownCode.
 
-Handles YouTube URLs and file uploads through a unified endpoint.
+Handles YouTube URLs and file uploads through a unified pipeline.
 Response format matches platform/hooks/analysisTypes.ts
 
 Pipeline:
   1. Source acquisition (YouTube download or file upload)
-  2. Feature extraction (spectral, temporal, harmonic)
-  3. Vocal analysis (pitch, vibrato, formant, breath)
-  4. Score fusion (local features + vocals + external services)
-  5. Response formatting (platform-compatible JSON)
+  2. Feature extraction, vocal analysis, CLAP and wav2vec2 in parallel
+  3. FST comparison model (external Space)
+  4. Score fusion, then the calibrated XAI verdict (LightGBM + votes + SHAP)
+  5. Meta-classifier second opinion, response formatting
+
+Endpoints:
+  POST /api/analyze                 one request, waits for the result
+  POST /api/analyze/jobs            starts a background job, returns its id
+  GET  /api/analyze/jobs/{job_id}   per-step progress, then the result
+
+Nothing here invents a result: when the audio can't be decoded or a
+link can't be downloaded, the response carries an error, not a guess.
 """
 
 from __future__ import annotations
 
 import asyncio
 import io
+import tempfile
 import time
 import uuid
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
-from typing import Optional, List, Dict
+from pathlib import Path
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
+
 from fastapi import (
     APIRouter, File, UploadFile, Form,
     Request, Depends, HTTPException, status,
 )
+from fastapi.responses import JSONResponse
 
 from .analyze_schemas import (
     AnalysisFeatures,
@@ -38,8 +50,16 @@ from .analyze_schemas import (
     VocalAnalysis,
     XAIExplanation,
 )
-from ..services.youtube_analysis import YouTubeAnalysisService
-from ..services.preview_model import PreviewModel
+from ..services.analysis_jobs import (
+    FILE_STEPS,
+    URL_STEPS,
+    STEP_DONE,
+    STEP_FAILED,
+    STEP_RUNNING,
+    STEP_SKIPPED,
+    Progress,
+    job_store,
+)
 from ..services.feature_extractor import (
     extract_features,
     AudioFeatures,
@@ -54,11 +74,30 @@ from ..services.fst_client import FSTClientService
 from ..services.wav2vec2_detector import Wav2Vec2DetectorService
 from ..services.inference_xai import get_xai_service
 from ..services.meta_classifier import MetaClassifierService, MODEL_VERSION_TRAINED
+from ..services.url_parser import parse_youtube_url
+from ..services.youtube_downloader import YouTubeDownloadError, YouTubeDownloader
 from ..services.logging_config import get_logger
 
 router = APIRouter()
 logger = get_logger(__name__)
 
+MAX_UPLOAD_BYTES = 30 * 1024 * 1024
+MIN_UPLOAD_BYTES = 1024
+AUDIO_EXTENSIONS = {".mp3", ".wav", ".flac", ".m4a", ".mp4", ".aac", ".ogg", ".opus", ".webm"}
+_EXT_MIME = {
+    ".mp3": "audio/mpeg", ".wav": "audio/wav", ".flac": "audio/flac",
+    ".m4a": "audio/mp4", ".mp4": "audio/mp4", ".aac": "audio/aac",
+    ".ogg": "audio/ogg", ".opus": "audio/ogg", ".webm": "audio/webm",
+}
+URL_SOURCES = ("youtube", "tiktok", "instagram", "soundcloud", "twitter")
+
+# A link only needs its opening: features read 60 s, wav2vec2 30 s.
+YOUTUBE_CLIP_SEC = 120.0
+YOUTUBE_MAX_BYTES = 100 * 1024 * 1024
+
+# cpu-basic has 2 vCPUs; more concurrent jobs only queue in the pool.
+MAX_RUNNING_JOBS = 4
+
 # Rate limiter for public heavy endpoints
 _analyze_rate_store: dict[str, list[float]] = defaultdict(list)
 _ANALYZE_RATE_WINDOW = 60  # seconds
@@ -66,9 +105,23 @@ _ANALYZE_RATE_MAX = 20  # requests per minute per IP
 _ANALYZE_MAX_IPS = 10_000
 
 
+def _client_ip(request: Request) -> str:
+    """The visitor's address behind the Hugging Face proxy.
+
+    uvicorn sees the proxy, so request.client alone would put every
+    visitor in one bucket; the proxy lists the original client first
+    in X-Forwarded-For.
+    """
+    forwarded = request.headers.get("x-forwarded-for", "")
+    first = forwarded.split(",")[0].strip()
+    if first:
+        return first
+    return request.client.host if request.client else "unknown"
+
+
 async def _analyze_rate_limit(request: Request) -> None:
     """Rate limit for analysis endpoints."""
-    client_ip = request.client.host if request.client else "unknown"
+    client_ip = _client_ip(request)
     now = time.time()
     cutoff = now - _ANALYZE_RATE_WINDOW
 
@@ -93,8 +146,6 @@ async def _analyze_rate_limit(request: Request) -> None:
     _analyze_rate_store[client_ip] = hits
 
 
-youtube_service = YouTubeAnalysisService()
-preview_model = PreviewModel()
 clap_service = CLAPDetectorService()
 fst_service = FSTClientService()
 wav2vec2_service = Wav2Vec2DetectorService()
@@ -103,167 +154,92 @@ meta_classifier_service = MetaClassifierService()
 
 # Thread pool for CPU-bound audio analysis (feature extraction + vocal)
 _analysis_pool = ThreadPoolExecutor(max_workers=3)
+# Downloads wait on the network; keep them off the analysis workers.
+_download_pool = ThreadPoolExecutor(max_workers=2)
 
 
-@router.post(
-    "/api/analyze",
-    response_model=AnalyzeResponse,
-    dependencies=[Depends(_analyze_rate_limit)],
-)
-async def analyze(
-    sourceType: str = Form(...),
-    url: Optional[str] = Form(None),
-    file: Optional[UploadFile] = File(None)
-) -> AnalyzeResponse:
-    """Unified analysis endpoint."""
-    request_id = str(uuid.uuid4())[:8]
-    start_time = time.monotonic()
-    logger.info(f"[{request_id}] Analysis: sourceType={sourceType}")
-
-    try:
-        if sourceType in ("youtube", "tiktok", "instagram", "soundcloud", "twitter"):
-            if not url:
-                return AnalyzeResponse(errors=["missing_url"])
-            return await _analyze_youtube(request_id, url, start_time)
+# ── helpers ───────────────────────────────────────────────────────────
 
-        elif sourceType == "file":
-            if not file:
-                return AnalyzeResponse(errors=["missing_file"])
-            return await _analyze_file(request_id, file, start_time)
+def _unique(values: List[str]) -> List[str]:
+    return list(dict.fromkeys(values))
 
-        elif sourceType in ["spotify", "apple"]:
-            return AnalyzeResponse(errors=["unsupported_source"])
 
+def _probe_audio(content: bytes) -> Tuple[Optional[int], Optional[int], Optional[float]]:
+    """Original sample rate, channels and full duration, when soundfile can
+    read the container (WAV/FLAC/OGG/MP3). The analysis itself loads mono
+    at 22.05 kHz and only the first 60 s, so it can't tell these."""
+    try:
+        import soundfile as sf
+        info = sf.info(io.BytesIO(content))
+        return int(info.samplerate), int(info.channels), float(info.duration)
+    except Exception:
+        return None, None, None
+
+
+def _track(future: "asyncio.Future[Any]", progress: Progress, step: str,
+           ok: Optional[Callable[[Any], bool]] = None) -> None:
+    """Report a parallel step as done/skipped/failed the moment it settles."""
+    def _settled(f: "asyncio.Future[Any]") -> None:
+        if f.cancelled() or f.exception() is not None:
+            progress.set(step, STEP_FAILED)
+        elif ok is not None and not ok(f.result()):
+            progress.set(step, STEP_SKIPPED)
         else:
-            return AnalyzeResponse(errors=["invalid_source_type"])
+            progress.set(step, STEP_DONE)
+    future.add_done_callback(_settled)
+
+
+async def _read_upload(file: UploadFile) -> Tuple[Optional[bytes], Optional[str], str]:
+    """Validate an upload. Returns (content, error_code, content_type)."""
+    filename = file.filename or "audio"
+    ext = Path(filename).suffix.lower()
+    content_type = (file.content_type or "").lower()
+    # Browsers label some audio containers as video (webm, mp4); trust the extension then.
+    if not (content_type.startswith("audio/") or ext in AUDIO_EXTENSIONS):
+        return None, "invalid_file_type", content_type
+    if not content_type.startswith("audio/"):
+        content_type = _EXT_MIME.get(ext, "audio/unknown")
 
-    except Exception as e:
-        logger.error(f"[{request_id}] Error: {e}", exc_info=True)
-        return AnalyzeResponse(errors=["internal_error"])
+    content = await file.read(MAX_UPLOAD_BYTES + 1)
+    if len(content) > MAX_UPLOAD_BYTES:
+        return None, "file_too_large", content_type
+    if len(content) < MIN_UPLOAD_BYTES:
+        return None, "file_too_small", content_type
+    return content, None, content_type
 
 
-async def _analyze_youtube(
-    request_id: str, url: str, start_time: float
-) -> AnalyzeResponse:
-    """Analyze YouTube URL with full pipeline."""
-    try:
-        yt_result = await youtube_service.analyze(url, include_raw=True)
-        processing_time = round(time.monotonic() - start_time, 3)
-
-        # If download succeeded, we have audio to analyze
-        # For now, use fusion with external service results + preview features
-        # TODO: When YouTube pipeline returns audio path, run full extraction
-
-        # Use preview model for feature scores until
-        # YouTube pipeline returns raw audio for local
-        # extraction with full feature + vocal analysis
-        preview = preview_model.analyze(yt_result.source.video_id)
-        features_data = preview.get("features", {})
-
-        decision = yt_result.summary.decision_source
-        result = AnalysisResult(
-            isAIGenerated=yt_result.summary.is_ai_generated,
-            confidence=yt_result.summary.confidence,
-            processingTime=processing_time,
-            modelVersion=yt_result.summary.model_version,
-            decisionSource=decision,
-            analysisMode="preview" if decision == "preview" else "production",
-            source={
-                "kind": "youtube",
-                "url": url,
-                "normalizedUrl": yt_result.source.normalized_url,
-                "videoId": yt_result.source.video_id,
-                "startTimeSec": yt_result.source.start_time_sec
-            },
-            features=AnalysisFeatures(
-                spectralRegularity=features_data.get(
-                    "spectral_regularity", 0.5,
-                ),
-                temporalPatterns=features_data.get(
-                    "temporal_patterns", 0.5,
-                ),
-                harmonicStructure=features_data.get(
-                    "harmonic_structure", 0.5,
-                ),
-                artificialIndicators=yt_result.summary.indicators
-            ),
-            audioInfo=AudioInfo(
-                duration=yt_result.source.duration_sec or 0.0,
-                sampleRate=44100,
-                bitrate=128,
-                format=yt_result.source.audio_format or "unknown"
-            )
-        )
-
-        return AnalyzeResponse(
-            result=result,
-            warnings=yt_result.warnings,
-            errors=yt_result.errors,
-        )
+# ── the pipeline ──────────────────────────────────────────────────────
 
-    except ValueError as e:
-        logger.warning(f"[{request_id}] Invalid URL: {e}")
-        return AnalyzeResponse(errors=["invalid_youtube_url"])
-    except Exception as e:
-        logger.error(f"[{request_id}] YouTube failed: {e}")
-        return AnalyzeResponse(errors=["youtube_analysis_failed"])
-
-
-async def _analyze_file(
+async def _run_pipeline(
     request_id: str,
-    file: UploadFile,
+    content: bytes,
+    *,
+    filename: str,
+    content_type: str,
+    source: Dict[str, Any],
     start_time: float,
+    progress: Progress,
+    warnings: Optional[List[str]] = None,
+    total_duration: Optional[float] = None,
 ) -> AnalyzeResponse:
-    """
-    Full audio analysis pipeline for uploaded files.
-
-    Pipeline:
-      1. Validate file
-      2. Extract audio features (spectral, temporal, harmonic)
-      3. Analyze vocals (pitch, vibrato, formant, breath)
-      4. Fuse all scores into final result
-    """
-    content_type = file.content_type or ""
-    if not content_type.startswith("audio/"):
-        return AnalyzeResponse(errors=["invalid_file_type"])
-
-    content = await file.read()
+    """Run every layer on one piece of audio and build the response."""
+    warnings = list(warnings or [])
     file_size = len(content)
-
-    if file_size > 30 * 1024 * 1024:
-        return AnalyzeResponse(errors=["file_too_large"])
-
-    if file_size < 1024:
-        return AnalyzeResponse(errors=["file_too_small"])
-
-    mb = file_size / 1024 / 1024
-    logger.info(
-        f"[{request_id}] File: {file.filename} "
-        f"({mb:.2f}MB)"
-    )
-
-    warnings: List[str] = []
-    loop = asyncio.get_event_loop()
-
-    # ── Run feature extraction + vocal analysis + CLAP + wav2vec2 in parallel ──
-    def _extract_features_sync():
-        return extract_features(io.BytesIO(content))
-
-    def _analyze_vocals_sync():
-        return analyze_vocals(io.BytesIO(content))
-
-    def _clap_predict_sync():
-        return clap_service.predict(io.BytesIO(content))
-
-    def _wav2vec2_predict_sync():
-        return wav2vec2_service.predict(io.BytesIO(content))
-
-    feat_future = loop.run_in_executor(_analysis_pool, _extract_features_sync)
-    vocal_future = loop.run_in_executor(_analysis_pool, _analyze_vocals_sync)
-    clap_future = loop.run_in_executor(_analysis_pool, _clap_predict_sync)
-    wav2vec2_future = loop.run_in_executor(_analysis_pool, _wav2vec2_predict_sync)
-
-    # Await feature extraction first (required for fusion)
+    logger.info(f"[{request_id}] Audio: {filename} ({file_size / 1024 / 1024:.2f}MB)")
+    loop = asyncio.get_running_loop()
+
+    # ── Feature extraction + vocal analysis + CLAP + wav2vec2 in parallel ──
+    progress.running("features", "vocals", "clap", "wav2vec2")
+    feat_future = loop.run_in_executor(_analysis_pool, lambda: extract_features(io.BytesIO(content)))
+    vocal_future = loop.run_in_executor(_analysis_pool, lambda: analyze_vocals(io.BytesIO(content)))
+    clap_future = loop.run_in_executor(_analysis_pool, lambda: clap_service.predict(io.BytesIO(content)))
+    wav2vec2_future = loop.run_in_executor(_analysis_pool, lambda: wav2vec2_service.predict(io.BytesIO(content)))
+    _track(feat_future, progress, "features")
+    _track(vocal_future, progress, "vocals")
+    _track(clap_future, progress, "clap", ok=lambda r: bool(r.available))
+    _track(wav2vec2_future, progress, "wav2vec2", ok=lambda r: bool(r.available))
+
+    # Feature extraction is required for everything below.
     try:
         features: AudioFeatures = await feat_future
         logger.info(
@@ -273,17 +249,18 @@ async def _analyze_file(
             f"H={features.harmonic_structure:.3f}"
         )
     except ValueError as e:
-        logger.warning(f"[{request_id}] Feature extraction failed: {e}")
-        return _fallback_analysis(
-            request_id, file, content, file_size, start_time, str(e),
-        )
+        logger.warning(f"[{request_id}] Feature extraction rejected the audio: {e}")
+        for step in ("fst", "xai", "meta"):
+            progress.set(step, STEP_SKIPPED)
+        message = str(e).lower()
+        code = "audio_too_short" if "short" in message else "audio_silent" if "silent" in message else "audio_decode_failed"
+        return AnalyzeResponse(errors=[code], warnings=_unique(warnings))
     except Exception as e:
         logger.error(f"[{request_id}] Feature extraction error: {e}", exc_info=True)
-        return _fallback_analysis(
-            request_id, file, content, file_size, start_time, str(e),
-        )
+        for step in ("fst", "xai", "meta"):
+            progress.set(step, STEP_SKIPPED)
+        return AnalyzeResponse(errors=["audio_decode_failed"], warnings=_unique(warnings))
 
-    # Await vocal analysis (ran in parallel)
     vocals: Optional[VocalFeatures] = None
     try:
         vocals = await vocal_future
@@ -299,106 +276,106 @@ async def _analyze_file(
         logger.warning(f"[{request_id}] Vocal analysis failed: {e}")
         warnings.append("vocal_analysis_unavailable")
 
-    # Await CLAP (ran in parallel)
     clap_result = None
     try:
         clap_result = await clap_future
         if clap_result.available:
             logger.info(
-                f"[{request_id}] CLAP: "
-                f"ai={clap_result.is_ai}, "
-                f"conf={clap_result.confidence:.3f}"
+                f"[{request_id}] CLAP ({clap_result.classifier_used}): "
+                f"ai={clap_result.is_ai}, conf={clap_result.confidence:.3f}"
             )
         else:
             logger.info(f"[{request_id}] CLAP unavailable: {clap_result.error}")
+            warnings.append("clap_analysis_unavailable")
     except Exception as e:
         logger.warning(f"[{request_id}] CLAP failed: {e}")
         warnings.append("clap_analysis_unavailable")
 
-    # Await wav2vec2 (ran in parallel)
     wav2vec2_result = None
     try:
         wav2vec2_result = await wav2vec2_future
         if wav2vec2_result.available:
-            logger.info(
-                f"[{request_id}] wav2vec2: "
-                f"p_ai={wav2vec2_result.p_ai:.3f}"
-            )
+            logger.info(f"[{request_id}] wav2vec2: p_ai={wav2vec2_result.p_ai:.3f}")
         else:
             logger.info(f"[{request_id}] wav2vec2 unavailable: {wav2vec2_result.error}")
+            warnings.append("wav2vec2_unavailable")
     except Exception as e:
         logger.warning(f"[{request_id}] wav2vec2 failed: {e}")
         warnings.append("wav2vec2_unavailable")
 
-    # FST detection (Layer 3) — async native
+    # ── FST detection (external Space) ──
     fst_result = None
+    progress.set("fst", STEP_RUNNING)
     try:
         fst_result = await fst_service.predict(io.BytesIO(content))
         if fst_result.available:
+            progress.set("fst", STEP_DONE)
             logger.info(
-                f"[{request_id}] FST: "
-                f"ai={fst_result.is_ai}, "
-                f"conf={fst_result.confidence:.3f}, "
-                f"label={fst_result.label}"
+                f"[{request_id}] FST: ai={fst_result.is_ai}, "
+                f"conf={fst_result.confidence:.3f}, label={fst_result.label}"
             )
         else:
+            progress.set("fst", STEP_SKIPPED)
             logger.info(f"[{request_id}] FST unavailable: {fst_result.error}")
+            warnings.append("fst_analysis_unavailable")
     except Exception as e:
+        progress.set("fst", STEP_FAILED)
         logger.warning(f"[{request_id}] FST failed: {e}")
         warnings.append("fst_analysis_unavailable")
 
-    # ── Step 5: Score fusion ───────────────
+    # ── Score fusion ──
     fusion: FusionResult = fuse_scores(
         features, vocals,
         clap_result=clap_result,
         fst_result=fst_result,
         wav2vec2_result=wav2vec2_result,
     )
-
-    processing_time = round(time.monotonic() - start_time, 3)
     logger.info(
         f"[{request_id}] Fusion: ai={fusion.is_ai_generated}, "
         f"confidence={fusion.confidence}, source={fusion.decision_source}"
     )
 
-    # ── Step 5b: XAI inference (explainable) ────
+    # ── XAI inference (calibrated verdict + votes + SHAP) ──
     xai_payload: Optional[XAIExplanation] = None
+    progress.set("xai", STEP_RUNNING)
     try:
-        if xai_service.available:
-            xai_result = xai_service.predict(features, vocals)
-            if xai_result is not None:
-                xai_dict = xai_service.to_dict(xai_result)
-                # Convert dicts to pydantic models
-                xai_payload = XAIExplanation(
-                    probability=xai_dict["probability"],
-                    threshold=xai_dict["threshold"],
-                    baseProbability=xai_dict["baseProbability"],
-                    confidenceBand=ConfidenceBandModel(**xai_dict["confidenceBand"]),
-                    modelVotes=[ModelVoteModel(**v) for v in xai_dict["modelVotes"]],
-                    bestModel=xai_dict["bestModel"],
-                    topContributions=[
-                        FeatureContributionModel(**c)
-                        for c in xai_dict["topContributions"]
-                    ],
-                    allFeatures={
-                        k: FeatureContributionModel(**v)
-                        for k, v in xai_dict["allFeatures"].items()
-                    },
-                    featureCount=xai_dict["featureCount"],
-                )
-                logger.info(
-                    f"[{request_id}] XAI: p={xai_result.probability:.3f}, "
-                    f"band={xai_result.confidence_band.tier}, "
-                    f"top={xai_result.top_contributions[0].name if xai_result.top_contributions else 'n/a'}"
-                )
+        xai_result = xai_service.predict(features, vocals) if xai_service.available else None
+        if xai_result is not None:
+            xai_dict = xai_service.to_dict(xai_result)
+            xai_payload = XAIExplanation(
+                probability=xai_dict["probability"],
+                threshold=xai_dict["threshold"],
+                baseProbability=xai_dict["baseProbability"],
+                confidenceBand=ConfidenceBandModel(**xai_dict["confidenceBand"]),
+                modelVotes=[ModelVoteModel(**v) for v in xai_dict["modelVotes"]],
+                bestModel=xai_dict["bestModel"],
+                topContributions=[
+                    FeatureContributionModel(**c)
+                    for c in xai_dict["topContributions"]
+                ],
+                allFeatures={
+                    k: FeatureContributionModel(**v)
+                    for k, v in xai_dict["allFeatures"].items()
+                },
+                featureCount=xai_dict["featureCount"],
+            )
+            progress.set("xai", STEP_DONE)
+            logger.info(
+                f"[{request_id}] XAI: p={xai_result.probability:.3f}, "
+                f"band={xai_result.confidence_band.tier}, "
+                f"top={xai_result.top_contributions[0].name if xai_result.top_contributions else 'n/a'}"
+            )
+        else:
+            progress.set("xai", STEP_SKIPPED)
+            warnings.append("xai_unavailable")
     except Exception as e:
+        progress.set("xai", STEP_FAILED)
         logger.warning(f"[{request_id}] XAI failed: {e}")
         warnings.append("xai_unavailable")
 
-    # ── Step 5c: Meta-classifier (trained stacking ensemble) ────
-    # A second, independent opinion shown for transparency — never
-    # overrides the primary XAI/fusion verdict above.
+    # ── Meta-classifier: a second opinion, never overrides the verdict ──
     meta_payload: Optional[MetaClassifierExplanation] = None
+    progress.set("meta", STEP_RUNNING)
     try:
         meta_result = meta_classifier_service.predict(
             features, vocals=vocals,
@@ -415,15 +392,20 @@ async def _analyze_file(
                     MetaFeatureImportance(**f) for f in meta_result.top_features
                 ],
             )
+            progress.set("meta", STEP_DONE)
             logger.info(
                 f"[{request_id}] Meta-classifier: ai={meta_result.is_ai_generated}, "
                 f"confidence={meta_result.confidence}"
             )
+        else:
+            progress.set("meta", STEP_SKIPPED)
+            warnings.append("meta_classifier_unavailable")
     except Exception as e:
+        progress.set("meta", STEP_FAILED)
         logger.warning(f"[{request_id}] Meta-classifier failed: {e}")
         warnings.append("meta_classifier_unavailable")
 
-    # Build tower scores dict for UI (multi-signal breakdown)
+    # ── Tower scores (P(AI) per independent layer) ──
     tower_scores: Dict[str, float] = {
         "local_features": round(
             (features.spectral_regularity + features.temporal_patterns
@@ -441,7 +423,29 @@ async def _analyze_file(
     if xai_payload is not None:
         tower_scores["xai_ensemble"] = round(xai_payload.probability, 3)
 
-    # ── Step 4: Build response ─────────────
+    # Which implementation each layer actually used this time.
+    layers: Dict[str, Dict[str, Any]] = {
+        "wav2vec2": {
+            "available": bool(wav2vec2_result and wav2vec2_result.available),
+            "error": getattr(wav2vec2_result, "error", None),
+        },
+        "clap": {
+            "available": bool(clap_result and clap_result.available),
+            "mode": getattr(clap_result, "classifier_used", None),
+            "error": getattr(clap_result, "error", None),
+        },
+        "fst": {
+            "available": bool(fst_result and fst_result.available),
+            "label": getattr(fst_result, "label", None),
+            "error": getattr(fst_result, "error", None),
+        },
+        "xai": {
+            "available": xai_payload is not None,
+            "ensembleLoaded": len(xai_service.ensemble_models),
+        },
+        "meta": {"available": meta_payload is not None},
+    }
+
     vocal_response = None
     if vocals and vocals.has_vocals:
         vocal_response = VocalAnalysis(
@@ -460,17 +464,14 @@ async def _analyze_file(
             indicators=vocals.indicators,
         )
 
-    # Estimate bitrate from file size + duration (kbps)
-    estimated_bitrate = 128
-    if features.duration_sec > 0:
-        estimated_bitrate = max(
-            32, min(
-                int((file_size * 8) / (features.duration_sec * 1000)),
-                2048,
-            )
-        )
+    # ── Audio facts: from the container when readable, else what was analysed ──
+    orig_sr, orig_channels, probed_duration = _probe_audio(content)
+    duration = total_duration or probed_duration or features.duration_sec
+    bitrate = 0
+    if probed_duration and probed_duration > 0:
+        bitrate = max(1, int((file_size * 8) / (probed_duration * 1000)))
 
-    # Prefer XAI verdict if available — it's the calibrated classifier
+    # Prefer the XAI verdict — it's the calibrated classifier.
     final_is_ai = fusion.is_ai_generated
     final_confidence = fusion.confidence
     final_decision = fusion.decision_source
@@ -482,16 +483,11 @@ async def _analyze_file(
     result = AnalysisResult(
         isAIGenerated=final_is_ai,
         confidence=final_confidence,
-        processingTime=processing_time,
+        processingTime=round(time.monotonic() - start_time, 3),
         modelVersion=fusion.model_version,
         decisionSource=final_decision,
         analysisMode=fusion.analysis_mode,
-        source={
-            "kind": "file",
-            "fileName": file.filename or "unknown",
-            "fileSizeBytes": file_size,
-            "mimeType": content_type,
-        },
+        source=source,
         features=AnalysisFeatures(
             spectralRegularity=fusion.spectral_regularity,
             temporalPatterns=fusion.temporal_patterns,
@@ -499,76 +495,212 @@ async def _analyze_file(
             artificialIndicators=fusion.indicators,
         ),
         audioInfo=AudioInfo(
-            duration=round(features.duration_sec, 2),
-            sampleRate=features.sample_rate,
-            bitrate=estimated_bitrate,
-            format=content_type.replace("audio/", ""),
-            channels=1,
+            duration=round(duration, 2),
+            analysedSec=round(features.duration_sec, 2),
+            sampleRate=orig_sr or features.sample_rate,
+            bitrate=bitrate,
+            format=Path(filename).suffix.lstrip(".").lower() or content_type.replace("audio/", ""),
+            channels=orig_channels or 0,
         ),
         vocalAnalysis=vocal_response,
         xai=xai_payload,
         towerScores=tower_scores,
         metaClassifier=meta_payload,
+        layers=layers,
     )
 
-    return AnalyzeResponse(result=result, warnings=warnings, errors=[])
+    return AnalyzeResponse(result=result, warnings=_unique(warnings), errors=[])
 
 
-def _fallback_analysis(
-    request_id: str,
-    file: UploadFile,
-    content: bytes,
-    file_size: int,
-    start_time: float,
-    reason: str,
+# ── sources ───────────────────────────────────────────────────────────
+
+def _download_youtube_sync(
+    normalized_url: str, video_id: str, start_sec: Optional[int],
+) -> Tuple[bytes, str, Optional[str], Optional[float], List[str]]:
+    """Download the opening of a video (from ?t= if given) to memory."""
+    start = float(start_sec or 0)
+    with tempfile.TemporaryDirectory() as tmp:
+        downloader = YouTubeDownloader(
+            Path(tmp),
+            clip=(start, start + YOUTUBE_CLIP_SEC),
+            max_filesize=YOUTUBE_MAX_BYTES,
+        )
+        result = downloader.download(normalized_url, video_id)
+        content = result.file_path.read_bytes()
+        return content, result.file_path.suffix.lower(), result.title, result.duration_sec, result.warnings
+
+
+async def _analyze_url(
+    request_id: str, url: str, start_time: float, progress: Progress,
 ) -> AnalyzeResponse:
-    """Fallback to preview model when real analysis fails."""
-    logger.warning(f"[{request_id}] Falling back to preview model: {reason}")
+    """Download a YouTube link's audio and run the same pipeline as a file."""
+    try:
+        parsed = parse_youtube_url(url)
+    except ValueError as e:
+        logger.warning(f"[{request_id}] Invalid URL: {e}")
+        progress.set("download", STEP_FAILED)
+        return AnalyzeResponse(errors=["invalid_youtube_url"])
+
+    progress.set("download", STEP_RUNNING)
+    loop = asyncio.get_running_loop()
+    try:
+        content, ext, title, duration, dl_warnings = await loop.run_in_executor(
+            _download_pool, _download_youtube_sync,
+            parsed.normalized_url, parsed.video_id, parsed.start_time_sec,
+        )
+    except YouTubeDownloadError as exc:
+        progress.set("download", STEP_FAILED)
+        logger.warning(f"[{request_id}] YouTube download failed ({exc.error_code}): {exc}")
+        errors = ["youtube_analysis_failed"]
+        if exc.error_code == "youtube_authentication_required":
+            errors.insert(0, "youtube_authentication_required")
+        return AnalyzeResponse(errors=errors, warnings=_unique(exc.warnings))
+    except Exception as exc:
+        progress.set("download", STEP_FAILED)
+        logger.error(f"[{request_id}] YouTube download error: {exc}", exc_info=True)
+        return AnalyzeResponse(errors=["youtube_analysis_failed"])
+    progress.set("download", STEP_DONE)
 
-    fingerprint = f"file_{file.filename}_{file_size}"
-    preview = preview_model.analyze(fingerprint)
-    features_data = preview.get("features", {})
-    processing_time = round(time.monotonic() - start_time, 3)
+    if len(content) < MIN_UPLOAD_BYTES:
+        return AnalyzeResponse(errors=["youtube_analysis_failed"])
 
-    result = AnalysisResult(
-        isAIGenerated=preview["is_ai_generated"],
-        confidence=preview["confidence"],
-        processingTime=processing_time,
-        modelVersion=preview["model_version"],
-        decisionSource="preview",
-        analysisMode="preview",
-        source={
-            "kind": "file",
-            "fileName": file.filename or "unknown",
-            "fileSizeBytes": file_size,
-            "mimeType": file.content_type or "audio/unknown",
-        },
-        features=AnalysisFeatures(
-            spectralRegularity=features_data.get(
-                "spectral_regularity", 0.5,
-            ),
-            temporalPatterns=features_data.get(
-                "temporal_patterns", 0.5,
-            ),
-            harmonicStructure=features_data.get(
-                "harmonic_structure", 0.5,
-            ),
-            artificialIndicators=preview.get(
-                "indicators", [],
-            ),
-        ),
-        audioInfo=AudioInfo(
-            duration=0.0,
-            sampleRate=44100,
-            bitrate=128,
-            format=(
-                file.content_type or "audio/unknown"
-            ).replace("audio/", ""),
-        ),
+    source: Dict[str, Any] = {
+        "kind": "youtube",
+        "url": url,
+        "normalizedUrl": parsed.normalized_url,
+        "videoId": parsed.video_id,
+        "startTimeSec": parsed.start_time_sec,
+    }
+    if title:
+        source["title"] = title
+
+    return await _run_pipeline(
+        request_id, content,
+        filename=f"{parsed.video_id}{ext}",
+        content_type=_EXT_MIME.get(ext, "audio/wav"),
+        source=source,
+        start_time=start_time,
+        progress=progress,
+        warnings=dl_warnings,
+        total_duration=duration,
     )
 
-    return AnalyzeResponse(
-        result=result,
-        warnings=["analysis_fallback_preview", f"reason: {reason}"],
-        errors=[],
-    )
+
+Work = Callable[[Progress], Awaitable[AnalyzeResponse]]
+
+
+async def _plan(
+    request_id: str,
+    source_type: str,
+    url: Optional[str],
+    file: Optional[UploadFile],
+) -> Tuple[Optional[AnalyzeResponse], Optional[Tuple[List[str], Work]]]:
+    """Validate a request. Returns an immediate error response, or the
+    steps plus the work to run (uploads are read here, while the request
+    is still open)."""
+    start_time = time.monotonic()
+    logger.info(f"[{request_id}] Analysis: sourceType={source_type}")
+
+    if source_type in URL_SOURCES:
+        if not url:
+            return AnalyzeResponse(errors=["missing_url"]), None
+        return None, (URL_STEPS, lambda p: _analyze_url(request_id, url, start_time, p))
+
+    if source_type == "file":
+        if not file:
+            return AnalyzeResponse(errors=["missing_file"]), None
+        content, error, content_type = await _read_upload(file)
+        if error or content is None:
+            return AnalyzeResponse(errors=[error or "invalid_file_type"]), None
+        filename = file.filename or "audio"
+        source = {
+            "kind": "file",
+            "fileName": filename,
+            "fileSizeBytes": len(content),
+            "mimeType": content_type,
+        }
+        return None, (FILE_STEPS, lambda p: _run_pipeline(
+            request_id, content,
+            filename=filename, content_type=content_type,
+            source=source, start_time=start_time, progress=p,
+        ))
+
+    if source_type in ("spotify", "apple"):
+        return AnalyzeResponse(errors=["unsupported_source"]), None
+    return AnalyzeResponse(errors=["invalid_source_type"]), None
+
+
+# ── endpoints ─────────────────────────────────────────────────────────
+
+@router.post(
+    "/api/analyze",
+    response_model=AnalyzeResponse,
+    dependencies=[Depends(_analyze_rate_limit)],
+)
+async def analyze(
+    sourceType: str = Form(...),
+    url: Optional[str] = Form(None),
+    file: Optional[UploadFile] = File(None)
+) -> AnalyzeResponse:
+    """Unified analysis endpoint (waits for the result)."""
+    request_id = str(uuid.uuid4())[:8]
+    try:
+        immediate, plan = await _plan(request_id, sourceType, url, file)
+        if immediate is not None or plan is None:
+            return immediate or AnalyzeResponse(errors=["internal_error"])
+        steps, work = plan
+        return await work(Progress(steps))
+    except Exception as e:
+        logger.error(f"[{request_id}] Error: {e}", exc_info=True)
+        return AnalyzeResponse(errors=["internal_error"])
+
+
+@router.post(
+    "/api/analyze/jobs",
+    dependencies=[Depends(_analyze_rate_limit)],
+)
+async def start_analysis_job(
+    sourceType: str = Form(...),
+    url: Optional[str] = Form(None),
+    file: Optional[UploadFile] = File(None)
+) -> JSONResponse:
+    """Start an analysis in the background; poll GET /api/analyze/jobs/{jobId}."""
+    request_id = str(uuid.uuid4())[:8]
+    immediate, plan = await _plan(request_id, sourceType, url, file)
+    if immediate is not None or plan is None:
+        response = immediate or AnalyzeResponse(errors=["internal_error"])
+        return JSONResponse(status_code=200, content={
+            "jobId": None, "sourceType": sourceType, "status": "error",
+            "steps": [], "elapsedSec": 0, "response": response.model_dump(),
+        })
+
+    if job_store.running_count() >= MAX_RUNNING_JOBS:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail={"code": "server_busy", "message": "Too many analyses running. Try again shortly."},
+        )
+
+    steps, work = plan
+
+    async def run(progress: Progress) -> Dict[str, Any]:
+        try:
+            return (await work(progress)).model_dump()
+        except Exception as e:
+            logger.error(f"[{request_id}] Job error: {e}", exc_info=True)
+            return AnalyzeResponse(errors=["internal_error"]).model_dump()
+
+    job = job_store.start(sourceType, steps, run)
+    logger.info(f"[{request_id}] Job {job.id} started")
+    return JSONResponse(status_code=202, content=job.to_dict())
+
+
+@router.get("/api/analyze/jobs/{job_id}")
+async def get_analysis_job(job_id: str) -> Dict[str, Any]:
+    """Per-step progress of a job; `response` is filled once it settles."""
+    job = job_store.get(job_id)
+    if job is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail={"code": "job_not_found", "message": "Unknown or expired job."},
+        )
+    return job.to_dict()

app/routes/analyze_schemas.py

@@ -45,10 +45,14 @@ class AudioInfo(BaseModel):
     """Audio file information."""
 
     duration: float = 0.0
+    # Seconds the models actually listened to (feature extraction reads the opening).
+    analysedSec: Optional[float] = None
     sampleRate: int = 44100
-    bitrate: int = 128
+    # 0 when unknown (container soundfile can't read).
+    bitrate: int = 0
     format: str = "unknown"
-    channels: int = 2
+    # 0 when unknown.
+    channels: int = 0
 
 
 class ConfidenceBandModel(BaseModel):

app/routes/health.py

@@ -4,12 +4,43 @@ Health check route for the backend service.
 
 from __future__ import annotations
 
+from typing import Any, Dict
+
 from fastapi import APIRouter
 
 from ..services.external_clients import service_status
+from ..services.logging_config import get_logger
 
 
 router = APIRouter()
+logger = get_logger(__name__)
+
+
+def _model_status() -> Dict[str, Any]:
+    """What the AURIS analysis actually has loaded right now.
+
+    Imported lazily: the analyze module builds the services at import
+    time, and health must answer even if that failed.
+    """
+    try:
+        from .analyze import (
+            clap_service,
+            meta_classifier_service,
+            wav2vec2_service,
+            xai_service,
+        )
+        from ..services.analysis_jobs import job_store
+
+        return {
+            "xai": xai_service.status(),
+            "wav2vec2": wav2vec2_service.status(),
+            "clap": clap_service.status(),
+            "meta": meta_classifier_service.status(),
+            "jobs": {"running": job_store.running_count()},
+        }
+    except Exception as e:  # noqa: BLE001
+        logger.warning(f"Model status unavailable: {e}")
+        return {"error": "model_status_unavailable"}
 
 
 @router.get("/api/health")
@@ -17,4 +48,5 @@ async def health() -> dict:
     return {
         "status": "ok",
         "services": service_status(),
+        "models": _model_status(),
     }

app/services/clap_detector.py

@@ -95,6 +95,14 @@ class CLAPDetectorService:
         self._classifier_svm: Optional[object] = None
         self._initialized = False
 
+    def status(self) -> dict:
+        """Whether the CLAP embedding model loaded (it initialises on first use);
+        without it the layer falls back to a spectral heuristic."""
+        return {
+            "initialized": self._initialized,
+            "embeddingModel": self._model is not None,
+        }
+
     def _ensure_initialized(self) -> bool:
         """Initialize CLAP model on first call."""
         if self._initialized:

app/services/inference_xai.py

@@ -474,6 +474,18 @@ class XAIInferenceService:
             logger.error(f"Failed to load XAI service: {e}", exc_info=True)
             self.available = False
 
+    def status(self) -> Dict[str, Any]:
+        """What actually loaded, for /api/health."""
+        return {
+            "available": self.available,
+            "bestModel": self.training_results.get("_best_model") if isinstance(self.training_results, dict) else None,
+            "threshold": round(float(self.threshold), 6) if self.available else None,
+            "features": len(self.feature_cols) if self.available else 0,
+            "ensembleLoaded": len(self.ensemble_models),
+            "ensembleTotal": len(_ML_MODEL_FILES) + len(_DL_MODEL_FILES),
+            "shap": self.shap_explainer is not None,
+        }
+
     def predict(
         self,
         features: AudioFeatures,

app/services/meta_classifier.py

@@ -83,6 +83,10 @@ class MetaClassifierService:
         self._initialized = False
         self._trained = False
 
+    def status(self) -> dict:
+        """Whether the trained stacking model is in use (it loads on first use)."""
+        return {"initialized": self._initialized, "trained": self._trained}
+
     def _ensure_loaded(self) -> bool:
         """Load trained meta-classifier if available."""
         if self._initialized:

app/services/wav2vec2_detector.py

@@ -56,6 +56,14 @@ class Wav2Vec2DetectorService:
         self._initialized = False
         self._model_path = model_path or _MODEL_PATH
 
+    def status(self) -> dict:
+        """Weights on disk / model in memory (it loads on first use)."""
+        return {
+            "weights": self._model_path.exists(),
+            "loaded": self._model is not None,
+            "initialized": self._initialized,
+        }
+
     def _ensure_loaded(self) -> bool:
         """Lazy-load model on first use."""
         if self._initialized:

tests/test_analysis_jobs.py

@@ -0,0 +1,60 @@
+"""Tests for the background job store and the confidence tiers (no model inference)."""
+
+from __future__ import annotations
+
+import asyncio
+
+from app.services.analysis_jobs import FILE_STEPS, JobStore, Progress, STEP_DONE
+from app.services.inference_xai import XAIInferenceService
+
+
+def test_job_reports_steps_and_settles() -> None:
+    async def scenario() -> None:
+        store = JobStore()
+
+        async def work(progress: Progress):
+            progress.running("features", "vocals")
+            await asyncio.sleep(0.02)
+            progress.set("features", STEP_DONE)
+            return {"result": {"ok": True}, "warnings": [], "errors": []}
+
+        job = store.start("file", FILE_STEPS, work)
+        assert job.to_dict()["status"] == "running"
+        await asyncio.sleep(0.05)
+
+        snapshot = store.get(job.id).to_dict()
+        assert snapshot["status"] == "done"
+        steps = {s["id"]: s for s in snapshot["steps"]}
+        assert steps["features"]["state"] == "done" and "seconds" in steps["features"]
+        assert steps["vocals"]["state"] == "running"
+        assert steps["meta"]["state"] == "pending"
+
+    asyncio.run(scenario())
+
+
+def test_failed_job_settles_as_error() -> None:
+    async def scenario() -> None:
+        store = JobStore()
+
+        async def boom(progress: Progress):
+            raise RuntimeError("boom")
+
+        job = store.start("file", FILE_STEPS, boom)
+        await asyncio.sleep(0.01)
+        assert job.status == "error"
+        assert job.response == {"result": None, "warnings": [], "errors": ["internal_error"]}
+
+    asyncio.run(scenario())
+
+
+def test_confidence_band_is_measured_from_the_threshold() -> None:
+    svc = XAIInferenceService.__new__(XAIInferenceService)
+    svc.threshold = 0.431577
+    # Just above the threshold: an AI verdict, but barely.
+    assert svc._confidence_band(0.45).tier == "uncertain"
+    # Below 0.5 yet clearly under the threshold's human side? No — close to it.
+    assert svc._confidence_band(0.40).tier == "uncertain"
+    assert svc._confidence_band(0.97).tier == "very_strong"
+    assert svc._confidence_band(0.02).tier == "very_strong"
+    band = svc._confidence_band(0.431577)
+    assert band.margin == 0.0

tests/test_analyze.py

@@ -105,10 +105,10 @@ def test_analyze_file_too_small(
     assert "file_too_small" in data["errors"]
 
 
-def test_analyze_file_fallback_on_bad_audio(
+def test_analyze_file_bad_audio_is_an_error(
     client: TestClient,
 ) -> None:
-    """Non-parseable audio falls back to preview model."""
+    """Undecodable audio returns an error, never an invented result."""
     audio_content = b"\x00" * 2048
     fake_audio = io.BytesIO(audio_content)
     response = client.post(
@@ -120,13 +120,75 @@ def test_analyze_file_fallback_on_bad_audio(
     )
     assert response.status_code == 200
     data = response.json()
-    assert data["result"] is not None
-    result = data["result"]
-    assert "isAIGenerated" in result
-    assert "confidence" in result
-    # Bad audio triggers fallback to preview
-    assert result["analysisMode"] == "preview"
-    assert result["decisionSource"] == "preview"
+    assert data["result"] is None
+    assert data["errors"][0] in {"audio_decode_failed", "audio_silent", "audio_too_short"}
+
+
+def test_analyze_accepts_audio_extension_with_video_mime(
+    client: TestClient,
+) -> None:
+    """A .webm labelled video/webm by the browser is audio, not an invalid type."""
+    response = client.post(
+        "/api/analyze",
+        data={"sourceType": "file"},
+        files={
+            "file": ("take.webm", io.BytesIO(b"\x1a" * 100), "video/webm"),
+        },
+    )
+    assert response.status_code == 200
+    data = response.json()
+    assert data["errors"] == ["file_too_small"]
+
+
+def test_jobs_rejects_invalid_source_immediately(
+    client: TestClient,
+) -> None:
+    """Validation errors come back at once, without starting a job."""
+    response = client.post(
+        "/api/analyze/jobs",
+        data={"sourceType": "unknown"},
+    )
+    assert response.status_code == 200
+    data = response.json()
+    assert data["jobId"] is None
+    assert data["status"] == "error"
+    assert "invalid_source_type" in data["response"]["errors"]
+
+
+def test_jobs_missing_url(
+    client: TestClient,
+) -> None:
+    response = client.post(
+        "/api/analyze/jobs",
+        data={"sourceType": "youtube"},
+    )
+    assert response.status_code == 200
+    assert "missing_url" in response.json()["response"]["errors"]
+
+
+def test_jobs_unknown_id_is_404(
+    client: TestClient,
+) -> None:
+    response = client.get("/api/analyze/jobs/does-not-exist")
+    assert response.status_code == 404
+    assert response.json()["detail"]["code"] == "job_not_found"
+
+
+def test_rate_limit_key_uses_forwarded_client() -> None:
+    """Behind the Space proxy each visitor gets their own bucket."""
+    from starlette.requests import Request
+
+    from app.routes.analyze import _client_ip
+
+    def make(headers):
+        return Request({
+            "type": "http",
+            "headers": [(k.encode(), v.encode()) for k, v in headers.items()],
+            "client": ("10.0.0.1", 1234),
+        })
+
+    assert _client_ip(make({"x-forwarded-for": "203.0.113.7, 10.0.0.1"})) == "203.0.113.7"
+    assert _client_ip(make({})) == "10.0.0.1"
 
 
 def _make_wav(

tests/test_health.py

@@ -22,3 +22,13 @@ def test_health_endpoint(client: TestClient) -> None:
     data = response.json()
     assert data["status"] == "ok"
     assert "services" in data
+
+
+def test_health_reports_model_inventory(client: TestClient) -> None:
+    """Health says what actually loaded, not just that the process is up."""
+    data = client.get("/api/health").json()
+    models = data["models"]
+    assert "xai" in models and "available" in models["xai"]
+    assert models["xai"]["ensembleTotal"] == 11
+    assert "wav2vec2" in models and "clap" in models and "meta" in models
+    assert models["jobs"]["running"] >= 0