Add Nemotron 3.5 Transcribe evaluation scripts

recipes/nemotron_transcribe/evaluate.py

@@ -0,0 +1,148 @@
+"""FFASR custom evaluator for NVIDIA Nemotron 3.5 Transcribe (BF16).
+
+The setup script provides a separate NeMo/vLLM environment. This module starts
+one local server before FFASR calls evaluate() for each sample. No external
+inference service receives benchmark audio.
+"""
+
+import atexit
+import base64
+import json
+import os
+from pathlib import Path
+import subprocess
+import time
+import urllib.error
+import urllib.request
+
+MODEL_ID = "nvidia/NVIDIA-Nemotron-3.5-Transcribe-30B-A4B"
+PROMPT = (
+    "Audio transcription. Language: as spoken. Wording: verbatim, including "
+    "fillers and repetitions. Formatting: punctuation and capitalization. "
+    "Output: transcript only."
+)
+_BASE_URL = "http://127.0.0.1:8000"
+_HTTP = urllib.request.build_opener(urllib.request.ProxyHandler({}))
+_SERVER = None
+_SERVER_LOG = None
+
+
+def _stop_server():
+    if _SERVER is not None and _SERVER.poll() is None:
+        _SERVER.terminate()
+        try:
+            _SERVER.wait(timeout=30)
+        except subprocess.TimeoutExpired:
+            _SERVER.kill()
+            _SERVER.wait()
+    if _SERVER_LOG is not None:
+        _SERVER_LOG.close()
+
+
+def _check_server():
+    with _HTTP.open(_BASE_URL + "/v1/models", timeout=5) as response:
+        models = json.load(response).get("data", [])
+    if not any(item.get("id") == MODEL_ID for item in models):
+        raise RuntimeError("The local server does not serve " + MODEL_ID)
+
+
+def _start_server():
+    global _SERVER, _SERVER_LOG
+    # Use external mode only for a server already running on this same machine.
+    if os.environ.get("NEMOTRON_FFASR_SERVER_MODE") == "external":
+        _check_server()
+        return
+
+    executable = os.environ.get("NEMOTRON_FFASR_VLLM")
+    checkpoint = os.environ.get("NEMOTRON_FFASR_CHECKPOINT")
+    if not executable or not checkpoint:
+        raise RuntimeError("Run setup.sh first through FFASR's setup field.")
+    try:
+        _check_server()
+    except (OSError, ValueError, RuntimeError):
+        pass
+    else:
+        raise RuntimeError("Port 8000 already serves this model. Use external mode explicitly.")
+
+    env = os.environ.copy()
+    # Do not expose the parent job's virtual environment to the server.
+    env.pop("VIRTUAL_ENV", None)
+    env.pop("PYTHONPATH", None)
+    env["VLLM_PLUGINS"] = "nemo_speechlm"
+    env["VLLM_MAX_AUDIO_DECODE_DURATION_S"] = "3600"
+    env["VLLM_MAX_AUDIO_CLIP_FILESIZE_MB"] = "180"
+    env["VLLM_USE_V2_MODEL_RUNNER"] = "0"
+    command = [
+        executable, "serve", checkpoint,
+        "--host", "127.0.0.1", "--port", "8000",
+        "--served-model-name", MODEL_ID,
+        "--trust-remote-code", "--dtype", "bfloat16",
+        "--gpu-memory-utilization", "0.94",
+        "--max-model-len", "65536", "--max-num-batched-tokens", "4096",
+        "--max-num-seqs", "192", "--enable-chunked-prefill",
+        "--no-enable-prefix-caching", "--use-replayssm",
+        "--mamba-backend", "flashinfer", "--mamba-cache-mode", "none",
+        "--mm-processor-cache-gb", "0", "--no-async-scheduling",
+        "--limit-mm-per-prompt", json.dumps({"audio": {"count": 6, "length": 120000000}}),
+        "--speculative-config", json.dumps({"method": "mtp", "num_speculative_tokens": 8}),
+        "--reasoning-parser", "nemotron_v3",
+        "--trust-request-mm-kwargs",
+        "--middleware", "nemo.collections.speechlm2.vllm.salm.ctc_serving.ctc_timestamp_middleware",
+    ]
+    log_path = Path("/tmp/nemotron_ffasr_vllm.log")
+    _SERVER_LOG = log_path.open("w", encoding="utf-8")
+    _SERVER = subprocess.Popen(command, env=env, stdout=_SERVER_LOG, stderr=subprocess.STDOUT)
+    atexit.register(_stop_server)
+    deadline = time.monotonic() + 1800
+    while time.monotonic() < deadline:
+        if _SERVER.poll() is not None:
+            tail = log_path.read_text(errors="replace")[-8000:]
+            raise RuntimeError("vLLM stopped during startup:\n" + tail)
+        try:
+            _check_server()
+            return
+        except (OSError, ValueError):
+            time.sleep(2)
+    _stop_server()
+    raise TimeoutError("vLLM startup exceeded 30 minutes. See " + str(log_path))
+
+
+def evaluate(file: Path) -> str:
+    """Return the raw transcript. FFASR applies its standard normalization."""
+    audio = base64.b64encode(Path(file).read_bytes()).decode("ascii")
+    payload = {
+        "model": MODEL_ID,
+        "messages": [{"role": "user", "content": [
+            {"type": "text", "text": PROMPT},
+            {"type": "audio_url", "audio_url": {"url": "data:audio/wav;base64," + audio}},
+        ]}],
+        "temperature": 0,
+        "top_p": 1.0,
+        "seed": 0,
+        "max_tokens": 16384,
+        "chat_template_kwargs": {"enable_thinking": False},
+        "skip_special_tokens": False,
+        "spaces_between_special_tokens": False,
+    }
+    request = urllib.request.Request(
+        _BASE_URL + "/v1/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json"},
+    )
+    try:
+        with _HTTP.open(request, timeout=600) as response:
+            result = json.load(response)
+    except urllib.error.HTTPError as exc:
+        detail = exc.read().decode("utf-8", errors="replace")[:2000]
+        raise RuntimeError(f"vLLM request failed ({exc.code}): {detail}") from exc
+    choice = result["choices"][0]
+    if choice.get("finish_reason") != "stop":
+        raise RuntimeError("Incomplete transcript: " + str(choice.get("finish_reason")))
+    text = choice["message"].get("content")
+    if not isinstance(text, str):
+        raise RuntimeError("The server did not return a transcript string.")
+    return text.strip()
+
+
+# Load the server before FFASR starts sample timing.
+_start_server()

recipes/nemotron_transcribe/setup.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Paste this file into FFASR's one-time setup field.
+# Run on Linux with one H100 80 GB and sufficient disk space for the BF16 model.
+: "${FFASR_ENV_FILE:?FFASR must provide its setup environment file}"
+task_root=$(mktemp -d /tmp/nemotron_ffasr.XXXXXX)
+task_repo="$task_root/Speech"
+task_venv="$task_repo/.venv"
+task_model="$task_root/checkpoint"
+
+git clone https://github.com/NVIDIA-NeMo/Speech.git "$task_repo"
+git -C "$task_repo" checkout 040574f7ffa3bb0ae4b92aa8424ecea454d03c91
+cd "$task_repo"
+uv venv --python 3.13 "$task_venv"
+UV_PROJECT_ENVIRONMENT="$task_venv" uv sync --extra asr --extra vllm --locked --no-install-project
+uv pip install --python "$task_venv/bin/python" --no-deps -e .
+uv pip install --python "$task_venv/bin/python" --no-deps av==19.0.2 opencv-python-headless==5.0.0.93
+"$task_venv/bin/python" -c 'import vllm; assert vllm.__version__ == "0.32.0", vllm.__version__'
+"$task_venv/bin/hf" download nvidia/NVIDIA-Nemotron-3.5-Transcribe-30B-A4B \
+    --revision ef905575a106fdad904f3b0ce661dccad654607d --local-dir "$task_model"
+
+printf 'NEMOTRON_FFASR_VLLM=%s\n' "$task_venv/bin/vllm" >> "$FFASR_ENV_FILE"
+printf 'NEMOTRON_FFASR_CHECKPOINT=%s\n' "$task_model" >> "$FFASR_ENV_FILE"

recipes/nemotron_transcribe/validation.txt

@@ -0,0 +1,12 @@
+Local validation completed on 28 October 2026.
+PASS: Python syntax.
+PASS: Shell syntax (bash -n setup.sh).
+PASS: Fixed NeMo Speech revision specifies vLLM 0.32.0.
+PASS: Mock server receives the correct model ID, prompt, decoding settings, and WAV bytes.
+PASS: Evaluator returns a transcript string.
+PASS: Evaluator rejects truncated output and missing transcript content.
+PASS: Mock process receives the BF16 and MTP8 server arguments.
+
+These checks used mock responses. No model inference was run.
+The Linux dependency installation, GPU memory use, startup time, WER, and speed
+remain untested. The files have not been submitted to FFASR or sent to Slack.