#!/usr/bin/env python3 """Download one pinned official Base model and make two CPU-only clone probes. This is a feasibility probe, not production material or speaker verification. It never edits references, uses GPUs, installs packages, or uses shared caches. """ from __future__ import annotations import argparse import hashlib import json import os from pathlib import Path import time import urllib.request import wave REPO = Path(__file__).resolve().parents[2] WORK = REPO / "data/hfd_third_party_multi_v5/work" MODEL_ID = "Qwen/Qwen3-TTS-12Hz-0.6B-Base" REVISION = "5d83992436eae1d760afd27aff78a71d676296fc" MODEL_DIR = WORK / "models/Qwen3-TTS-12Hz-0.6B-Base" OUTPUT = WORK / "tts_probe" RECIPES = REPO / "data/hfd_third_party_cross_v4/production/manifests/session_recipes.jsonl" PHRASES = { "zh": "今天的咖啡有一点苦,午饭倒是挺好吃的。", "en": "The coffee tastes a little bitter today, but lunch was really good.", } FILES = { "README.md", "config.json", "generation_config.json", "merges.txt", "model.safetensors", "preprocessor_config.json", "tokenizer_config.json", "vocab.json", "speech_tokenizer/config.json", "speech_tokenizer/configuration.json", "speech_tokenizer/model.safetensors", "speech_tokenizer/preprocessor_config.json", } def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def safe_output(path: Path) -> None: path.absolute().relative_to(WORK.absolute()) for parent in (path, *path.parents): if parent.is_symlink(): raise RuntimeError(f"Refusing symlink output: {parent}") if parent == REPO: break path.resolve().relative_to(WORK.resolve()) def write_json(path: Path, value: object) -> None: safe_output(path) path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + ".tmp") safe_output(temporary) temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") temporary.replace(path) def fetch_metadata() -> dict: url = f"https://huggingface.co/api/models/{MODEL_ID}/revision/{REVISION}?blobs=true" with urllib.request.urlopen(url, timeout=30) as response: data = json.load(response) if data["sha"] != REVISION or data["id"] != MODEL_ID: raise RuntimeError("Official model revision mismatch") selected = [item for item in data["siblings"] if item["rfilename"] in FILES] if {item["rfilename"] for item in selected} != FILES: raise RuntimeError("Pinned model files incomplete") return {"model_id": MODEL_ID, "revision": REVISION, "files": selected, "required_bytes": sum(item["size"] for item in selected), "source_api": url} def verify_file(path: Path, item: dict) -> str: if path.stat().st_size != item["size"]: raise RuntimeError(f"Size mismatch: {path}") digest = sha256(path) if item.get("lfs"): if digest != item["lfs"]["sha256"]: raise RuntimeError(f"Official LFS SHA256 mismatch: {path}") else: blob_hash = hashlib.sha1() blob_hash.update(f"blob {item['size']}\0".encode()) with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): blob_hash.update(chunk) if blob_hash.hexdigest() != item["blobId"]: raise RuntimeError(f"Official Git blob hash mismatch: {path}") return digest def download() -> None: manifest = fetch_metadata() print(json.dumps({"revision": REVISION, "required_bytes": manifest["required_bytes"]}), flush=True) safe_output(MODEL_DIR) MODEL_DIR.mkdir(parents=True, exist_ok=True) write_json(MODEL_DIR / "official_download_plan.json", manifest) completed = [] for item in manifest["files"]: name = item["rfilename"] path = MODEL_DIR / name safe_output(path) path.parent.mkdir(parents=True, exist_ok=True) started = time.monotonic() if not path.exists(): temporary = path.with_suffix(path.suffix + ".part") safe_output(temporary) url = f"https://huggingface.co/{MODEL_ID}/resolve/{REVISION}/{name}" print(f"download {name} bytes={item['size']}", flush=True) with urllib.request.urlopen(url, timeout=60) as response, temporary.open("wb") as handle: for chunk in iter(lambda: response.read(8 * 1024 * 1024), b""): handle.write(chunk) verify_file(temporary, item) temporary.replace(path) digest = verify_file(path, item) completed.append({**item, "sha256": digest, "download_or_verify_seconds": time.monotonic() - started}) print(f"verified {name} sha256={digest}", flush=True) write_json(MODEL_DIR / "verified_download.json", {**manifest, "files": completed, "complete": True}) def references() -> list[dict]: selected: dict[str, dict] = {} with RECIPES.open(encoding="utf-8") as handle: for line in handle: recipe = json.loads(line) language = recipe["session_language"] if recipe["split"] != "train" or language in selected: continue for turn in recipe["turns"]: path = Path(turn["tts_audio_path"]) with wave.open(str(path), "rb") as audio: duration = audio.getnframes() / audio.getframerate() if not 3 <= duration <= 10: continue digest = sha256(path) if digest != turn["tts_audio_sha256"]: raise RuntimeError(f"Reference changed: {path}") selected[language] = { "language": language, "reference_path": str(path), "reference_sha256": digest, "reference_duration_seconds": duration, "reference_text": turn["instruction_text"], "voice_name": turn["voice_name"], "reference_source_key": turn["tts_source_key"], "split": "train", "session_id": recipe["session_id"], "generated_text": PHRASES[language], } break if len(selected) == 2: return [selected[language] for language in ("zh", "en")] raise RuntimeError("No suitable >=3 second Chinese and English references") def probe(threads: int) -> None: if not 1 <= threads <= 8: raise ValueError("CPU thread count must be 1..8") os.environ["CUDA_VISIBLE_DEVICES"] = "" os.environ["HIP_VISIBLE_DEVICES"] = "" os.environ["OMP_NUM_THREADS"] = str(threads) os.environ["MKL_NUM_THREADS"] = str(threads) os.environ["OPENBLAS_NUM_THREADS"] = str(threads) os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["HF_HUB_OFFLINE"] = "1" os.environ["TRANSFORMERS_OFFLINE"] = "1" os.environ["HF_HOME"] = str(WORK / "models/hf_cache") os.environ["HF_HUB_CACHE"] = str(WORK / "models/hf_cache/hub") os.environ["XDG_CACHE_HOME"] = str(WORK / "models/cache") safe_output(OUTPUT) OUTPUT.mkdir(parents=True, exist_ok=True) if (OUTPUT / "probe_result.json").exists(): raise RuntimeError("Probe result exists; refusing implicit overwrite") manifest_path = MODEL_DIR / "verified_download.json" manifest = json.loads(manifest_path.read_text()) if not manifest["complete"] or manifest["revision"] != REVISION: raise RuntimeError("Model download is not verified") for item in manifest["files"]: if verify_file(MODEL_DIR / item["rfilename"], item) != item["sha256"]: raise RuntimeError("Local verified model checksum mismatch") chosen = references() result = { "format": "hfd_same_voice_cpu_feasibility_probe_v1", "model_id": MODEL_ID, "model_revision": REVISION, "model_manifest_sha256": sha256(manifest_path), "device": "cpu", "cpu_threads": threads, "dtype": "float32", "seed": 26092807, "status": "starting", "speaker_verification": "not_performed", "human_listening_review": "pending", "production_asset": False, "samples": chosen, } write_json(OUTPUT / "probe_plan.json", result) started = time.monotonic() try: import numpy as np import torch import soundfile as sf from qwen_tts import Qwen3TTSModel torch.set_num_threads(threads) torch.set_num_interop_threads(1) torch.manual_seed(result["seed"]) model = Qwen3TTSModel.from_pretrained(str(MODEL_DIR), device_map="cpu", dtype=torch.float32, attn_implementation="eager", local_files_only=True) if any(parameter.device.type != "cpu" for parameter in model.model.parameters()): raise RuntimeError("Model unexpectedly not entirely on CPU") result["load_seconds"] = time.monotonic() - started print(f"model loaded on CPU seconds={result['load_seconds']:.2f}", flush=True) for sample in chosen: begin = time.monotonic() with torch.inference_mode(): prompt = model.create_voice_clone_prompt(ref_audio=sample["reference_path"], ref_text=sample["reference_text"], x_vector_only_mode=False) wavs, rate = model.generate_voice_clone(text=sample["generated_text"], language="Chinese" if sample["language"] == "zh" else "English", voice_clone_prompt=prompt, non_streaming_mode=True, max_new_tokens=128, do_sample=True) values = np.asarray(wavs[0], dtype=np.float32).reshape(-1) if not values.size or not np.isfinite(values).all(): raise RuntimeError("Invalid generated waveform") path = OUTPUT / f"{sample['language']}_reference_clone_probe.wav" safe_output(path) if path.exists(): raise RuntimeError(f"Probe waveform already exists: {path}") sf.write(path, values, int(rate), subtype="PCM_16") sample.update({"output_path": str(path), "output_sha256": sha256(path), "sample_rate": int(rate), "duration_seconds": len(values) / rate, "peak": float(np.max(np.abs(values))), "generation_seconds": time.monotonic() - begin}) print(json.dumps(sample, ensure_ascii=False), flush=True) if sha256(Path(sample["reference_path"])) != sample["reference_sha256"]: raise RuntimeError("Read-only reference changed during probe") result["status"] = "cpu_generation_completed_pending_review" except Exception as error: result["status"] = "failed" result["error"] = f"{type(error).__name__}: {error}" raise finally: result["total_seconds"] = time.monotonic() - started write_json(OUTPUT / "probe_result.json", result) def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("mode", choices=("inspect", "download", "probe")) parser.add_argument("--threads", type=int, default=8) args = parser.parse_args() if args.mode == "inspect": print(json.dumps(fetch_metadata(), ensure_ascii=False, indent=2)) elif args.mode == "download": download() else: probe(args.threads) if __name__ == "__main__": main()