#!/usr/bin/env python3 """Generate reference-bound asides on CPU; never automatically mark QA eligible.""" from __future__ import annotations import argparse from collections import OrderedDict import hashlib import json import math import os from pathlib import Path import re import sys import time import wave sys.path.insert(0, str(Path(__file__).resolve().parent)) import probe_hfd_same_voice_tts as probe WORK = probe.WORK PLAN = WORK / "plan/tts_jobs.jsonl" ASSETS = WORK / "tts_assets" FORMAT = "hfd_reference_bound_aside_generated_v1" PUBLIC = Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt") def canonical_hash(value: object) -> str: return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()).hexdigest() def configure_cpu(threads: int) -> None: if not 1 <= threads <= 8: raise ValueError("CPU thread count must be 1..8") for name in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES"): os.environ[name] = "" for name in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS"): os.environ[name] = str(threads) os.environ.update(TOKENIZERS_PARALLELISM="false", HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", HF_HOME=str(WORK / "models/hf_cache"), HF_HUB_CACHE=str(WORK / "models/hf_cache/hub"), XDG_CACHE_HOME=str(WORK / "models/cache")) def read_jobs(path: Path) -> tuple[list[dict], str]: digest = probe.sha256(path) report = json.loads((path.parent / "preparation_report.json").read_text()) if report["output_hashes"][path.name] != digest: raise ValueError("Frozen TTS job manifest changed") jobs = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] if len({row["job_id"] for row in jobs}) != len(jobs): raise ValueError("Duplicate job IDs") for row in jobs: if not re.fullmatch(r"[0-9a-f]{64}", row["job_id"]): raise ValueError("Invalid job ID") if row["language"] not in {"zh", "en"} or row["split"] not in {"train", "val", "validation"}: raise ValueError("Unsupported language/split") if not row["text"].strip() or not row["reference_text"].strip(): raise ValueError("Missing synthesis/reference text") if row.get("voice_relation") != "reference_clone": raise ValueError("Only actual reference clones are supported") return jobs, digest def select_jobs(jobs: list[dict], offset: int, limit: int, shard_index: int, shard_count: int) -> list[dict]: if offset < 0 or limit < 0 or shard_count < 1 or not 0 <= shard_index < shard_count: raise ValueError("Invalid job slice/shard") selected = [job for index, job in enumerate(jobs) if index % shard_count == shard_index][offset:] return selected[:limit] if limit else selected def paths(job_id: str) -> tuple[Path, Path, Path]: return (ASSETS / "raw" / f"{job_id}.wav", ASSETS / "normalized" / f"{job_id}.wav", ASSETS / "provenance" / f"{job_id}.json") def verify_resume(job: dict, jobs_sha: str, model_sha: str) -> dict | None: raw, normalized, sidecar = paths(job["job_id"]) for path in (raw, normalized, sidecar): probe.safe_output(path) if not sidecar.exists(): if raw.exists() or normalized.exists(): raise ValueError(f"Uncommitted output collision: {job['job_id']}") return None record = json.loads(sidecar.read_text()) if (record["job_sha256"] != canonical_hash(job) or record["jobs_manifest_sha256"] != jobs_sha or record["model_manifest_sha256"] != model_sha or record["model_revision"] != probe.REVISION or record["status"] != "generated_pending_qa" or record["eligible"] is not False or record["generation"]["ended_with_eos"] is not True): raise ValueError(f"Existing provenance contract mismatch: {job['job_id']}") for key, expected_path in (("raw", raw), ("normalized", normalized)): if record[key]["path"] != str(expected_path) or probe.sha256(expected_path) != record[key]["sha256"]: raise ValueError(f"Existing {key} output hash mismatch: {job['job_id']}") return record def verified_model() -> tuple[dict, str]: path = probe.MODEL_DIR / "verified_download.json" manifest = json.loads(path.read_text()) if not manifest["complete"] or manifest["revision"] != probe.REVISION or manifest["model_id"] != probe.MODEL_ID: raise ValueError("Unexpected TTS model version") for item in manifest["files"]: if probe.verify_file(probe.MODEL_DIR / item["rfilename"], item) != item["sha256"]: raise ValueError("Verified model file drift") return manifest, probe.sha256(path) def verify_reference(job: dict) -> None: path = Path(job["reference_audio"]) if probe.sha256(path) != job["reference_sha256"]: raise ValueError(f"Read-only reference changed: {path}") with wave.open(str(path), "rb") as handle: if handle.getnframes() / handle.getframerate() < 3: raise ValueError("Reference must be >=3 seconds") def eos_flags(sequences: list[list[int]], eos_token_id: int) -> list[bool]: """A token-limit stop with no actual EOS is not successful synthesis.""" return [eos_token_id in sequence for sequence in sequences] class PromptCache: def __init__(self, capacity: int = 64): self.capacity = capacity self.values: OrderedDict[str, object] = OrderedDict() def get(self, key: str, factory): if key not in self.values: self.values[key] = factory() self.values.move_to_end(key) while len(self.values) > self.capacity: self.values.popitem(last=False) return self.values[key] def write_pcm(path: Path, values, sample_rate: int) -> dict: import numpy as np import soundfile as sf probe.safe_output(path) if path.exists(): raise ValueError(f"Refusing output overwrite: {path}") values = np.asarray(values, dtype=np.float32).reshape(-1) if not values.size or not np.isfinite(values).all() or np.max(np.abs(values)) >= 1: raise ValueError("Empty, nonfinite, or clipping TTS waveform") path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(".wav.tmp") probe.safe_output(temporary) sf.write(temporary, values, sample_rate, format="WAV", subtype="PCM_16") temporary.replace(path) return dict(path=str(path), sha256=probe.sha256(path), sample_rate=sample_rate, sample_count=int(values.size), duration_seconds=values.size / sample_rate, peak=float(np.max(np.abs(values))), rms=float(np.sqrt(np.mean(values.astype(np.float64) ** 2)))) def refresh_registry(jobs: list[dict], jobs_sha: str, model_sha: str) -> dict: records = [] for job in jobs: record = verify_resume(job, jobs_sha, model_sha) if record is not None: records.append(record) path = ASSETS / "registry.jsonl" probe.safe_output(path) path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(".jsonl.tmp") probe.safe_output(temporary) temporary.write_text("".join(json.dumps(r, sort_keys=True, ensure_ascii=False) + "\n" for r in records)) temporary.replace(path) summary = dict(format=FORMAT, planned_jobs=len(jobs), generated_jobs=len(records), eligible_jobs=0, status="generated_pending_qa", jobs_manifest_sha256=jobs_sha, registry_sha256=probe.sha256(path), model_manifest_sha256=model_sha, same_speaker_verified=False, human_review="pending") probe.write_json(ASSETS / "generation_summary.json", summary) return summary def generate(args) -> dict: configure_cpu(args.threads) import numpy as np import torch from scipy.signal import resample_poly from qwen_tts import Qwen3TTSModel torch.set_num_threads(args.threads) torch.set_num_interop_threads(1) jobs, jobs_sha = read_jobs(args.jobs) chosen = select_jobs(jobs, args.offset, args.limit, args.shard_index, args.shard_count) _, model_sha = verified_model() pending = [] for job in chosen: verify_reference(job) previous = verify_resume(job, jobs_sha, model_sha) if previous is None: pending.append(job) elif not args.resume: raise ValueError("Existing generation requires explicit --resume") if not pending: return refresh_registry(jobs, jobs_sha, model_sha) if not 1 <= args.batch_size <= 8 or args.max_new_tokens < 128: raise ValueError("Batch size must be 1..8 and max tokens >=128") started = time.monotonic() model = Qwen3TTSModel.from_pretrained(str(probe.MODEL_DIR), device_map="cpu", dtype=torch.float32, attn_implementation="eager", local_files_only=True) if any(p.device.type != "cpu" for p in model.model.parameters()): raise RuntimeError("Model unexpectedly not entirely on CPU") print(f"CPU model load seconds={time.monotonic() - started:.2f}", flush=True) prompt_cache = PromptCache(64) original_generate = model.model.talker.generate stops: list[bool] = [] lengths: list[int] = [] def capture_generate(*positional, **keywords): result = original_generate(*positional, **keywords) sequences = result.sequences.detach().cpu().tolist() stops[:] = eos_flags(sequences, int(keywords["eos_token_id"])) lengths[:] = [len(sequence) for sequence in sequences] return result model.model.talker.generate = capture_generate for start in range(0, len(pending), args.batch_size): batch = pending[start:start + args.batch_size] begin = time.monotonic() prompts = [] for job in batch: key = canonical_hash((job["reference_sha256"], job["reference_text"])) prompt = prompt_cache.get(key, lambda j=job: model.create_voice_clone_prompt( ref_audio=j["reference_audio"], ref_text=j["reference_text"], x_vector_only_mode=False)[0]) prompts.append(prompt) seed = int(canonical_hash([job["job_id"] for job in batch])[:8], 16) torch.manual_seed(seed) stops.clear() with torch.inference_mode(): wavs, rate = model.generate_voice_clone(text=[job["text"] for job in batch], language=["Chinese" if job["language"] == "zh" else "English" for job in batch], voice_clone_prompt=prompts, non_streaming_mode=True, max_new_tokens=args.max_new_tokens, do_sample=True) if len(wavs) != len(batch) or len(stops) != len(batch) or not all(stops): probe.write_json(ASSETS / "last_generation_failure.json", dict( job_ids=[job["job_id"] for job in batch], eos_flags=stops, status="rejected_missing_eos_or_batch_mismatch", max_new_tokens=args.max_new_tokens)) raise RuntimeError("Batch rejected: missing EOS (possible token truncation) or wrong size") if int(rate) != 24000: raise RuntimeError(f"Unexpected TTS sample rate: {rate}") elapsed = time.monotonic() - begin for index, (job, waveform) in enumerate(zip(batch, wavs)): verify_reference(job) raw, normalized, sidecar = paths(job["job_id"]) values = np.asarray(waveform, dtype=np.float32).reshape(-1) normalized_values = resample_poly(values, 2, 3) record = dict(format=FORMAT, job_id=job["job_id"], split=job["split"], language=job["language"], job=job, job_sha256=canonical_hash(job), jobs_manifest_sha256=jobs_sha, model_id=probe.MODEL_ID, model_revision=probe.REVISION, model_manifest_sha256=model_sha, status="generated_pending_qa", eligible=False, same_speaker_verified=False, human_review="pending", voice_relation="reference_clone", generation=dict(device="cpu", threads=args.threads, dtype="float32", attention="eager", seed=seed, batch_job_ids=[j["job_id"] for j in batch], batch_seconds=elapsed, seconds_per_job_average=elapsed / len(batch), ended_with_eos=stops[index], padded_sequence_tokens=lengths[index], max_new_tokens=args.max_new_tokens, generator_script_sha256=probe.sha256(Path(__file__)))) # Reject clipping before either output is committed. if (not values.size or not np.isfinite(values).all() or not np.isfinite(normalized_values).all() or np.max(np.abs(values)) >= 1 or np.max(np.abs(normalized_values)) >= 1): raise RuntimeError("TTS waveform invalid/clipping before commit") record["raw"] = write_pcm(raw, values, 24000) record["normalized"] = write_pcm(normalized, normalized_values, 16000) probe.write_json(sidecar, record) print(json.dumps(dict(job_id=job["job_id"], language=job["language"], text=job["text"], duration_seconds=record["raw"]["duration_seconds"], batch_seconds=elapsed, status=record["status"]), ensure_ascii=False), flush=True) return refresh_registry(jobs, jobs_sha, model_sha) def normalized_units(text: str, language: str) -> list[str]: if language == "zh": return re.findall(r"[\u4e00-\u9fffA-Za-z0-9]", text.lower()) return re.findall(r"[a-z0-9]+", text.lower()) def error_rate(expected: str, actual: str, language: str) -> float: left, right = normalized_units(expected, language), normalized_units(actual, language) if not left: raise ValueError("Empty normalized expected text") previous = list(range(len(right) + 1)) for i, token in enumerate(left, 1): current = [i] for j, candidate in enumerate(right, 1): current.append(min(previous[j] + 1, current[-1] + 1, previous[j - 1] + (token != candidate))) previous = current return previous[-1] / len(left) def qa(args) -> dict: configure_cpu(args.threads) import numpy as np import onnxruntime as ort import soundfile as sf from scipy.signal import resample_poly import torch import torchaudio from transformers import WhisperForConditionalGeneration, WhisperProcessor torch.set_num_threads(args.threads) torch.set_num_interop_threads(1) jobs, jobs_sha = read_jobs(args.jobs) chosen = select_jobs(jobs, args.offset, args.limit, args.shard_index, args.shard_count) manifest = probe.MODEL_DIR / "verified_download.json" model_sha = probe.sha256(manifest) records = [verify_resume(job, jobs_sha, model_sha) for job in chosen] if any(record is None for record in records): raise ValueError("QA selection includes ungenerated jobs") asr_path = PUBLIC / "openai/whisper-tiny" processor = WhisperProcessor.from_pretrained(str(asr_path), local_files_only=True) asr = WhisperForConditionalGeneration.from_pretrained(str(asr_path), local_files_only=True).to("cpu").eval() sessions = {} embeddings = {} evidence = [] def load_audio(path: str): samples, rate = sf.read(path, dtype="float32", always_2d=False) if samples.ndim == 2: samples = samples.mean(axis=1) if rate != 16000: divisor = math.gcd(rate, 16000) samples = resample_poly(samples, 16000 // divisor, rate // divisor) return samples.astype(np.float32) def speaker_embedding(path: str, language: str): key = (path, language) if key not in embeddings: values = load_audio(path) features = torchaudio.compliance.kaldi.fbank(torch.from_numpy(values).unsqueeze(0) * 32768, num_mel_bins=80, frame_length=25, frame_shift=10, sample_frequency=16000, dither=0, window_type="hamming", use_energy=False) features = features - features.mean(dim=0, keepdim=True) result = sessions[language].run(None, {"feats": features.numpy()[None].astype(np.float32)})[0][0] embeddings[key] = result / max(float(np.linalg.norm(result)), 1e-12) return embeddings[key] for language in sorted({job["language"] for job in chosen}): name = "wespeaker_cn_cnceleb_resnet34_LM.onnx" if language == "zh" else "wespeaker_en_voxceleb_resnet34_LM.onnx" path = PUBLIC / "vidproc/speaker_diarization/wespeaker-voxceleb-resnet34-LM" / name options = ort.SessionOptions() options.intra_op_num_threads = args.threads options.inter_op_num_threads = 1 sessions[language] = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"]) evidence.append(dict(model=str(path), sha256=probe.sha256(path), providers=sessions[language].get_providers())) results = [] for job, record in zip(chosen, records): begin = time.monotonic() verify_reference(job) language = job["language"] values = load_audio(record["normalized"]["path"]) inputs = processor(values, sampling_rate=16000, return_tensors="pt", return_attention_mask=True) with torch.inference_mode(): ids = asr.generate(inputs.input_features, attention_mask=inputs.attention_mask, language="chinese" if language == "zh" else "english", task="transcribe", max_new_tokens=128) transcript = processor.batch_decode(ids, skip_special_tokens=True)[0].strip() generated_embedding = speaker_embedding(record["normalized"]["path"], language) reference_embedding = speaker_embedding(job["reference_audio"], language) negatives = [candidate for candidate in chosen if candidate["language"] == language and candidate["reference_sha256"] != job["reference_sha256"]] comparisons = [dict(reference_source_key=other["reference_source_key"], cosine=float(np.dot(generated_embedding, speaker_embedding(other["reference_audio"], language)))) for other in negatives] result = dict(job_id=job["job_id"], language=language, expected_text=job["text"], asr_text=transcript, error_rate=error_rate(job["text"], transcript, language), error_metric="CER" if language == "zh" else "WER", reference_text=job["reference_text"], reference_source_key=job["reference_source_key"], reference_cosine=float(np.dot(generated_embedding, reference_embedding)), other_reference_cosines=comparisons, audio_sha256=record["normalized"]["sha256"], duration_seconds=record["normalized"]["duration_seconds"], qa_seconds=time.monotonic() - begin, status="measurements_only_pending_human_review", eligible=False, same_speaker_verified=False) results.append(result) print(json.dumps(result, ensure_ascii=False), flush=True) report = dict(format="hfd_aside_cpu_qa_probe_v1", jobs_manifest_sha256=jobs_sha, speaker_models=evidence, asr_model=str(asr_path), asr_model_sha256=probe.sha256(asr_path / "model.safetensors"), speaker_threshold=None, note="No calibrated speaker threshold; Whisper-tiny errors are screening evidence, not ground truth.", text_metric_note="Raw CER/WER; number spelling, simplified/traditional forms, dialect spelling and fillers are not normalized.", qa_script_sha256=probe.sha256(Path(__file__)), eligible=False, same_speaker_verified=False, human_review="pending", samples=results) probe.write_json(ASSETS / "qa" / f"sample_{args.offset}_{len(chosen)}_shard_{args.shard_index}.json", report) return dict(samples=len(results), status="measurements_only_pending_human_review") def main() -> None: global ASSETS parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("mode", choices=("generate", "qa", "summarize")) parser.add_argument("--jobs", type=Path, default=PLAN) parser.add_argument("--output-root", type=Path, default=ASSETS) parser.add_argument("--threads", type=int, default=8) parser.add_argument("--batch-size", type=int, default=4) parser.add_argument("--max-new-tokens", type=int, default=384) parser.add_argument("--offset", type=int, default=0) parser.add_argument("--limit", type=int, default=8, help="0 explicitly selects the full remaining manifest") parser.add_argument("--shard-index", type=int, default=0) parser.add_argument("--shard-count", type=int, default=1) parser.add_argument("--resume", action="store_true") args = parser.parse_args() ASSETS = args.output_root.absolute() probe.safe_output(ASSETS) if args.mode == "generate": result = generate(args) elif args.mode == "qa": result = qa(args) else: jobs, digest = read_jobs(args.jobs) _, model_digest = verified_model() result = refresh_registry(jobs, digest, model_digest) print(json.dumps(result, ensure_ascii=False), flush=True) if __name__ == "__main__": main()