Download scripts/data/generate_hfd_same_voice_asides.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 21.3 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/generate_hfd_same_voice_asides.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/generate_hfd_same_voice_asides.py
-
curl -L -o generate_hfd_same_voice_asides.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/generate_hfd_same_voice_asides.py
21.3 kB
| #!/usr/bin/env python3 | |
| """Generate reference-bound asides on CPU; never automatically mark QA eligible.""" | |
| from __future__ import annotations | |
| import argparse | |
| from collections import OrderedDict | |
| import hashlib | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import re | |
| import sys | |
| import time | |
| import wave | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| import probe_hfd_same_voice_tts as probe | |
| WORK = probe.WORK | |
| PLAN = WORK / "plan/tts_jobs.jsonl" | |
| ASSETS = WORK / "tts_assets" | |
| FORMAT = "hfd_reference_bound_aside_generated_v1" | |
| PUBLIC = Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt") | |
| def canonical_hash(value: object) -> str: | |
| return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()).hexdigest() | |
| def configure_cpu(threads: int) -> None: | |
| if not 1 <= threads <= 8: | |
| raise ValueError("CPU thread count must be 1..8") | |
| for name in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES"): | |
| os.environ[name] = "" | |
| for name in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS"): | |
| os.environ[name] = str(threads) | |
| os.environ.update(TOKENIZERS_PARALLELISM="false", HF_HUB_OFFLINE="1", | |
| TRANSFORMERS_OFFLINE="1", HF_HOME=str(WORK / "models/hf_cache"), | |
| HF_HUB_CACHE=str(WORK / "models/hf_cache/hub"), | |
| XDG_CACHE_HOME=str(WORK / "models/cache")) | |
| def read_jobs(path: Path) -> tuple[list[dict], str]: | |
| digest = probe.sha256(path) | |
| report = json.loads((path.parent / "preparation_report.json").read_text()) | |
| if report["output_hashes"][path.name] != digest: | |
| raise ValueError("Frozen TTS job manifest changed") | |
| jobs = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] | |
| if len({row["job_id"] for row in jobs}) != len(jobs): | |
| raise ValueError("Duplicate job IDs") | |
| for row in jobs: | |
| if not re.fullmatch(r"[0-9a-f]{64}", row["job_id"]): | |
| raise ValueError("Invalid job ID") | |
| if row["language"] not in {"zh", "en"} or row["split"] not in {"train", "val", "validation"}: | |
| raise ValueError("Unsupported language/split") | |
| if not row["text"].strip() or not row["reference_text"].strip(): | |
| raise ValueError("Missing synthesis/reference text") | |
| if row.get("voice_relation") != "reference_clone": | |
| raise ValueError("Only actual reference clones are supported") | |
| return jobs, digest | |
| def select_jobs(jobs: list[dict], offset: int, limit: int, shard_index: int, shard_count: int) -> list[dict]: | |
| if offset < 0 or limit < 0 or shard_count < 1 or not 0 <= shard_index < shard_count: | |
| raise ValueError("Invalid job slice/shard") | |
| selected = [job for index, job in enumerate(jobs) if index % shard_count == shard_index][offset:] | |
| return selected[:limit] if limit else selected | |
| def paths(job_id: str) -> tuple[Path, Path, Path]: | |
| return (ASSETS / "raw" / f"{job_id}.wav", ASSETS / "normalized" / f"{job_id}.wav", | |
| ASSETS / "provenance" / f"{job_id}.json") | |
| def verify_resume(job: dict, jobs_sha: str, model_sha: str) -> dict | None: | |
| raw, normalized, sidecar = paths(job["job_id"]) | |
| for path in (raw, normalized, sidecar): | |
| probe.safe_output(path) | |
| if not sidecar.exists(): | |
| if raw.exists() or normalized.exists(): | |
| raise ValueError(f"Uncommitted output collision: {job['job_id']}") | |
| return None | |
| record = json.loads(sidecar.read_text()) | |
| if (record["job_sha256"] != canonical_hash(job) or record["jobs_manifest_sha256"] != jobs_sha | |
| or record["model_manifest_sha256"] != model_sha or record["model_revision"] != probe.REVISION | |
| or record["status"] != "generated_pending_qa" or record["eligible"] is not False | |
| or record["generation"]["ended_with_eos"] is not True): | |
| raise ValueError(f"Existing provenance contract mismatch: {job['job_id']}") | |
| for key, expected_path in (("raw", raw), ("normalized", normalized)): | |
| if record[key]["path"] != str(expected_path) or probe.sha256(expected_path) != record[key]["sha256"]: | |
| raise ValueError(f"Existing {key} output hash mismatch: {job['job_id']}") | |
| return record | |
| def verified_model() -> tuple[dict, str]: | |
| path = probe.MODEL_DIR / "verified_download.json" | |
| manifest = json.loads(path.read_text()) | |
| if not manifest["complete"] or manifest["revision"] != probe.REVISION or manifest["model_id"] != probe.MODEL_ID: | |
| raise ValueError("Unexpected TTS model version") | |
| for item in manifest["files"]: | |
| if probe.verify_file(probe.MODEL_DIR / item["rfilename"], item) != item["sha256"]: | |
| raise ValueError("Verified model file drift") | |
| return manifest, probe.sha256(path) | |
| def verify_reference(job: dict) -> None: | |
| path = Path(job["reference_audio"]) | |
| if probe.sha256(path) != job["reference_sha256"]: | |
| raise ValueError(f"Read-only reference changed: {path}") | |
| with wave.open(str(path), "rb") as handle: | |
| if handle.getnframes() / handle.getframerate() < 3: | |
| raise ValueError("Reference must be >=3 seconds") | |
| def eos_flags(sequences: list[list[int]], eos_token_id: int) -> list[bool]: | |
| """A token-limit stop with no actual EOS is not successful synthesis.""" | |
| return [eos_token_id in sequence for sequence in sequences] | |
| class PromptCache: | |
| def __init__(self, capacity: int = 64): | |
| self.capacity = capacity | |
| self.values: OrderedDict[str, object] = OrderedDict() | |
| def get(self, key: str, factory): | |
| if key not in self.values: | |
| self.values[key] = factory() | |
| self.values.move_to_end(key) | |
| while len(self.values) > self.capacity: | |
| self.values.popitem(last=False) | |
| return self.values[key] | |
| def write_pcm(path: Path, values, sample_rate: int) -> dict: | |
| import numpy as np | |
| import soundfile as sf | |
| probe.safe_output(path) | |
| if path.exists(): | |
| raise ValueError(f"Refusing output overwrite: {path}") | |
| values = np.asarray(values, dtype=np.float32).reshape(-1) | |
| if not values.size or not np.isfinite(values).all() or np.max(np.abs(values)) >= 1: | |
| raise ValueError("Empty, nonfinite, or clipping TTS waveform") | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_suffix(".wav.tmp") | |
| probe.safe_output(temporary) | |
| sf.write(temporary, values, sample_rate, format="WAV", subtype="PCM_16") | |
| temporary.replace(path) | |
| return dict(path=str(path), sha256=probe.sha256(path), sample_rate=sample_rate, | |
| sample_count=int(values.size), duration_seconds=values.size / sample_rate, | |
| peak=float(np.max(np.abs(values))), rms=float(np.sqrt(np.mean(values.astype(np.float64) ** 2)))) | |
| def refresh_registry(jobs: list[dict], jobs_sha: str, model_sha: str) -> dict: | |
| records = [] | |
| for job in jobs: | |
| record = verify_resume(job, jobs_sha, model_sha) | |
| if record is not None: | |
| records.append(record) | |
| path = ASSETS / "registry.jsonl" | |
| probe.safe_output(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_suffix(".jsonl.tmp") | |
| probe.safe_output(temporary) | |
| temporary.write_text("".join(json.dumps(r, sort_keys=True, ensure_ascii=False) + "\n" for r in records)) | |
| temporary.replace(path) | |
| summary = dict(format=FORMAT, planned_jobs=len(jobs), generated_jobs=len(records), eligible_jobs=0, | |
| status="generated_pending_qa", jobs_manifest_sha256=jobs_sha, | |
| registry_sha256=probe.sha256(path), model_manifest_sha256=model_sha, | |
| same_speaker_verified=False, human_review="pending") | |
| probe.write_json(ASSETS / "generation_summary.json", summary) | |
| return summary | |
| def generate(args) -> dict: | |
| configure_cpu(args.threads) | |
| import numpy as np | |
| import torch | |
| from scipy.signal import resample_poly | |
| from qwen_tts import Qwen3TTSModel | |
| torch.set_num_threads(args.threads) | |
| torch.set_num_interop_threads(1) | |
| jobs, jobs_sha = read_jobs(args.jobs) | |
| chosen = select_jobs(jobs, args.offset, args.limit, args.shard_index, args.shard_count) | |
| _, model_sha = verified_model() | |
| pending = [] | |
| for job in chosen: | |
| verify_reference(job) | |
| previous = verify_resume(job, jobs_sha, model_sha) | |
| if previous is None: | |
| pending.append(job) | |
| elif not args.resume: | |
| raise ValueError("Existing generation requires explicit --resume") | |
| if not pending: | |
| return refresh_registry(jobs, jobs_sha, model_sha) | |
| if not 1 <= args.batch_size <= 8 or args.max_new_tokens < 128: | |
| raise ValueError("Batch size must be 1..8 and max tokens >=128") | |
| started = time.monotonic() | |
| model = Qwen3TTSModel.from_pretrained(str(probe.MODEL_DIR), device_map="cpu", dtype=torch.float32, | |
| attn_implementation="eager", local_files_only=True) | |
| if any(p.device.type != "cpu" for p in model.model.parameters()): | |
| raise RuntimeError("Model unexpectedly not entirely on CPU") | |
| print(f"CPU model load seconds={time.monotonic() - started:.2f}", flush=True) | |
| prompt_cache = PromptCache(64) | |
| original_generate = model.model.talker.generate | |
| stops: list[bool] = [] | |
| lengths: list[int] = [] | |
| def capture_generate(*positional, **keywords): | |
| result = original_generate(*positional, **keywords) | |
| sequences = result.sequences.detach().cpu().tolist() | |
| stops[:] = eos_flags(sequences, int(keywords["eos_token_id"])) | |
| lengths[:] = [len(sequence) for sequence in sequences] | |
| return result | |
| model.model.talker.generate = capture_generate | |
| for start in range(0, len(pending), args.batch_size): | |
| batch = pending[start:start + args.batch_size] | |
| begin = time.monotonic() | |
| prompts = [] | |
| for job in batch: | |
| key = canonical_hash((job["reference_sha256"], job["reference_text"])) | |
| prompt = prompt_cache.get(key, lambda j=job: model.create_voice_clone_prompt( | |
| ref_audio=j["reference_audio"], ref_text=j["reference_text"], x_vector_only_mode=False)[0]) | |
| prompts.append(prompt) | |
| seed = int(canonical_hash([job["job_id"] for job in batch])[:8], 16) | |
| torch.manual_seed(seed) | |
| stops.clear() | |
| with torch.inference_mode(): | |
| wavs, rate = model.generate_voice_clone(text=[job["text"] for job in batch], | |
| language=["Chinese" if job["language"] == "zh" else "English" for job in batch], | |
| voice_clone_prompt=prompts, non_streaming_mode=True, | |
| max_new_tokens=args.max_new_tokens, do_sample=True) | |
| if len(wavs) != len(batch) or len(stops) != len(batch) or not all(stops): | |
| probe.write_json(ASSETS / "last_generation_failure.json", dict( | |
| job_ids=[job["job_id"] for job in batch], eos_flags=stops, | |
| status="rejected_missing_eos_or_batch_mismatch", max_new_tokens=args.max_new_tokens)) | |
| raise RuntimeError("Batch rejected: missing EOS (possible token truncation) or wrong size") | |
| if int(rate) != 24000: | |
| raise RuntimeError(f"Unexpected TTS sample rate: {rate}") | |
| elapsed = time.monotonic() - begin | |
| for index, (job, waveform) in enumerate(zip(batch, wavs)): | |
| verify_reference(job) | |
| raw, normalized, sidecar = paths(job["job_id"]) | |
| values = np.asarray(waveform, dtype=np.float32).reshape(-1) | |
| normalized_values = resample_poly(values, 2, 3) | |
| record = dict(format=FORMAT, job_id=job["job_id"], split=job["split"], language=job["language"], | |
| job=job, job_sha256=canonical_hash(job), jobs_manifest_sha256=jobs_sha, | |
| model_id=probe.MODEL_ID, model_revision=probe.REVISION, model_manifest_sha256=model_sha, | |
| status="generated_pending_qa", eligible=False, same_speaker_verified=False, | |
| human_review="pending", voice_relation="reference_clone", | |
| generation=dict(device="cpu", threads=args.threads, dtype="float32", attention="eager", | |
| seed=seed, batch_job_ids=[j["job_id"] for j in batch], batch_seconds=elapsed, | |
| seconds_per_job_average=elapsed / len(batch), ended_with_eos=stops[index], | |
| padded_sequence_tokens=lengths[index], max_new_tokens=args.max_new_tokens, | |
| generator_script_sha256=probe.sha256(Path(__file__)))) | |
| # Reject clipping before either output is committed. | |
| if (not values.size or not np.isfinite(values).all() or not np.isfinite(normalized_values).all() | |
| or np.max(np.abs(values)) >= 1 or np.max(np.abs(normalized_values)) >= 1): | |
| raise RuntimeError("TTS waveform invalid/clipping before commit") | |
| record["raw"] = write_pcm(raw, values, 24000) | |
| record["normalized"] = write_pcm(normalized, normalized_values, 16000) | |
| probe.write_json(sidecar, record) | |
| print(json.dumps(dict(job_id=job["job_id"], language=job["language"], text=job["text"], | |
| duration_seconds=record["raw"]["duration_seconds"], | |
| batch_seconds=elapsed, status=record["status"]), ensure_ascii=False), flush=True) | |
| return refresh_registry(jobs, jobs_sha, model_sha) | |
| def normalized_units(text: str, language: str) -> list[str]: | |
| if language == "zh": | |
| return re.findall(r"[\u4e00-\u9fffA-Za-z0-9]", text.lower()) | |
| return re.findall(r"[a-z0-9]+", text.lower()) | |
| def error_rate(expected: str, actual: str, language: str) -> float: | |
| left, right = normalized_units(expected, language), normalized_units(actual, language) | |
| if not left: | |
| raise ValueError("Empty normalized expected text") | |
| previous = list(range(len(right) + 1)) | |
| for i, token in enumerate(left, 1): | |
| current = [i] | |
| for j, candidate in enumerate(right, 1): | |
| current.append(min(previous[j] + 1, current[-1] + 1, previous[j - 1] + (token != candidate))) | |
| previous = current | |
| return previous[-1] / len(left) | |
| def qa(args) -> dict: | |
| configure_cpu(args.threads) | |
| import numpy as np | |
| import onnxruntime as ort | |
| import soundfile as sf | |
| from scipy.signal import resample_poly | |
| import torch | |
| import torchaudio | |
| from transformers import WhisperForConditionalGeneration, WhisperProcessor | |
| torch.set_num_threads(args.threads) | |
| torch.set_num_interop_threads(1) | |
| jobs, jobs_sha = read_jobs(args.jobs) | |
| chosen = select_jobs(jobs, args.offset, args.limit, args.shard_index, args.shard_count) | |
| manifest = probe.MODEL_DIR / "verified_download.json" | |
| model_sha = probe.sha256(manifest) | |
| records = [verify_resume(job, jobs_sha, model_sha) for job in chosen] | |
| if any(record is None for record in records): | |
| raise ValueError("QA selection includes ungenerated jobs") | |
| asr_path = PUBLIC / "openai/whisper-tiny" | |
| processor = WhisperProcessor.from_pretrained(str(asr_path), local_files_only=True) | |
| asr = WhisperForConditionalGeneration.from_pretrained(str(asr_path), local_files_only=True).to("cpu").eval() | |
| sessions = {} | |
| embeddings = {} | |
| evidence = [] | |
| def load_audio(path: str): | |
| samples, rate = sf.read(path, dtype="float32", always_2d=False) | |
| if samples.ndim == 2: | |
| samples = samples.mean(axis=1) | |
| if rate != 16000: | |
| divisor = math.gcd(rate, 16000) | |
| samples = resample_poly(samples, 16000 // divisor, rate // divisor) | |
| return samples.astype(np.float32) | |
| def speaker_embedding(path: str, language: str): | |
| key = (path, language) | |
| if key not in embeddings: | |
| values = load_audio(path) | |
| features = torchaudio.compliance.kaldi.fbank(torch.from_numpy(values).unsqueeze(0) * 32768, | |
| num_mel_bins=80, frame_length=25, frame_shift=10, sample_frequency=16000, | |
| dither=0, window_type="hamming", use_energy=False) | |
| features = features - features.mean(dim=0, keepdim=True) | |
| result = sessions[language].run(None, {"feats": features.numpy()[None].astype(np.float32)})[0][0] | |
| embeddings[key] = result / max(float(np.linalg.norm(result)), 1e-12) | |
| return embeddings[key] | |
| for language in sorted({job["language"] for job in chosen}): | |
| name = "wespeaker_cn_cnceleb_resnet34_LM.onnx" if language == "zh" else "wespeaker_en_voxceleb_resnet34_LM.onnx" | |
| path = PUBLIC / "vidproc/speaker_diarization/wespeaker-voxceleb-resnet34-LM" / name | |
| options = ort.SessionOptions() | |
| options.intra_op_num_threads = args.threads | |
| options.inter_op_num_threads = 1 | |
| sessions[language] = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"]) | |
| evidence.append(dict(model=str(path), sha256=probe.sha256(path), providers=sessions[language].get_providers())) | |
| results = [] | |
| for job, record in zip(chosen, records): | |
| begin = time.monotonic() | |
| verify_reference(job) | |
| language = job["language"] | |
| values = load_audio(record["normalized"]["path"]) | |
| inputs = processor(values, sampling_rate=16000, return_tensors="pt", return_attention_mask=True) | |
| with torch.inference_mode(): | |
| ids = asr.generate(inputs.input_features, attention_mask=inputs.attention_mask, | |
| language="chinese" if language == "zh" else "english", | |
| task="transcribe", max_new_tokens=128) | |
| transcript = processor.batch_decode(ids, skip_special_tokens=True)[0].strip() | |
| generated_embedding = speaker_embedding(record["normalized"]["path"], language) | |
| reference_embedding = speaker_embedding(job["reference_audio"], language) | |
| negatives = [candidate for candidate in chosen if candidate["language"] == language | |
| and candidate["reference_sha256"] != job["reference_sha256"]] | |
| comparisons = [dict(reference_source_key=other["reference_source_key"], | |
| cosine=float(np.dot(generated_embedding, speaker_embedding(other["reference_audio"], language)))) | |
| for other in negatives] | |
| result = dict(job_id=job["job_id"], language=language, expected_text=job["text"], asr_text=transcript, | |
| error_rate=error_rate(job["text"], transcript, language), error_metric="CER" if language == "zh" else "WER", | |
| reference_text=job["reference_text"], reference_source_key=job["reference_source_key"], | |
| reference_cosine=float(np.dot(generated_embedding, reference_embedding)), | |
| other_reference_cosines=comparisons, audio_sha256=record["normalized"]["sha256"], | |
| duration_seconds=record["normalized"]["duration_seconds"], qa_seconds=time.monotonic() - begin, | |
| status="measurements_only_pending_human_review", eligible=False, same_speaker_verified=False) | |
| results.append(result) | |
| print(json.dumps(result, ensure_ascii=False), flush=True) | |
| report = dict(format="hfd_aside_cpu_qa_probe_v1", jobs_manifest_sha256=jobs_sha, | |
| speaker_models=evidence, asr_model=str(asr_path), asr_model_sha256=probe.sha256(asr_path / "model.safetensors"), | |
| speaker_threshold=None, note="No calibrated speaker threshold; Whisper-tiny errors are screening evidence, not ground truth.", | |
| text_metric_note="Raw CER/WER; number spelling, simplified/traditional forms, dialect spelling and fillers are not normalized.", | |
| qa_script_sha256=probe.sha256(Path(__file__)), | |
| eligible=False, same_speaker_verified=False, human_review="pending", samples=results) | |
| probe.write_json(ASSETS / "qa" / f"sample_{args.offset}_{len(chosen)}_shard_{args.shard_index}.json", report) | |
| return dict(samples=len(results), status="measurements_only_pending_human_review") | |
| def main() -> None: | |
| global ASSETS | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("mode", choices=("generate", "qa", "summarize")) | |
| parser.add_argument("--jobs", type=Path, default=PLAN) | |
| parser.add_argument("--output-root", type=Path, default=ASSETS) | |
| parser.add_argument("--threads", type=int, default=8) | |
| parser.add_argument("--batch-size", type=int, default=4) | |
| parser.add_argument("--max-new-tokens", type=int, default=384) | |
| parser.add_argument("--offset", type=int, default=0) | |
| parser.add_argument("--limit", type=int, default=8, help="0 explicitly selects the full remaining manifest") | |
| parser.add_argument("--shard-index", type=int, default=0) | |
| parser.add_argument("--shard-count", type=int, default=1) | |
| parser.add_argument("--resume", action="store_true") | |
| args = parser.parse_args() | |
| ASSETS = args.output_root.absolute() | |
| probe.safe_output(ASSETS) | |
| if args.mode == "generate": | |
| result = generate(args) | |
| elif args.mode == "qa": | |
| result = qa(args) | |
| else: | |
| jobs, digest = read_jobs(args.jobs) | |
| _, model_digest = verified_model() | |
| result = refresh_registry(jobs, digest, model_digest) | |
| print(json.dumps(result, ensure_ascii=False), flush=True) | |
| if __name__ == "__main__": | |
| main() | |