Download scripts/data/qa_hfd_same_voice_asides.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 33.5 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/qa_hfd_same_voice_asides.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/qa_hfd_same_voice_asides.py
-
curl -L -o qa_hfd_same_voice_asides.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/qa_hfd_same_voice_asides.py
33.5 kB
| #!/usr/bin/env python3 | |
| """CPU-only ASR/speaker screening; engineering eligibility is not human approval.""" | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import re | |
| import sys | |
| import time | |
| import unicodedata | |
| import urllib.request | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| import generate_hfd_same_voice_asides as generation | |
| import probe_hfd_same_voice_tts as probe | |
| if str(probe.REPO) not in sys.path: | |
| sys.path.insert(0, str(probe.REPO)) | |
| from scripts.data.build_hfd_diverse_catalog import filter_text | |
| from scripts.data.build_hfd_cross_catalog import new_source_filter_reason | |
| MODEL_ID = "openai/whisper-base" | |
| REVISION = "e37978b90ca9030d5170a5c07aadb050351a65bb" | |
| MODEL = probe.WORK / "models/whisper-base" | |
| OPENCC_REVISION = "2939943bd6f4d459b46d7fdcf07a885ab3f01761" | |
| OPENCC = probe.WORK / "models/opencc_t2s" | |
| NORMALIZATION_VERSION = "strict_t2s_numbers_spelling_adjacent_neutral_filler_v2" | |
| OPENCC_FILES = { | |
| "data/dictionary/TSCharacters.txt": (104520, "cbe56a8327ced95c7d91eacb011090301bd158fc"), | |
| "data/dictionary/TSPhrases.txt": (8835, "2faff4ece295d20bdec857fb8838e2ba1b9bfdb9"), | |
| "data/config/t2s.json": (589, "c6e389cfc5aaba56cccdfa287758347a0d8f1398"), | |
| "LICENSE": (9165, "9f7a310eb2b143dfc4b73551ea60ca17eb2b981f"), | |
| } | |
| MODEL_FILES = {"README.md", "added_tokens.json", "config.json", "generation_config.json", "merges.txt", | |
| "model.safetensors", "normalizer.json", "preprocessor_config.json", "special_tokens_map.json", | |
| "tokenizer.json", "tokenizer_config.json", "vocab.json"} | |
| def download_file(url: str, path: Path, item: dict) -> dict: | |
| probe.safe_output(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| if not path.exists(): | |
| temporary = path.with_suffix(path.suffix + ".part") | |
| probe.safe_output(temporary) | |
| with urllib.request.urlopen(url, timeout=60) as response, temporary.open("wb") as handle: | |
| for chunk in iter(lambda: response.read(8 * 1024 * 1024), b""): | |
| handle.write(chunk) | |
| probe.verify_file(temporary, item) | |
| temporary.replace(path) | |
| return {**item, "sha256": probe.verify_file(path, item), "url": url, "path": str(path)} | |
| def download() -> dict: | |
| api = f"https://huggingface.co/api/models/{MODEL_ID}/revision/{REVISION}?blobs=true" | |
| with urllib.request.urlopen(api, timeout=30) as response: | |
| metadata = json.load(response) | |
| if metadata["sha"] != REVISION or metadata["id"] != MODEL_ID: | |
| raise ValueError("Official ASR revision mismatch") | |
| files = [item for item in metadata["siblings"] if item["rfilename"] in MODEL_FILES] | |
| if {item["rfilename"] for item in files} != MODEL_FILES: | |
| raise ValueError("Required ASR file missing") | |
| total = sum(item["size"] for item in files) | |
| if total + sum(item[0] for item in OPENCC_FILES.values()) > 1_000_000_000: | |
| raise ValueError("Download exceeds approved 1 GB budget") | |
| result = dict(model_id=MODEL_ID, revision=REVISION, required_bytes=total, files=[]) | |
| print(json.dumps({"required_bytes": total, "revision": REVISION}), flush=True) | |
| for item in files: | |
| name = item["rfilename"] | |
| url = f"https://huggingface.co/{MODEL_ID}/resolve/{REVISION}/{name}" | |
| result["files"].append(download_file(url, MODEL / name, item)) | |
| print(f"verified ASR {name}", flush=True) | |
| result["complete"] = True | |
| probe.write_json(MODEL / "verified_download.json", result) | |
| dictionaries = [] | |
| for name, (size, blob) in OPENCC_FILES.items(): | |
| item = dict(rfilename=name, size=size, blobId=blob) | |
| url = f"https://raw.githubusercontent.com/BYVoid/OpenCC/{OPENCC_REVISION}/{name}" | |
| dictionaries.append(download_file(url, OPENCC / Path(name).name, item)) | |
| probe.write_json(OPENCC / "verified_download.json", dict(revision=OPENCC_REVISION, files=dictionaries, complete=True)) | |
| return dict(asr_bytes=total, opencc_bytes=sum(item[0] for item in OPENCC_FILES.values()), complete=True) | |
| def verify_models() -> list[dict]: | |
| evidence = [] | |
| for root, expected_revision in ((MODEL, REVISION), (OPENCC, OPENCC_REVISION)): | |
| manifest_path = root / "verified_download.json" | |
| manifest = json.loads(manifest_path.read_text()) | |
| if not manifest["complete"] or manifest["revision"] != expected_revision: | |
| raise ValueError("QA model/dictionary revision drift") | |
| for item in manifest["files"]: | |
| path = root / Path(item["rfilename"]).name | |
| if probe.verify_file(path, item) != item["sha256"]: | |
| raise ValueError("QA model/dictionary content drift") | |
| evidence.append(dict(root=str(root), revision=expected_revision, manifest_sha256=probe.sha256(manifest_path))) | |
| return evidence | |
| def read_dictionary(path: Path) -> dict[str, str]: | |
| result = {} | |
| for line in path.read_text().splitlines(): | |
| if not line.strip() or line.startswith("#"): | |
| continue | |
| source, replacements = line.split("\t", 1) | |
| result[source] = replacements.split(" ")[0] | |
| return result | |
| class StrictNormalizer: | |
| """No removal of bracket contents, fillers, negatives, or changed numbers.""" | |
| def __init__(self, characters: dict, phrases: dict, spelling: dict): | |
| self.mapping = {**characters, **phrases} | |
| self.max_key_length = max((len(key) for key in self.mapping), default=1) | |
| self.spelling = spelling | |
| def to_simplified(self, text: str) -> str: | |
| result = [] | |
| index = 0 | |
| while index < len(text): | |
| for size in range(min(self.max_key_length, len(text) - index), 0, -1): | |
| key = text[index:index + size] | |
| if key in self.mapping: | |
| result.append(self.mapping[key]) | |
| index += size | |
| break | |
| else: | |
| result.append(text[index]) | |
| index += 1 | |
| return "".join(result) | |
| def normalize(self, text: str, language: str) -> str: | |
| text = unicodedata.normalize("NFKC", text).lower().replace("’", "'") | |
| if language == "zh": | |
| value = "".join(char for char in self.to_simplified(text) if char.isalnum()) | |
| # Only contiguous repetitions of explicitly neutral hesitation markers. | |
| # Never drop a single filler, negation, number or full referential phrase. | |
| return re.sub(r"(那个|嗯|呃)(?:\1)+", r"\1", value) | |
| if language != "en": | |
| raise ValueError("Unsupported normalization language") | |
| from transformers.models.whisper.english_normalizer import EnglishNumberNormalizer, EnglishSpellingNormalizer | |
| replacements = {"can't": "can not", "won't": "will not", "i'm": "i am", "it's": "it is", | |
| "that's": "that is", "there's": "there is", "let's": "let us"} | |
| for source, replacement in replacements.items(): | |
| text = re.sub(r"\b" + re.escape(source) + r"\b", replacement, text) | |
| text = re.sub(r"\b([a-z]+)n't\b", r"\1 not", text) | |
| text = re.sub(r"\b([a-z]+)'re\b", r"\1 are", text) | |
| text = re.sub(r"\b([a-z]+)'ve\b", r"\1 have", text) | |
| text = re.sub(r"\b([a-z]+)'ll\b", r"\1 will", text) | |
| text = re.sub(r"(?<=[a-z])-(?=[a-z])", " ", text) | |
| text = re.sub(r"\ba (?=hundred\b|thousand\b|million\b|billion\b)", "one ", text) | |
| # Retain bracket contents and decimal/sign/currency semantics. | |
| text = " ".join(re.findall(r"[a-z]+(?:'[a-z]+)?|[+-]?\d+(?:\.\d+)?|[%$£€]", text)) | |
| text = EnglishSpellingNormalizer(self.spelling)(EnglishNumberNormalizer()(text)) | |
| return re.sub(r"\b(um|uh|hmm)(?:\s+\1)+\b", r"\1", text) | |
| def levenshtein_rate(expected: str, observed: str, language: str) -> float: | |
| left, right = (list(expected), list(observed)) if language == "zh" else (expected.split(), observed.split()) | |
| if not left: | |
| raise ValueError("Expected normalized text empty") | |
| previous = list(range(len(right) + 1)) | |
| for i, token in enumerate(left, 1): | |
| current = [i] | |
| for j, candidate in enumerate(right, 1): | |
| current.append(min(previous[j] + 1, current[-1] + 1, previous[j - 1] + (token != candidate))) | |
| previous = current | |
| return previous[-1] / len(left) | |
| def critical_tokens_preserved(expected: str, observed: str, language: str) -> bool: | |
| from collections import Counter | |
| if language == "en": | |
| extractor = lambda value: re.findall(r"\b(?:not|no|never)\b|[+-]?\d+(?:\.\d+)?", value) | |
| else: | |
| extractor = lambda value: re.findall(r"[不没无别]|[零〇一二两三四五六七八九十百千万亿半壹贰叁肆伍陆柒捌玖拾佰仟]+|[0-9]+(?:\.[0-9]+)?", value) | |
| return Counter(extractor(expected)) == Counter(extractor(observed)) | |
| def engineering_gate(rate: float, cosine: float, maximum_comparison: float, critical_match: bool, | |
| observed_eos: bool, *, max_rate: float = .15, min_cosine: float = .75, | |
| min_margin: float = .05) -> dict: | |
| reasons = [] | |
| import math | |
| if not all(math.isfinite(value) for value in (rate, cosine, maximum_comparison)): | |
| reasons.append("nonfinite_measurement") | |
| if rate > max_rate: | |
| reasons.append("asr_text_difference") | |
| if not critical_match: | |
| reasons.append("negation_or_numeric_difference") | |
| if cosine < min_cosine: | |
| reasons.append("low_reference_similarity") | |
| if cosine - maximum_comparison < min_margin: | |
| reasons.append("low_relative_reference_margin") | |
| if not observed_eos: | |
| reasons.append("missing_generation_eos") | |
| return dict(engineering_eligible=not reasons, reasons=reasons, human_review="pending", | |
| same_speaker_verified=False, thresholds_calibrated=False) | |
| def reference_comparisons(job: dict, pool: list[dict], count: int) -> list[dict]: | |
| voice = Path(job["reference_audio"]).stem | |
| gender = voice.split("_")[0] | |
| candidates = [row for row in pool if row["language"] == job["language"] | |
| and row["reference_sha256"] != job["reference_sha256"]] | |
| candidates.sort(key=lambda row: (Path(row["reference_audio"]).stem != voice, | |
| Path(row["reference_audio"]).stem.split("_")[0] != gender, | |
| row["reference_sha256"])) | |
| unique = {} | |
| for row in candidates: | |
| unique.setdefault(row["reference_sha256"], row) | |
| return list(unique.values())[:count] | |
| def asr_semantic_reason(transcript: str, language: str, source: dict, semantics: dict, | |
| normalizer: StrictNormalizer) -> str | None: | |
| # Preserve punctuation/question marks while normalizing traditional glyphs. | |
| checked_text = normalizer.to_simplified(unicodedata.normalize("NFKC", transcript)) if language == "zh" else transcript | |
| return (filter_text(checked_text, language, set(semantics["target_phrases"][language]), semantics=semantics) | |
| or new_source_filter_reason({**source, "transcript": checked_text}, semantics)) | |
| def candidate_row(job: dict, record: dict, qa_path: Path, jobs_sha: str) -> dict: | |
| provenance = generation.paths(job["job_id"])[2] | |
| return dict(asset_id="tts_clone:" + job["job_id"], job_id=job["job_id"], | |
| reference_audio=job["reference_audio"], reference_sha256=job["reference_sha256"], | |
| reference_text=job["reference_text"], reference_source_key=job["reference_source_key"], | |
| split=job["split"], language=job["language"], transcript=job["text"], role="non_command_speech", | |
| normalized_path=record["normalized"]["path"], normalized_sha256=record["normalized"]["sha256"], | |
| sample_count=record["normalized"]["sample_count"], sample_rate=16000, | |
| speaker_id="tts_ref:" + job["reference_sha256"], source_group="tts_ref:" + job["reference_sha256"], | |
| voice_relation="reference_clone", eligible_for_mixing=True, engineering_eligible=True, | |
| job_sha256=record["job_sha256"], jobs_manifest_sha256=jobs_sha, | |
| generation_provenance_path=str(provenance), generation_provenance_sha256=probe.sha256(provenance), | |
| qa_path=str(qa_path), qa_sha256=probe.sha256(qa_path), human_review="pending", same_speaker_verified=False, | |
| text_source_asset_id=job["text_source_asset_id"], normalization_version=NORMALIZATION_VERSION) | |
| def requested_jobs(jobs: list[dict], path: Path | None) -> tuple[list[dict], dict]: | |
| if path is None: | |
| return jobs, dict(subset_scope=False, requested_count=len(jobs), full_plan_job_count=len(jobs)) | |
| text = path.read_text(encoding="utf-8").strip() | |
| if not text: | |
| raise ValueError("Job-ID selection must not be empty") | |
| if text[0] in "[{": | |
| value = json.loads(text) | |
| ids = value.get("job_ids") if isinstance(value, dict) else value | |
| else: | |
| ids = [line.strip() for line in text.splitlines() if line.strip()] | |
| if not isinstance(ids, list) or not ids or any(not isinstance(i, str) or not re.fullmatch(r"[0-9a-f]{64}", i) for i in ids): | |
| raise ValueError("Job IDs must be a nonempty JSON list/job_ids object or one ID per line") | |
| if len(ids) != len(set(ids)): | |
| raise ValueError("Duplicate requested job IDs") | |
| selected = [job for job in jobs if job["job_id"] in set(ids)] | |
| if len(selected) != len(ids): | |
| raise ValueError("Unknown job ID outside frozen complete manifest") | |
| return selected, dict(subset_scope=True, requested_count=len(ids), full_plan_job_count=len(jobs), | |
| job_ids_file=str(path), job_ids_file_sha256=probe.sha256(path)) | |
| def protect_subset_output(root: Path, subset_scope: bool) -> None: | |
| if subset_scope and not re.fullmatch(r"tts_assets_retry[1-9][0-9]*", root.name): | |
| raise ValueError("Requested-subset QA must use an isolated tts_assets_retryN output, never frozen originals") | |
| def write_candidates(path: Path, rows: list[dict]) -> None: | |
| probe.safe_output(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_suffix(".jsonl.tmp") | |
| probe.safe_output(temporary) | |
| temporary.write_text("".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in rows)) | |
| temporary.replace(path) | |
| def recheck(args) -> dict: | |
| """Re-evaluate saved ASR/speaker evidence without regenerating or retranscribing.""" | |
| generation.ASSETS = args.assets.absolute() | |
| probe.safe_output(generation.ASSETS) | |
| if args.input_report is None or not re.fullmatch(r"[a-z0-9_]+", args.report_tag): | |
| raise ValueError("Recheck requires an existing report and safe new report tag") | |
| prior_sha = probe.sha256(args.input_report) | |
| report = json.loads(args.input_report.read_text()) | |
| jobs, jobs_sha = generation.read_jobs(args.jobs) | |
| if args.job_ids_file is not None: | |
| raise ValueError("Recheck preserves its original report scope; use run/publish for job-ID selection") | |
| by_id = {job["job_id"]: job for job in jobs} | |
| if jobs_sha != report["jobs_manifest_sha256"]: | |
| raise ValueError("Prior report belongs to a different frozen plan") | |
| model_sha = probe.sha256(probe.MODEL_DIR / "verified_download.json") | |
| metadata = json.loads((args.jobs.parent / "preparation_report.json").read_text()) | |
| semantics_path = Path(metadata["reference_root"]) / "work/catalog/manifests/command_semantics.json" | |
| if probe.sha256(semantics_path) != report["command_semantics_sha256"]: | |
| raise ValueError("Semantic policy drift") | |
| semantics = json.loads(semantics_path.read_text()) | |
| source_path = args.jobs.parent / "selected_real_assets.jsonl" | |
| if probe.sha256(source_path) != metadata["output_hashes"][source_path.name]: | |
| raise ValueError("Text source registry drift") | |
| sources = {row["asset_id"]: row for row in map(json.loads, source_path.read_text().splitlines())} | |
| verify_models() | |
| normalizer = StrictNormalizer(read_dictionary(OPENCC / "TSCharacters.txt"), read_dictionary(OPENCC / "TSPhrases.txt"), | |
| json.loads((MODEL / "normalizer.json").read_text())) | |
| qa_root = generation.ASSETS / args.report_tag | |
| candidates = [] | |
| for result in report["samples"]: | |
| job = by_id[result["job_id"]] | |
| record = generation.verify_resume(job, jobs_sha, model_sha) | |
| if record is None or result["audio_sha256"] != record["normalized"]["sha256"] or result["job_sha256"] != record["job_sha256"]: | |
| raise ValueError("Saved QA evidence does not match immutable generation") | |
| expected = normalizer.normalize(job["text"], job["language"]) | |
| observed = normalizer.normalize(result["asr_text"], job["language"]) | |
| rate = levenshtein_rate(expected, observed, job["language"]) | |
| gate = engineering_gate(rate, result["reference_cosine"], result["maximum_comparison_cosine"], | |
| critical_tokens_preserved(expected, observed, job["language"]), | |
| result["asr_ended_with_eos"] and record["generation"]["ended_with_eos"]) | |
| reason = asr_semantic_reason(result["asr_text"], job["language"], sources[job["text_source_asset_id"]], semantics, normalizer) | |
| if reason: | |
| gate["engineering_eligible"] = False | |
| gate["reasons"].append("asr_semantic_filter:" + reason) | |
| result.update(gate, expected_normalized=expected, asr_normalized=observed, normalized_error_rate=rate, | |
| asr_semantic_filter_reason=reason, normalization_version=NORMALIZATION_VERSION, | |
| critical_token_policy="negation_arabic_and_chinese_numerals_v2", prior_qa_summary_sha256=prior_sha) | |
| path = qa_root / f"{job['job_id']}.json" | |
| probe.write_json(path, result) | |
| if result["engineering_eligible"]: | |
| candidates.append(candidate_row(job, record, path, jobs_sha)) | |
| report.update(automatically_eligible=len(candidates), normalization_version=NORMALIZATION_VERSION, | |
| critical_token_policy="negation_arabic_and_chinese_numerals_v2", | |
| assessment_mode="recheck_saved_asr_and_speaker_evidence", prior_qa_summary_path=str(args.input_report), | |
| prior_qa_summary_sha256=prior_sha, qa_script_sha256=probe.sha256(Path(__file__))) | |
| probe.write_json(qa_root / f"summary_recheck_{len(report['samples'])}.json", report) | |
| write_candidates(qa_root / f"eligible_candidates_recheck_{len(report['samples'])}.jsonl", candidates) | |
| return dict(samples=len(report["samples"]), automatically_eligible=len(candidates), human_review="pending") | |
| def publish(args) -> dict: | |
| from collections import Counter | |
| generation.ASSETS = args.assets.absolute() | |
| probe.safe_output(generation.ASSETS) | |
| if not re.fullmatch(r"[a-z0-9_]+", args.report_tag): | |
| raise ValueError("Unsafe QA report tag") | |
| jobs, jobs_sha = generation.read_jobs(args.jobs) | |
| chosen, scope = requested_jobs(jobs, args.job_ids_file) | |
| protect_subset_output(generation.ASSETS, scope["subset_scope"]) | |
| model_sha = probe.sha256(probe.MODEL_DIR / "verified_download.json") | |
| candidates, assessed, rejected = [], [], [] | |
| for job in chosen: | |
| path = generation.ASSETS / args.report_tag / f"{job['job_id']}.json" | |
| if not path.exists(): | |
| continue | |
| result = json.loads(path.read_text()) | |
| record = generation.verify_resume(job, jobs_sha, model_sha) | |
| if (record is None or result["job_sha256"] != record["job_sha256"] | |
| or result["audio_sha256"] != record["normalized"]["sha256"] | |
| or result.get("critical_token_policy") != "negation_arabic_and_chinese_numerals_v2" | |
| or result["normalization_version"] != NORMALIZATION_VERSION): | |
| raise ValueError("Candidate QA/audio/job policy contract mismatch") | |
| assessed.append(job["job_id"]) | |
| if result["engineering_eligible"]: | |
| if result["same_speaker_verified"] or result["human_review"] != "pending": | |
| raise ValueError("This automatic screen cannot claim human identity approval") | |
| candidates.append(candidate_row(job, record, path, jobs_sha)) | |
| else: | |
| rejected.append(dict(job_id=job["job_id"], reasons=result["reasons"])) | |
| path = generation.ASSETS / "candidate_registry.jsonl" | |
| write_candidates(path, candidates) | |
| report = dict(format="hfd_same_voice_candidate_registry_v1", candidate_registry_sha256=probe.sha256(path), | |
| planned_jobs=len(jobs), assessed_jobs=len(assessed), candidate_jobs=len(candidates), **scope, | |
| candidate_counts={"/".join(key): count for key, count in Counter((r["split"], r["language"]) for r in candidates).items()}, | |
| jobs_manifest_sha256=jobs_sha, qa_report_tag=args.report_tag, complete=len(assessed) == len(chosen), | |
| complete_scope="requested_subset" if scope["subset_scope"] else "full_plan", | |
| full_plan_complete=not scope["subset_scope"] and len(assessed) == len(jobs), | |
| state="final_qa_candidate_snapshot" if len(assessed) == len(chosen) else "incremental_candidate_snapshot_do_not_freeze_yet", | |
| rejected_jobs=rejected, human_review="pending", same_speaker_verified=False, production_modified=False) | |
| probe.write_json(generation.ASSETS / "candidate_report.json", report) | |
| return {key: report[key] for key in ("planned_jobs", "requested_count", "subset_scope", "assessed_jobs", "candidate_jobs", "candidate_counts", "complete")} | |
| def run(args) -> dict: | |
| generation.configure_cpu(args.threads) | |
| import math | |
| import numpy as np | |
| import onnxruntime as ort | |
| import soundfile as sf | |
| from scipy.signal import resample_poly | |
| import torch | |
| import torchaudio | |
| from transformers import WhisperForConditionalGeneration, WhisperProcessor | |
| torch.set_num_threads(args.threads) | |
| torch.set_num_interop_threads(1) | |
| generation.ASSETS = args.assets.absolute() | |
| probe.safe_output(generation.ASSETS) | |
| if not re.fullmatch(r"[a-z0-9_]+", args.report_tag): | |
| raise ValueError("Unsafe QA report tag") | |
| qa_root = generation.ASSETS / args.report_tag | |
| jobs, jobs_sha = generation.read_jobs(args.jobs) | |
| pool, pool_sha = generation.read_jobs(args.reference_pool) | |
| plan_report = json.loads((args.jobs.parent / "preparation_report.json").read_text()) | |
| semantics_path = Path(plan_report["reference_root"]) / "work/catalog/manifests/command_semantics.json" | |
| semantics_sha = probe.sha256(semantics_path) | |
| if semantics_sha != plan_report["semantics_sha256"]: | |
| raise ValueError("Frozen semantic filter changed") | |
| semantics = json.loads(semantics_path.read_text()) | |
| source_path = args.jobs.parent / "selected_real_assets.jsonl" | |
| if probe.sha256(source_path) != plan_report["output_hashes"][source_path.name]: | |
| raise ValueError("Frozen text-source catalog changed") | |
| sources = {row["asset_id"]: row for row in (json.loads(line) for line in source_path.read_text().splitlines())} | |
| requested, scope = requested_jobs(jobs, args.job_ids_file) | |
| protect_subset_output(generation.ASSETS, scope["subset_scope"]) | |
| if scope["subset_scope"] and (args.offset != 0 or args.limit not in (0, 8)): | |
| raise ValueError("Explicit job-ID subset cannot be additionally sliced") | |
| chosen = requested if scope["subset_scope"] else generation.select_jobs(jobs, args.offset, args.limit, 0, 1) | |
| if not 8 <= args.comparisons <= 16: | |
| raise ValueError("Use 8..16 comparison references") | |
| model_sha = probe.sha256(probe.MODEL_DIR / "verified_download.json") | |
| records = ([None] * len(chosen) if args.wait_for_generation | |
| else [generation.verify_resume(job, jobs_sha, model_sha) for job in chosen]) | |
| if not args.wait_for_generation and any(record is None for record in records): | |
| raise ValueError("Cannot screen ungenerated jobs") | |
| evidence = verify_models() | |
| normalizer = StrictNormalizer(read_dictionary(OPENCC / "TSCharacters.txt"), | |
| read_dictionary(OPENCC / "TSPhrases.txt"), | |
| json.loads((MODEL / "normalizer.json").read_text())) | |
| processor = WhisperProcessor.from_pretrained(str(MODEL), local_files_only=True) | |
| model = WhisperForConditionalGeneration.from_pretrained(str(MODEL), local_files_only=True).to("cpu").eval() | |
| sessions, embeddings = {}, {} | |
| def load_audio(path: str): | |
| values, rate = sf.read(path, dtype="float32", always_2d=False) | |
| if values.ndim == 2: | |
| values = values.mean(axis=1) | |
| if rate != 16000: | |
| divisor = math.gcd(rate, 16000) | |
| values = resample_poly(values, 16000 // divisor, rate // divisor) | |
| return values.astype(np.float32) | |
| def embedding(path: str, language: str): | |
| key = path, language | |
| if key not in embeddings: | |
| features = torchaudio.compliance.kaldi.fbank(torch.from_numpy(load_audio(path)).unsqueeze(0) * 32768, | |
| num_mel_bins=80, frame_length=25, frame_shift=10, sample_frequency=16000, | |
| dither=0, window_type="hamming", use_energy=False) | |
| features -= features.mean(dim=0, keepdim=True) | |
| output = sessions[language].run(None, {"feats": features.numpy()[None].astype(np.float32)})[0][0] | |
| embeddings[key] = output / max(float(np.linalg.norm(output)), 1e-12) | |
| return embeddings[key] | |
| for language in sorted({job["language"] for job in chosen}): | |
| name = "wespeaker_cn_cnceleb_resnet34_LM.onnx" if language == "zh" else "wespeaker_en_voxceleb_resnet34_LM.onnx" | |
| path = generation.PUBLIC / "vidproc/speaker_diarization/wespeaker-voxceleb-resnet34-LM" / name | |
| options = ort.SessionOptions() | |
| options.intra_op_num_threads, options.inter_op_num_threads = args.threads, 1 | |
| sessions[language] = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"]) | |
| evidence.append(dict(model=str(path), sha256=probe.sha256(path), providers=sessions[language].get_providers())) | |
| results = [] | |
| for index, (job, record) in enumerate(zip(chosen, records)): | |
| if record is None: | |
| wait_started = time.monotonic() | |
| sidecar = generation.paths(job["job_id"])[2] | |
| if not sidecar.exists(): | |
| print(f"waiting for committed generation {job['job_id']}", flush=True) | |
| while not sidecar.exists(): | |
| if time.monotonic() - wait_started > 1800: | |
| raise TimeoutError("Timed out waiting for immutable generated provenance") | |
| time.sleep(2) | |
| record = generation.verify_resume(job, jobs_sha, model_sha) | |
| if record is None: | |
| raise ValueError("Generation vanished during QA preflight") | |
| records[index] = record | |
| started = time.monotonic() | |
| generation.verify_reference(job) | |
| comparison_jobs = reference_comparisons(job, pool, args.comparisons) | |
| if len(comparison_jobs) < 8: | |
| raise ValueError("Insufficient independent reference comparisons") | |
| for comparison in comparison_jobs: | |
| generation.verify_reference(comparison) | |
| language = job["language"] | |
| inputs = processor(load_audio(record["normalized"]["path"]), sampling_rate=16000, | |
| return_tensors="pt", return_attention_mask=True) | |
| with torch.inference_mode(): | |
| generated = model.generate(inputs.input_features, attention_mask=inputs.attention_mask, | |
| language="chinese" if language == "zh" else "english", | |
| task="transcribe", max_new_tokens=192, do_sample=False, | |
| return_dict_in_generate=True, return_timestamps=False) | |
| ids = generated.sequences | |
| transcript = processor.batch_decode(ids, skip_special_tokens=True)[0].strip() | |
| eos = bool((ids == model.config.eos_token_id).any().item()) | |
| expected = normalizer.normalize(job["text"], language) | |
| observed = normalizer.normalize(transcript, language) | |
| rate = levenshtein_rate(expected, observed, language) | |
| predicted = embedding(record["normalized"]["path"], language) | |
| positive = float(np.dot(predicted, embedding(job["reference_audio"], language))) | |
| comparisons = [dict(reference_source_key=other["reference_source_key"], reference_sha256=other["reference_sha256"], | |
| cosine=float(np.dot(predicted, embedding(other["reference_audio"], language)))) for other in comparison_jobs] | |
| maximum = max(item["cosine"] for item in comparisons) | |
| gate = engineering_gate(rate, positive, maximum, critical_tokens_preserved(expected, observed, language), | |
| record["generation"]["ended_with_eos"] and eos) | |
| semantic_reason = asr_semantic_reason(transcript, language, sources[job["text_source_asset_id"]], semantics, normalizer) | |
| if semantic_reason: | |
| gate["reasons"].append("asr_semantic_filter:" + semantic_reason) | |
| gate["engineering_eligible"] = False | |
| result = dict(job_id=job["job_id"], language=language, split=job["split"], expected_text=job["text"], | |
| asr_text=transcript, expected_normalized=expected, asr_normalized=observed, | |
| normalized_error_rate=rate, error_metric="CER" if language == "zh" else "WER", | |
| asr_semantic_filter_reason=semantic_reason, command_semantics_sha256=semantics_sha, | |
| raw_error_rate=generation.error_rate(job["text"], transcript, language), asr_ended_with_eos=eos, | |
| reference_cosine=positive, maximum_comparison_cosine=maximum, relative_margin=positive - maximum, | |
| comparisons=comparisons, status="automatic_screen_only_human_pending", **gate, | |
| audio_sha256=record["normalized"]["sha256"], job_sha256=record["job_sha256"], | |
| normalization_version=NORMALIZATION_VERSION, | |
| critical_token_policy="negation_arabic_and_chinese_numerals_v2", | |
| **scope, | |
| jobs_manifest_sha256=jobs_sha, reference_pool_sha256=pool_sha, qa_seconds=time.monotonic() - started) | |
| results.append(result) | |
| probe.write_json(qa_root / f"{job['job_id']}.json", result) | |
| print(json.dumps({k: result[k] for k in ("job_id", "language", "expected_text", "asr_text", | |
| "normalized_error_rate", "reference_cosine", "relative_margin", "engineering_eligible", "reasons")}, | |
| ensure_ascii=False), flush=True) | |
| report = dict(format="hfd_aside_asr_base_engineering_screen_v1", model_evidence=evidence, | |
| jobs_manifest_sha256=jobs_sha, reference_pool_sha256=pool_sha, qa_script_sha256=probe.sha256(Path(__file__)), | |
| command_semantics_sha256=semantics_sha, | |
| normalization_version=NORMALIZATION_VERSION, | |
| **scope, | |
| thresholds=dict(max_normalized_error_rate=.15, min_reference_cosine=.75, min_relative_margin=.05, | |
| calibrated=False, basis="conservative_engineering_filter_not_identity_authentication"), | |
| speaker_comparison_note="Different source WAVs are proxy comparisons, not gold different-speaker identities.", | |
| samples=results, automatically_eligible=sum(row["engineering_eligible"] for row in results), | |
| human_review="pending", same_speaker_verified=False, production_modified=False) | |
| probe.write_json(qa_root / f"summary_{args.offset}_{len(chosen)}.json", report) | |
| candidate_path = qa_root / f"eligible_candidates_{args.offset}_{len(chosen)}.jsonl" | |
| probe.safe_output(candidate_path) | |
| candidates = [] | |
| for job, record, result in zip(chosen, records, results): | |
| if result["engineering_eligible"]: | |
| qa_path = qa_root / f"{job['job_id']}.json" | |
| candidates.append(candidate_row(job, record, qa_path, jobs_sha)) | |
| write_candidates(candidate_path, candidates) | |
| if args.job_ids_file is not None and probe.sha256(args.job_ids_file) != scope["job_ids_file_sha256"]: | |
| raise ValueError("Requested job-ID selection changed during QA") | |
| return dict(samples=len(results), automatically_eligible=report["automatically_eligible"], human_review="pending", **scope) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("mode", choices=("download", "run", "recheck", "publish")) | |
| parser.add_argument("--input-report", type=Path) | |
| parser.add_argument("--job-ids-file", type=Path, help="Explicit retry subset; supersedes default --limit 8 and requires retry output") | |
| parser.add_argument("--jobs", type=Path, default=probe.WORK / "plan_reuse/tts_jobs.jsonl") | |
| parser.add_argument("--assets", type=Path, default=probe.WORK / "tts_assets_reuse") | |
| parser.add_argument("--reference-pool", type=Path, default=probe.WORK / "plan_reuse/tts_jobs.jsonl") | |
| parser.add_argument("--threads", type=int, default=4) | |
| parser.add_argument("--comparisons", type=int, default=16) | |
| parser.add_argument("--offset", type=int, default=0) | |
| parser.add_argument("--limit", type=int, default=8) | |
| parser.add_argument("--report-tag", default="qa_base_v2") | |
| parser.add_argument("--wait-for-generation", action="store_true", help="Wait only for immutable per-job provenance; never reads partial WAVs") | |
| args = parser.parse_args() | |
| result = (download() if args.mode == "download" else recheck(args) if args.mode == "recheck" | |
| else publish(args) if args.mode == "publish" else run(args)) | |
| print(json.dumps(result, ensure_ascii=False), flush=True) | |
| if __name__ == "__main__": | |
| main() | |