code-HFD_lzk / scripts /data /qa_hfd_same_voice_asides.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
8562e9b verified
Raw History Blame Contribute Delete
33.5 kB
#!/usr/bin/env python3
"""CPU-only ASR/speaker screening; engineering eligibility is not human approval."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import re
import sys
import time
import unicodedata
import urllib.request
sys.path.insert(0, str(Path(__file__).resolve().parent))
import generate_hfd_same_voice_asides as generation
import probe_hfd_same_voice_tts as probe
if str(probe.REPO) not in sys.path:
sys.path.insert(0, str(probe.REPO))
from scripts.data.build_hfd_diverse_catalog import filter_text
from scripts.data.build_hfd_cross_catalog import new_source_filter_reason
MODEL_ID = "openai/whisper-base"
REVISION = "e37978b90ca9030d5170a5c07aadb050351a65bb"
MODEL = probe.WORK / "models/whisper-base"
OPENCC_REVISION = "2939943bd6f4d459b46d7fdcf07a885ab3f01761"
OPENCC = probe.WORK / "models/opencc_t2s"
NORMALIZATION_VERSION = "strict_t2s_numbers_spelling_adjacent_neutral_filler_v2"
OPENCC_FILES = {
"data/dictionary/TSCharacters.txt": (104520, "cbe56a8327ced95c7d91eacb011090301bd158fc"),
"data/dictionary/TSPhrases.txt": (8835, "2faff4ece295d20bdec857fb8838e2ba1b9bfdb9"),
"data/config/t2s.json": (589, "c6e389cfc5aaba56cccdfa287758347a0d8f1398"),
"LICENSE": (9165, "9f7a310eb2b143dfc4b73551ea60ca17eb2b981f"),
}
MODEL_FILES = {"README.md", "added_tokens.json", "config.json", "generation_config.json", "merges.txt",
"model.safetensors", "normalizer.json", "preprocessor_config.json", "special_tokens_map.json",
"tokenizer.json", "tokenizer_config.json", "vocab.json"}
def download_file(url: str, path: Path, item: dict) -> dict:
probe.safe_output(path)
path.parent.mkdir(parents=True, exist_ok=True)
if not path.exists():
temporary = path.with_suffix(path.suffix + ".part")
probe.safe_output(temporary)
with urllib.request.urlopen(url, timeout=60) as response, temporary.open("wb") as handle:
for chunk in iter(lambda: response.read(8 * 1024 * 1024), b""):
handle.write(chunk)
probe.verify_file(temporary, item)
temporary.replace(path)
return {**item, "sha256": probe.verify_file(path, item), "url": url, "path": str(path)}
def download() -> dict:
api = f"https://huggingface.co/api/models/{MODEL_ID}/revision/{REVISION}?blobs=true"
with urllib.request.urlopen(api, timeout=30) as response:
metadata = json.load(response)
if metadata["sha"] != REVISION or metadata["id"] != MODEL_ID:
raise ValueError("Official ASR revision mismatch")
files = [item for item in metadata["siblings"] if item["rfilename"] in MODEL_FILES]
if {item["rfilename"] for item in files} != MODEL_FILES:
raise ValueError("Required ASR file missing")
total = sum(item["size"] for item in files)
if total + sum(item[0] for item in OPENCC_FILES.values()) > 1_000_000_000:
raise ValueError("Download exceeds approved 1 GB budget")
result = dict(model_id=MODEL_ID, revision=REVISION, required_bytes=total, files=[])
print(json.dumps({"required_bytes": total, "revision": REVISION}), flush=True)
for item in files:
name = item["rfilename"]
url = f"https://huggingface.co/{MODEL_ID}/resolve/{REVISION}/{name}"
result["files"].append(download_file(url, MODEL / name, item))
print(f"verified ASR {name}", flush=True)
result["complete"] = True
probe.write_json(MODEL / "verified_download.json", result)
dictionaries = []
for name, (size, blob) in OPENCC_FILES.items():
item = dict(rfilename=name, size=size, blobId=blob)
url = f"https://raw.githubusercontent.com/BYVoid/OpenCC/{OPENCC_REVISION}/{name}"
dictionaries.append(download_file(url, OPENCC / Path(name).name, item))
probe.write_json(OPENCC / "verified_download.json", dict(revision=OPENCC_REVISION, files=dictionaries, complete=True))
return dict(asr_bytes=total, opencc_bytes=sum(item[0] for item in OPENCC_FILES.values()), complete=True)
def verify_models() -> list[dict]:
evidence = []
for root, expected_revision in ((MODEL, REVISION), (OPENCC, OPENCC_REVISION)):
manifest_path = root / "verified_download.json"
manifest = json.loads(manifest_path.read_text())
if not manifest["complete"] or manifest["revision"] != expected_revision:
raise ValueError("QA model/dictionary revision drift")
for item in manifest["files"]:
path = root / Path(item["rfilename"]).name
if probe.verify_file(path, item) != item["sha256"]:
raise ValueError("QA model/dictionary content drift")
evidence.append(dict(root=str(root), revision=expected_revision, manifest_sha256=probe.sha256(manifest_path)))
return evidence
def read_dictionary(path: Path) -> dict[str, str]:
result = {}
for line in path.read_text().splitlines():
if not line.strip() or line.startswith("#"):
continue
source, replacements = line.split("\t", 1)
result[source] = replacements.split(" ")[0]
return result
class StrictNormalizer:
"""No removal of bracket contents, fillers, negatives, or changed numbers."""
def __init__(self, characters: dict, phrases: dict, spelling: dict):
self.mapping = {**characters, **phrases}
self.max_key_length = max((len(key) for key in self.mapping), default=1)
self.spelling = spelling
def to_simplified(self, text: str) -> str:
result = []
index = 0
while index < len(text):
for size in range(min(self.max_key_length, len(text) - index), 0, -1):
key = text[index:index + size]
if key in self.mapping:
result.append(self.mapping[key])
index += size
break
else:
result.append(text[index])
index += 1
return "".join(result)
def normalize(self, text: str, language: str) -> str:
text = unicodedata.normalize("NFKC", text).lower().replace("’", "'")
if language == "zh":
value = "".join(char for char in self.to_simplified(text) if char.isalnum())
# Only contiguous repetitions of explicitly neutral hesitation markers.
# Never drop a single filler, negation, number or full referential phrase.
return re.sub(r"(那个|嗯|呃)(?:\1)+", r"\1", value)
if language != "en":
raise ValueError("Unsupported normalization language")
from transformers.models.whisper.english_normalizer import EnglishNumberNormalizer, EnglishSpellingNormalizer
replacements = {"can't": "can not", "won't": "will not", "i'm": "i am", "it's": "it is",
"that's": "that is", "there's": "there is", "let's": "let us"}
for source, replacement in replacements.items():
text = re.sub(r"\b" + re.escape(source) + r"\b", replacement, text)
text = re.sub(r"\b([a-z]+)n't\b", r"\1 not", text)
text = re.sub(r"\b([a-z]+)'re\b", r"\1 are", text)
text = re.sub(r"\b([a-z]+)'ve\b", r"\1 have", text)
text = re.sub(r"\b([a-z]+)'ll\b", r"\1 will", text)
text = re.sub(r"(?<=[a-z])-(?=[a-z])", " ", text)
text = re.sub(r"\ba (?=hundred\b|thousand\b|million\b|billion\b)", "one ", text)
# Retain bracket contents and decimal/sign/currency semantics.
text = " ".join(re.findall(r"[a-z]+(?:'[a-z]+)?|[+-]?\d+(?:\.\d+)?|[%$£€]", text))
text = EnglishSpellingNormalizer(self.spelling)(EnglishNumberNormalizer()(text))
return re.sub(r"\b(um|uh|hmm)(?:\s+\1)+\b", r"\1", text)
def levenshtein_rate(expected: str, observed: str, language: str) -> float:
left, right = (list(expected), list(observed)) if language == "zh" else (expected.split(), observed.split())
if not left:
raise ValueError("Expected normalized text empty")
previous = list(range(len(right) + 1))
for i, token in enumerate(left, 1):
current = [i]
for j, candidate in enumerate(right, 1):
current.append(min(previous[j] + 1, current[-1] + 1, previous[j - 1] + (token != candidate)))
previous = current
return previous[-1] / len(left)
def critical_tokens_preserved(expected: str, observed: str, language: str) -> bool:
from collections import Counter
if language == "en":
extractor = lambda value: re.findall(r"\b(?:not|no|never)\b|[+-]?\d+(?:\.\d+)?", value)
else:
extractor = lambda value: re.findall(r"[不没无别]|[零〇一二两三四五六七八九十百千万亿半壹贰叁肆伍陆柒捌玖拾佰仟]+|[0-9]+(?:\.[0-9]+)?", value)
return Counter(extractor(expected)) == Counter(extractor(observed))
def engineering_gate(rate: float, cosine: float, maximum_comparison: float, critical_match: bool,
observed_eos: bool, *, max_rate: float = .15, min_cosine: float = .75,
min_margin: float = .05) -> dict:
reasons = []
import math
if not all(math.isfinite(value) for value in (rate, cosine, maximum_comparison)):
reasons.append("nonfinite_measurement")
if rate > max_rate:
reasons.append("asr_text_difference")
if not critical_match:
reasons.append("negation_or_numeric_difference")
if cosine < min_cosine:
reasons.append("low_reference_similarity")
if cosine - maximum_comparison < min_margin:
reasons.append("low_relative_reference_margin")
if not observed_eos:
reasons.append("missing_generation_eos")
return dict(engineering_eligible=not reasons, reasons=reasons, human_review="pending",
same_speaker_verified=False, thresholds_calibrated=False)
def reference_comparisons(job: dict, pool: list[dict], count: int) -> list[dict]:
voice = Path(job["reference_audio"]).stem
gender = voice.split("_")[0]
candidates = [row for row in pool if row["language"] == job["language"]
and row["reference_sha256"] != job["reference_sha256"]]
candidates.sort(key=lambda row: (Path(row["reference_audio"]).stem != voice,
Path(row["reference_audio"]).stem.split("_")[0] != gender,
row["reference_sha256"]))
unique = {}
for row in candidates:
unique.setdefault(row["reference_sha256"], row)
return list(unique.values())[:count]
def asr_semantic_reason(transcript: str, language: str, source: dict, semantics: dict,
normalizer: StrictNormalizer) -> str | None:
# Preserve punctuation/question marks while normalizing traditional glyphs.
checked_text = normalizer.to_simplified(unicodedata.normalize("NFKC", transcript)) if language == "zh" else transcript
return (filter_text(checked_text, language, set(semantics["target_phrases"][language]), semantics=semantics)
or new_source_filter_reason({**source, "transcript": checked_text}, semantics))
def candidate_row(job: dict, record: dict, qa_path: Path, jobs_sha: str) -> dict:
provenance = generation.paths(job["job_id"])[2]
return dict(asset_id="tts_clone:" + job["job_id"], job_id=job["job_id"],
reference_audio=job["reference_audio"], reference_sha256=job["reference_sha256"],
reference_text=job["reference_text"], reference_source_key=job["reference_source_key"],
split=job["split"], language=job["language"], transcript=job["text"], role="non_command_speech",
normalized_path=record["normalized"]["path"], normalized_sha256=record["normalized"]["sha256"],
sample_count=record["normalized"]["sample_count"], sample_rate=16000,
speaker_id="tts_ref:" + job["reference_sha256"], source_group="tts_ref:" + job["reference_sha256"],
voice_relation="reference_clone", eligible_for_mixing=True, engineering_eligible=True,
job_sha256=record["job_sha256"], jobs_manifest_sha256=jobs_sha,
generation_provenance_path=str(provenance), generation_provenance_sha256=probe.sha256(provenance),
qa_path=str(qa_path), qa_sha256=probe.sha256(qa_path), human_review="pending", same_speaker_verified=False,
text_source_asset_id=job["text_source_asset_id"], normalization_version=NORMALIZATION_VERSION)
def requested_jobs(jobs: list[dict], path: Path | None) -> tuple[list[dict], dict]:
if path is None:
return jobs, dict(subset_scope=False, requested_count=len(jobs), full_plan_job_count=len(jobs))
text = path.read_text(encoding="utf-8").strip()
if not text:
raise ValueError("Job-ID selection must not be empty")
if text[0] in "[{":
value = json.loads(text)
ids = value.get("job_ids") if isinstance(value, dict) else value
else:
ids = [line.strip() for line in text.splitlines() if line.strip()]
if not isinstance(ids, list) or not ids or any(not isinstance(i, str) or not re.fullmatch(r"[0-9a-f]{64}", i) for i in ids):
raise ValueError("Job IDs must be a nonempty JSON list/job_ids object or one ID per line")
if len(ids) != len(set(ids)):
raise ValueError("Duplicate requested job IDs")
selected = [job for job in jobs if job["job_id"] in set(ids)]
if len(selected) != len(ids):
raise ValueError("Unknown job ID outside frozen complete manifest")
return selected, dict(subset_scope=True, requested_count=len(ids), full_plan_job_count=len(jobs),
job_ids_file=str(path), job_ids_file_sha256=probe.sha256(path))
def protect_subset_output(root: Path, subset_scope: bool) -> None:
if subset_scope and not re.fullmatch(r"tts_assets_retry[1-9][0-9]*", root.name):
raise ValueError("Requested-subset QA must use an isolated tts_assets_retryN output, never frozen originals")
def write_candidates(path: Path, rows: list[dict]) -> None:
probe.safe_output(path)
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(".jsonl.tmp")
probe.safe_output(temporary)
temporary.write_text("".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in rows))
temporary.replace(path)
def recheck(args) -> dict:
"""Re-evaluate saved ASR/speaker evidence without regenerating or retranscribing."""
generation.ASSETS = args.assets.absolute()
probe.safe_output(generation.ASSETS)
if args.input_report is None or not re.fullmatch(r"[a-z0-9_]+", args.report_tag):
raise ValueError("Recheck requires an existing report and safe new report tag")
prior_sha = probe.sha256(args.input_report)
report = json.loads(args.input_report.read_text())
jobs, jobs_sha = generation.read_jobs(args.jobs)
if args.job_ids_file is not None:
raise ValueError("Recheck preserves its original report scope; use run/publish for job-ID selection")
by_id = {job["job_id"]: job for job in jobs}
if jobs_sha != report["jobs_manifest_sha256"]:
raise ValueError("Prior report belongs to a different frozen plan")
model_sha = probe.sha256(probe.MODEL_DIR / "verified_download.json")
metadata = json.loads((args.jobs.parent / "preparation_report.json").read_text())
semantics_path = Path(metadata["reference_root"]) / "work/catalog/manifests/command_semantics.json"
if probe.sha256(semantics_path) != report["command_semantics_sha256"]:
raise ValueError("Semantic policy drift")
semantics = json.loads(semantics_path.read_text())
source_path = args.jobs.parent / "selected_real_assets.jsonl"
if probe.sha256(source_path) != metadata["output_hashes"][source_path.name]:
raise ValueError("Text source registry drift")
sources = {row["asset_id"]: row for row in map(json.loads, source_path.read_text().splitlines())}
verify_models()
normalizer = StrictNormalizer(read_dictionary(OPENCC / "TSCharacters.txt"), read_dictionary(OPENCC / "TSPhrases.txt"),
json.loads((MODEL / "normalizer.json").read_text()))
qa_root = generation.ASSETS / args.report_tag
candidates = []
for result in report["samples"]:
job = by_id[result["job_id"]]
record = generation.verify_resume(job, jobs_sha, model_sha)
if record is None or result["audio_sha256"] != record["normalized"]["sha256"] or result["job_sha256"] != record["job_sha256"]:
raise ValueError("Saved QA evidence does not match immutable generation")
expected = normalizer.normalize(job["text"], job["language"])
observed = normalizer.normalize(result["asr_text"], job["language"])
rate = levenshtein_rate(expected, observed, job["language"])
gate = engineering_gate(rate, result["reference_cosine"], result["maximum_comparison_cosine"],
critical_tokens_preserved(expected, observed, job["language"]),
result["asr_ended_with_eos"] and record["generation"]["ended_with_eos"])
reason = asr_semantic_reason(result["asr_text"], job["language"], sources[job["text_source_asset_id"]], semantics, normalizer)
if reason:
gate["engineering_eligible"] = False
gate["reasons"].append("asr_semantic_filter:" + reason)
result.update(gate, expected_normalized=expected, asr_normalized=observed, normalized_error_rate=rate,
asr_semantic_filter_reason=reason, normalization_version=NORMALIZATION_VERSION,
critical_token_policy="negation_arabic_and_chinese_numerals_v2", prior_qa_summary_sha256=prior_sha)
path = qa_root / f"{job['job_id']}.json"
probe.write_json(path, result)
if result["engineering_eligible"]:
candidates.append(candidate_row(job, record, path, jobs_sha))
report.update(automatically_eligible=len(candidates), normalization_version=NORMALIZATION_VERSION,
critical_token_policy="negation_arabic_and_chinese_numerals_v2",
assessment_mode="recheck_saved_asr_and_speaker_evidence", prior_qa_summary_path=str(args.input_report),
prior_qa_summary_sha256=prior_sha, qa_script_sha256=probe.sha256(Path(__file__)))
probe.write_json(qa_root / f"summary_recheck_{len(report['samples'])}.json", report)
write_candidates(qa_root / f"eligible_candidates_recheck_{len(report['samples'])}.jsonl", candidates)
return dict(samples=len(report["samples"]), automatically_eligible=len(candidates), human_review="pending")
def publish(args) -> dict:
from collections import Counter
generation.ASSETS = args.assets.absolute()
probe.safe_output(generation.ASSETS)
if not re.fullmatch(r"[a-z0-9_]+", args.report_tag):
raise ValueError("Unsafe QA report tag")
jobs, jobs_sha = generation.read_jobs(args.jobs)
chosen, scope = requested_jobs(jobs, args.job_ids_file)
protect_subset_output(generation.ASSETS, scope["subset_scope"])
model_sha = probe.sha256(probe.MODEL_DIR / "verified_download.json")
candidates, assessed, rejected = [], [], []
for job in chosen:
path = generation.ASSETS / args.report_tag / f"{job['job_id']}.json"
if not path.exists():
continue
result = json.loads(path.read_text())
record = generation.verify_resume(job, jobs_sha, model_sha)
if (record is None or result["job_sha256"] != record["job_sha256"]
or result["audio_sha256"] != record["normalized"]["sha256"]
or result.get("critical_token_policy") != "negation_arabic_and_chinese_numerals_v2"
or result["normalization_version"] != NORMALIZATION_VERSION):
raise ValueError("Candidate QA/audio/job policy contract mismatch")
assessed.append(job["job_id"])
if result["engineering_eligible"]:
if result["same_speaker_verified"] or result["human_review"] != "pending":
raise ValueError("This automatic screen cannot claim human identity approval")
candidates.append(candidate_row(job, record, path, jobs_sha))
else:
rejected.append(dict(job_id=job["job_id"], reasons=result["reasons"]))
path = generation.ASSETS / "candidate_registry.jsonl"
write_candidates(path, candidates)
report = dict(format="hfd_same_voice_candidate_registry_v1", candidate_registry_sha256=probe.sha256(path),
planned_jobs=len(jobs), assessed_jobs=len(assessed), candidate_jobs=len(candidates), **scope,
candidate_counts={"/".join(key): count for key, count in Counter((r["split"], r["language"]) for r in candidates).items()},
jobs_manifest_sha256=jobs_sha, qa_report_tag=args.report_tag, complete=len(assessed) == len(chosen),
complete_scope="requested_subset" if scope["subset_scope"] else "full_plan",
full_plan_complete=not scope["subset_scope"] and len(assessed) == len(jobs),
state="final_qa_candidate_snapshot" if len(assessed) == len(chosen) else "incremental_candidate_snapshot_do_not_freeze_yet",
rejected_jobs=rejected, human_review="pending", same_speaker_verified=False, production_modified=False)
probe.write_json(generation.ASSETS / "candidate_report.json", report)
return {key: report[key] for key in ("planned_jobs", "requested_count", "subset_scope", "assessed_jobs", "candidate_jobs", "candidate_counts", "complete")}
def run(args) -> dict:
generation.configure_cpu(args.threads)
import math
import numpy as np
import onnxruntime as ort
import soundfile as sf
from scipy.signal import resample_poly
import torch
import torchaudio
from transformers import WhisperForConditionalGeneration, WhisperProcessor
torch.set_num_threads(args.threads)
torch.set_num_interop_threads(1)
generation.ASSETS = args.assets.absolute()
probe.safe_output(generation.ASSETS)
if not re.fullmatch(r"[a-z0-9_]+", args.report_tag):
raise ValueError("Unsafe QA report tag")
qa_root = generation.ASSETS / args.report_tag
jobs, jobs_sha = generation.read_jobs(args.jobs)
pool, pool_sha = generation.read_jobs(args.reference_pool)
plan_report = json.loads((args.jobs.parent / "preparation_report.json").read_text())
semantics_path = Path(plan_report["reference_root"]) / "work/catalog/manifests/command_semantics.json"
semantics_sha = probe.sha256(semantics_path)
if semantics_sha != plan_report["semantics_sha256"]:
raise ValueError("Frozen semantic filter changed")
semantics = json.loads(semantics_path.read_text())
source_path = args.jobs.parent / "selected_real_assets.jsonl"
if probe.sha256(source_path) != plan_report["output_hashes"][source_path.name]:
raise ValueError("Frozen text-source catalog changed")
sources = {row["asset_id"]: row for row in (json.loads(line) for line in source_path.read_text().splitlines())}
requested, scope = requested_jobs(jobs, args.job_ids_file)
protect_subset_output(generation.ASSETS, scope["subset_scope"])
if scope["subset_scope"] and (args.offset != 0 or args.limit not in (0, 8)):
raise ValueError("Explicit job-ID subset cannot be additionally sliced")
chosen = requested if scope["subset_scope"] else generation.select_jobs(jobs, args.offset, args.limit, 0, 1)
if not 8 <= args.comparisons <= 16:
raise ValueError("Use 8..16 comparison references")
model_sha = probe.sha256(probe.MODEL_DIR / "verified_download.json")
records = ([None] * len(chosen) if args.wait_for_generation
else [generation.verify_resume(job, jobs_sha, model_sha) for job in chosen])
if not args.wait_for_generation and any(record is None for record in records):
raise ValueError("Cannot screen ungenerated jobs")
evidence = verify_models()
normalizer = StrictNormalizer(read_dictionary(OPENCC / "TSCharacters.txt"),
read_dictionary(OPENCC / "TSPhrases.txt"),
json.loads((MODEL / "normalizer.json").read_text()))
processor = WhisperProcessor.from_pretrained(str(MODEL), local_files_only=True)
model = WhisperForConditionalGeneration.from_pretrained(str(MODEL), local_files_only=True).to("cpu").eval()
sessions, embeddings = {}, {}
def load_audio(path: str):
values, rate = sf.read(path, dtype="float32", always_2d=False)
if values.ndim == 2:
values = values.mean(axis=1)
if rate != 16000:
divisor = math.gcd(rate, 16000)
values = resample_poly(values, 16000 // divisor, rate // divisor)
return values.astype(np.float32)
def embedding(path: str, language: str):
key = path, language
if key not in embeddings:
features = torchaudio.compliance.kaldi.fbank(torch.from_numpy(load_audio(path)).unsqueeze(0) * 32768,
num_mel_bins=80, frame_length=25, frame_shift=10, sample_frequency=16000,
dither=0, window_type="hamming", use_energy=False)
features -= features.mean(dim=0, keepdim=True)
output = sessions[language].run(None, {"feats": features.numpy()[None].astype(np.float32)})[0][0]
embeddings[key] = output / max(float(np.linalg.norm(output)), 1e-12)
return embeddings[key]
for language in sorted({job["language"] for job in chosen}):
name = "wespeaker_cn_cnceleb_resnet34_LM.onnx" if language == "zh" else "wespeaker_en_voxceleb_resnet34_LM.onnx"
path = generation.PUBLIC / "vidproc/speaker_diarization/wespeaker-voxceleb-resnet34-LM" / name
options = ort.SessionOptions()
options.intra_op_num_threads, options.inter_op_num_threads = args.threads, 1
sessions[language] = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"])
evidence.append(dict(model=str(path), sha256=probe.sha256(path), providers=sessions[language].get_providers()))
results = []
for index, (job, record) in enumerate(zip(chosen, records)):
if record is None:
wait_started = time.monotonic()
sidecar = generation.paths(job["job_id"])[2]
if not sidecar.exists():
print(f"waiting for committed generation {job['job_id']}", flush=True)
while not sidecar.exists():
if time.monotonic() - wait_started > 1800:
raise TimeoutError("Timed out waiting for immutable generated provenance")
time.sleep(2)
record = generation.verify_resume(job, jobs_sha, model_sha)
if record is None:
raise ValueError("Generation vanished during QA preflight")
records[index] = record
started = time.monotonic()
generation.verify_reference(job)
comparison_jobs = reference_comparisons(job, pool, args.comparisons)
if len(comparison_jobs) < 8:
raise ValueError("Insufficient independent reference comparisons")
for comparison in comparison_jobs:
generation.verify_reference(comparison)
language = job["language"]
inputs = processor(load_audio(record["normalized"]["path"]), sampling_rate=16000,
return_tensors="pt", return_attention_mask=True)
with torch.inference_mode():
generated = model.generate(inputs.input_features, attention_mask=inputs.attention_mask,
language="chinese" if language == "zh" else "english",
task="transcribe", max_new_tokens=192, do_sample=False,
return_dict_in_generate=True, return_timestamps=False)
ids = generated.sequences
transcript = processor.batch_decode(ids, skip_special_tokens=True)[0].strip()
eos = bool((ids == model.config.eos_token_id).any().item())
expected = normalizer.normalize(job["text"], language)
observed = normalizer.normalize(transcript, language)
rate = levenshtein_rate(expected, observed, language)
predicted = embedding(record["normalized"]["path"], language)
positive = float(np.dot(predicted, embedding(job["reference_audio"], language)))
comparisons = [dict(reference_source_key=other["reference_source_key"], reference_sha256=other["reference_sha256"],
cosine=float(np.dot(predicted, embedding(other["reference_audio"], language)))) for other in comparison_jobs]
maximum = max(item["cosine"] for item in comparisons)
gate = engineering_gate(rate, positive, maximum, critical_tokens_preserved(expected, observed, language),
record["generation"]["ended_with_eos"] and eos)
semantic_reason = asr_semantic_reason(transcript, language, sources[job["text_source_asset_id"]], semantics, normalizer)
if semantic_reason:
gate["reasons"].append("asr_semantic_filter:" + semantic_reason)
gate["engineering_eligible"] = False
result = dict(job_id=job["job_id"], language=language, split=job["split"], expected_text=job["text"],
asr_text=transcript, expected_normalized=expected, asr_normalized=observed,
normalized_error_rate=rate, error_metric="CER" if language == "zh" else "WER",
asr_semantic_filter_reason=semantic_reason, command_semantics_sha256=semantics_sha,
raw_error_rate=generation.error_rate(job["text"], transcript, language), asr_ended_with_eos=eos,
reference_cosine=positive, maximum_comparison_cosine=maximum, relative_margin=positive - maximum,
comparisons=comparisons, status="automatic_screen_only_human_pending", **gate,
audio_sha256=record["normalized"]["sha256"], job_sha256=record["job_sha256"],
normalization_version=NORMALIZATION_VERSION,
critical_token_policy="negation_arabic_and_chinese_numerals_v2",
**scope,
jobs_manifest_sha256=jobs_sha, reference_pool_sha256=pool_sha, qa_seconds=time.monotonic() - started)
results.append(result)
probe.write_json(qa_root / f"{job['job_id']}.json", result)
print(json.dumps({k: result[k] for k in ("job_id", "language", "expected_text", "asr_text",
"normalized_error_rate", "reference_cosine", "relative_margin", "engineering_eligible", "reasons")},
ensure_ascii=False), flush=True)
report = dict(format="hfd_aside_asr_base_engineering_screen_v1", model_evidence=evidence,
jobs_manifest_sha256=jobs_sha, reference_pool_sha256=pool_sha, qa_script_sha256=probe.sha256(Path(__file__)),
command_semantics_sha256=semantics_sha,
normalization_version=NORMALIZATION_VERSION,
**scope,
thresholds=dict(max_normalized_error_rate=.15, min_reference_cosine=.75, min_relative_margin=.05,
calibrated=False, basis="conservative_engineering_filter_not_identity_authentication"),
speaker_comparison_note="Different source WAVs are proxy comparisons, not gold different-speaker identities.",
samples=results, automatically_eligible=sum(row["engineering_eligible"] for row in results),
human_review="pending", same_speaker_verified=False, production_modified=False)
probe.write_json(qa_root / f"summary_{args.offset}_{len(chosen)}.json", report)
candidate_path = qa_root / f"eligible_candidates_{args.offset}_{len(chosen)}.jsonl"
probe.safe_output(candidate_path)
candidates = []
for job, record, result in zip(chosen, records, results):
if result["engineering_eligible"]:
qa_path = qa_root / f"{job['job_id']}.json"
candidates.append(candidate_row(job, record, qa_path, jobs_sha))
write_candidates(candidate_path, candidates)
if args.job_ids_file is not None and probe.sha256(args.job_ids_file) != scope["job_ids_file_sha256"]:
raise ValueError("Requested job-ID selection changed during QA")
return dict(samples=len(results), automatically_eligible=report["automatically_eligible"], human_review="pending", **scope)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=("download", "run", "recheck", "publish"))
parser.add_argument("--input-report", type=Path)
parser.add_argument("--job-ids-file", type=Path, help="Explicit retry subset; supersedes default --limit 8 and requires retry output")
parser.add_argument("--jobs", type=Path, default=probe.WORK / "plan_reuse/tts_jobs.jsonl")
parser.add_argument("--assets", type=Path, default=probe.WORK / "tts_assets_reuse")
parser.add_argument("--reference-pool", type=Path, default=probe.WORK / "plan_reuse/tts_jobs.jsonl")
parser.add_argument("--threads", type=int, default=4)
parser.add_argument("--comparisons", type=int, default=16)
parser.add_argument("--offset", type=int, default=0)
parser.add_argument("--limit", type=int, default=8)
parser.add_argument("--report-tag", default="qa_base_v2")
parser.add_argument("--wait-for-generation", action="store_true", help="Wait only for immutable per-job provenance; never reads partial WAVs")
args = parser.parse_args()
result = (download() if args.mode == "download" else recheck(args) if args.mode == "recheck"
else publish(args) if args.mode == "publish" else run(args))
print(json.dumps(result, ensure_ascii=False), flush=True)
if __name__ == "__main__":
main()