#!/usr/bin/env python3 """Verified IHM mirror fallback; keep only fully contained annotated sentences.""" from __future__ import annotations import argparse from collections import Counter, defaultdict from concurrent.futures import ThreadPoolExecutor import hashlib import io import json from pathlib import Path import struct import subprocess import sys import wave import xml.etree.ElementTree as ET import zipfile import numpy as np import pyarrow.parquet as pq ROOT = Path(__file__).resolve().parents[2] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from scripts.data.hfd_third_party_common import read_jsonl, sha256_file, write_json_atomic, write_jsonl_atomic REVISION = '46f28f2503e2ec48f8867a84eef356c70476beab' BASE = 'https://huggingface.co/datasets/edinburghcstr/ami' def decode_mirror_wav(payload): """The mirror stores original PCM16 values in IEEE float WAV containers.""" if payload[:4] != b'RIFF' or payload[8:12] != b'WAVE': raise ValueError('Mirror audio is not a WAV') pos = 12 fmt = pcm = None while pos + 8 <= len(payload): kind, size = struct.unpack_from('<4sI', payload, pos) body = payload[pos + 8:pos + 8 + size] if len(body) != size: raise ValueError('Truncated mirror WAV') if kind == b'fmt ': fmt = struct.unpack_from(' 1e-4: raise ValueError('Mirror audio is not losslessly representable as PCM16') if np.min(values, initial=0) < -32768 or np.max(values, initial=0) > 32767: raise ValueError('Mirror PCM16 range exceeded') return np.rint(values).astype(' 16 or abs(row['end_time'] * 16000 - end) > 16: raise ValueError('Mirror timestamps disagree with segment ID') return start, end def annotated_spoken_bounds(planned, archive): """Drop optional context padding, never any annotated spoken sample.""" cache = {} with zipfile.ZipFile(archive) as source: for row in planned: member = row['words_member'] if member not in cache: words = list(ET.fromstring(source.read(member))) indices = {w.get('{http://nite.sourceforge.net/}id'): i for i, w in enumerate(words)} cache[member] = words, indices words, indices = cache[member] selected = words[indices[row['word_id_first']]:indices[row['word_id_last']] + 1] spoken = [w for w in selected if w.tag == 'w' and w.get('punc') != 'true'] first = min(float(w.get('starttime')) for w in spoken) last = max(float(w.get('endtime')) for w in spoken) if first < row['start_seconds'] - .0001 or last > row['end_seconds'] + .0001: raise ValueError('Spoken words lie outside planned sentence') row['planned_context_start_seconds'] = row['start_seconds'] row['planned_context_end_seconds'] = row['end_seconds'] row['start_seconds'], row['end_seconds'] = first, last row['boundary_policy'] = 'all_annotated_words_no_optional_silence_margin' return planned def download(raw, workers): revision = json.loads((raw / 'ami_hf_revision.json').read_text())['sha'] if revision != REVISION: raise ValueError('Unexpected mirror revision; review before changing the pin') files = json.loads((raw / 'ami_hf_tree.json').read_text()) directory = raw / 'ami_hf' directory.mkdir(exist_ok=True) def fetch(row): path = directory / Path(row['path']).name expected = row['lfs']['oid'] if not path.exists() or path.stat().st_size != row['size'] or sha256_file(path) != expected: subprocess.run(['curl', '-fsSL', '--retry', '3', '--retry-all-errors', '--connect-timeout', '20', '--max-time', '900', '-o', str(path), f"{BASE}/resolve/{REVISION}/{row['path']}?download=true"], check=True) if path.stat().st_size != row['size'] or sha256_file(path) != expected: raise ValueError(f'Mirror LFS hash mismatch: {path.name}') print(f'verified {path.name}', flush=True) with ThreadPoolExecutor(max_workers=workers) as pool: list(pool.map(fetch, [r for r in files if r['path'].endswith('.parquet')])) def materialize(raw): planned = annotated_spoken_bounds(read_jsonl(raw / 'ami_plan.jsonl'), raw / 'ami_public_manual_1.6.2.zip') groups = defaultdict(list) for row in planned: groups[row['meeting'], row['raw_speaker'], f"H{int(row['channel']):02d}"].append(row) accepted = {} rejected = {} directory = raw / 'ami_mirror_clips' directory.mkdir(exist_ok=True) for info in json.loads((raw / 'ami_hf_tree.json').read_text()): if not info['path'].endswith('.parquet'): continue path = raw / 'ami_hf' / Path(info['path']).name digest = sha256_file(path) if digest != info['lfs']['oid']: raise ValueError(f'Mirror changed: {path}') file = pq.ParquetFile(path) row_index = 0 for batch in file.iter_batches(batch_size=128): for sample in batch.to_pylist(): key = sample['meeting_id'], sample['speaker_id'], sample['microphone_id'] candidates = groups.get(key, []) begin, end = mirror_bounds(sample) fits = [r for r in candidates if r['asset_id'] not in accepted and begin <= round(r['start_seconds'] * 16000) and round(r['end_seconds'] * 16000) <= end] if fits: try: audio = decode_mirror_wav(sample['audio']['bytes']) except ValueError as exc: for row in fits: rejected[row['asset_id']] = 'unsupported_source_audio: ' + str(exc) row_index += 1 continue if abs(len(audio) - (end - begin)) > 1: raise ValueError('Mirror audio length disagrees with time range') for row in fits: first, last = round(row['start_seconds'] * 16000), round(row['end_seconds'] * 16000) if last - begin > len(audio): continue clip = directory / (row['asset_id'].replace(':', '_') + '.wav') pcm = audio[first - begin:last - begin].tobytes() with wave.open(str(clip), 'wb') as wav: wav.setnchannels(1); wav.setsampwidth(2); wav.setframerate(16000); wav.writeframes(pcm) record = {**row, 'raw_path': str(clip.resolve()), 'raw_sha256': sha256_file(clip), 'raw_storage_policy': 'lossless_contained_sentence_from_verified_ihm_mirror', 'mirror_url': f"{BASE}/resolve/{REVISION}/{info['path']}", 'mirror_revision': REVISION, 'mirror_path': str(path.resolve()), 'mirror_sha256': digest, 'mirror_row_index': row_index, 'mirror_audio_id': sample['audio_id'], 'mirror_text': sample['text'], 'mirror_begin_sample_16k': begin, 'mirror_end_sample_16k': end, 'original_start_sample_16k': first, 'original_end_sample_16k': last, 'downloaded_pcm_sha256': hashlib.sha256(pcm).hexdigest()} write_json_atomic(clip.with_suffix('.download.json'), record) accepted[row['asset_id']] = record row_index += 1 print(f'{path.name}: {len(accepted)} contained sentences so far', flush=True) records = [accepted[key] for key in sorted(accepted)] write_jsonl_atomic(raw / 'ami_materialized.jsonl', records) write_jsonl_atomic(raw / 'ami_mirror_unavailable.jsonl', [ {'asset_id': r['asset_id'], 'reason': rejected.get(r['asset_id'], 'no_single_mirror_segment_contains_full_annotated_sentence')} for r in planned if r['asset_id'] not in accepted]) result = {'planned_clips': len(planned), 'materialized_clips': len(records), 'speakers': len({r['speaker_id'] for r in records}), 'splits': dict(Counter(r['split'] for r in records)), 'hours': sum(r['end_seconds'] - r['start_seconds'] for r in records) / 3600, 'mirror_revision': REVISION, 'no_incomplete_sentence_or_silence_padding': True} write_json_atomic(raw / 'ami_mirror_materialization.json', result) print(json.dumps(result, indent=2), flush=True) if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--raw-root', required=True, type=Path) parser.add_argument('--mode', choices=('download', 'materialize'), required=True) parser.add_argument('--workers', type=int, default=6) args = parser.parse_args() if args.mode == 'download': download(args.raw_root.resolve(), args.workers) else: materialize(args.raw_root.resolve())