Download scripts/data/materialize_hfd_ami_mirror.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 10 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/materialize_hfd_ami_mirror.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/data/materialize_hfd_ami_mirror.py
-
curl -L -o materialize_hfd_ami_mirror.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/data/materialize_hfd_ami_mirror.py
10 kB
| #!/usr/bin/env python3 | |
| """Verified IHM mirror fallback; keep only fully contained annotated sentences.""" | |
| from __future__ import annotations | |
| import argparse | |
| from collections import Counter, defaultdict | |
| from concurrent.futures import ThreadPoolExecutor | |
| import hashlib | |
| import io | |
| import json | |
| from pathlib import Path | |
| import struct | |
| import subprocess | |
| import sys | |
| import wave | |
| import xml.etree.ElementTree as ET | |
| import zipfile | |
| import numpy as np | |
| import pyarrow.parquet as pq | |
| ROOT = Path(__file__).resolve().parents[2] | |
| if str(ROOT) not in sys.path: | |
| sys.path.insert(0, str(ROOT)) | |
| from scripts.data.hfd_third_party_common import read_jsonl, sha256_file, write_json_atomic, write_jsonl_atomic | |
| REVISION = '46f28f2503e2ec48f8867a84eef356c70476beab' | |
| BASE = 'https://huggingface.co/datasets/edinburghcstr/ami' | |
| def decode_mirror_wav(payload): | |
| """The mirror stores original PCM16 values in IEEE float WAV containers.""" | |
| if payload[:4] != b'RIFF' or payload[8:12] != b'WAVE': | |
| raise ValueError('Mirror audio is not a WAV') | |
| pos = 12 | |
| fmt = pcm = None | |
| while pos + 8 <= len(payload): | |
| kind, size = struct.unpack_from('<4sI', payload, pos) | |
| body = payload[pos + 8:pos + 8 + size] | |
| if len(body) != size: | |
| raise ValueError('Truncated mirror WAV') | |
| if kind == b'fmt ': | |
| fmt = struct.unpack_from('<HHIIHH', body) | |
| if kind == b'data': | |
| pcm = body | |
| pos += 8 + size + size % 2 | |
| if fmt == (3, 1, 16000, 64000, 4, 32): | |
| floats = np.frombuffer(pcm, dtype='<f4') | |
| values = floats.astype(np.float64) * 32768 | |
| if not np.isfinite(values).all() or np.max(np.abs(values - np.rint(values)), initial=0) > 1e-4: | |
| raise ValueError('Mirror audio is not losslessly representable as PCM16') | |
| if np.min(values, initial=0) < -32768 or np.max(values, initial=0) > 32767: | |
| raise ValueError('Mirror PCM16 range exceeded') | |
| return np.rint(values).astype('<i2') | |
| if fmt == (1, 1, 16000, 32000, 2, 16): | |
| return np.frombuffer(pcm, dtype='<i2') | |
| raise ValueError(f'Unexpected mirror WAV format: {fmt}') | |
| def mirror_bounds(row): | |
| # Avoid float32 timestamp rounding: IDs encode centiseconds exactly. | |
| first, last = row['audio_id'].rsplit('_', 2)[-2:] | |
| start, end = int(first) * 160, int(last) * 160 | |
| if abs(row['begin_time'] * 16000 - start) > 16 or abs(row['end_time'] * 16000 - end) > 16: | |
| raise ValueError('Mirror timestamps disagree with segment ID') | |
| return start, end | |
| def annotated_spoken_bounds(planned, archive): | |
| """Drop optional context padding, never any annotated spoken sample.""" | |
| cache = {} | |
| with zipfile.ZipFile(archive) as source: | |
| for row in planned: | |
| member = row['words_member'] | |
| if member not in cache: | |
| words = list(ET.fromstring(source.read(member))) | |
| indices = {w.get('{http://nite.sourceforge.net/}id'): i for i, w in enumerate(words)} | |
| cache[member] = words, indices | |
| words, indices = cache[member] | |
| selected = words[indices[row['word_id_first']]:indices[row['word_id_last']] + 1] | |
| spoken = [w for w in selected if w.tag == 'w' and w.get('punc') != 'true'] | |
| first = min(float(w.get('starttime')) for w in spoken) | |
| last = max(float(w.get('endtime')) for w in spoken) | |
| if first < row['start_seconds'] - .0001 or last > row['end_seconds'] + .0001: | |
| raise ValueError('Spoken words lie outside planned sentence') | |
| row['planned_context_start_seconds'] = row['start_seconds'] | |
| row['planned_context_end_seconds'] = row['end_seconds'] | |
| row['start_seconds'], row['end_seconds'] = first, last | |
| row['boundary_policy'] = 'all_annotated_words_no_optional_silence_margin' | |
| return planned | |
| def download(raw, workers): | |
| revision = json.loads((raw / 'ami_hf_revision.json').read_text())['sha'] | |
| if revision != REVISION: | |
| raise ValueError('Unexpected mirror revision; review before changing the pin') | |
| files = json.loads((raw / 'ami_hf_tree.json').read_text()) | |
| directory = raw / 'ami_hf' | |
| directory.mkdir(exist_ok=True) | |
| def fetch(row): | |
| path = directory / Path(row['path']).name | |
| expected = row['lfs']['oid'] | |
| if not path.exists() or path.stat().st_size != row['size'] or sha256_file(path) != expected: | |
| subprocess.run(['curl', '-fsSL', '--retry', '3', '--retry-all-errors', | |
| '--connect-timeout', '20', '--max-time', '900', '-o', str(path), | |
| f"{BASE}/resolve/{REVISION}/{row['path']}?download=true"], check=True) | |
| if path.stat().st_size != row['size'] or sha256_file(path) != expected: | |
| raise ValueError(f'Mirror LFS hash mismatch: {path.name}') | |
| print(f'verified {path.name}', flush=True) | |
| with ThreadPoolExecutor(max_workers=workers) as pool: | |
| list(pool.map(fetch, [r for r in files if r['path'].endswith('.parquet')])) | |
| def materialize(raw): | |
| planned = annotated_spoken_bounds(read_jsonl(raw / 'ami_plan.jsonl'), raw / 'ami_public_manual_1.6.2.zip') | |
| groups = defaultdict(list) | |
| for row in planned: | |
| groups[row['meeting'], row['raw_speaker'], f"H{int(row['channel']):02d}"].append(row) | |
| accepted = {} | |
| rejected = {} | |
| directory = raw / 'ami_mirror_clips' | |
| directory.mkdir(exist_ok=True) | |
| for info in json.loads((raw / 'ami_hf_tree.json').read_text()): | |
| if not info['path'].endswith('.parquet'): | |
| continue | |
| path = raw / 'ami_hf' / Path(info['path']).name | |
| digest = sha256_file(path) | |
| if digest != info['lfs']['oid']: | |
| raise ValueError(f'Mirror changed: {path}') | |
| file = pq.ParquetFile(path) | |
| row_index = 0 | |
| for batch in file.iter_batches(batch_size=128): | |
| for sample in batch.to_pylist(): | |
| key = sample['meeting_id'], sample['speaker_id'], sample['microphone_id'] | |
| candidates = groups.get(key, []) | |
| begin, end = mirror_bounds(sample) | |
| fits = [r for r in candidates if r['asset_id'] not in accepted | |
| and begin <= round(r['start_seconds'] * 16000) | |
| and round(r['end_seconds'] * 16000) <= end] | |
| if fits: | |
| try: | |
| audio = decode_mirror_wav(sample['audio']['bytes']) | |
| except ValueError as exc: | |
| for row in fits: | |
| rejected[row['asset_id']] = 'unsupported_source_audio: ' + str(exc) | |
| row_index += 1 | |
| continue | |
| if abs(len(audio) - (end - begin)) > 1: | |
| raise ValueError('Mirror audio length disagrees with time range') | |
| for row in fits: | |
| first, last = round(row['start_seconds'] * 16000), round(row['end_seconds'] * 16000) | |
| if last - begin > len(audio): | |
| continue | |
| clip = directory / (row['asset_id'].replace(':', '_') + '.wav') | |
| pcm = audio[first - begin:last - begin].tobytes() | |
| with wave.open(str(clip), 'wb') as wav: | |
| wav.setnchannels(1); wav.setsampwidth(2); wav.setframerate(16000); wav.writeframes(pcm) | |
| record = {**row, 'raw_path': str(clip.resolve()), | |
| 'raw_sha256': sha256_file(clip), | |
| 'raw_storage_policy': 'lossless_contained_sentence_from_verified_ihm_mirror', | |
| 'mirror_url': f"{BASE}/resolve/{REVISION}/{info['path']}", | |
| 'mirror_revision': REVISION, 'mirror_path': str(path.resolve()), | |
| 'mirror_sha256': digest, 'mirror_row_index': row_index, | |
| 'mirror_audio_id': sample['audio_id'], 'mirror_text': sample['text'], | |
| 'mirror_begin_sample_16k': begin, 'mirror_end_sample_16k': end, | |
| 'original_start_sample_16k': first, 'original_end_sample_16k': last, | |
| 'downloaded_pcm_sha256': hashlib.sha256(pcm).hexdigest()} | |
| write_json_atomic(clip.with_suffix('.download.json'), record) | |
| accepted[row['asset_id']] = record | |
| row_index += 1 | |
| print(f'{path.name}: {len(accepted)} contained sentences so far', flush=True) | |
| records = [accepted[key] for key in sorted(accepted)] | |
| write_jsonl_atomic(raw / 'ami_materialized.jsonl', records) | |
| write_jsonl_atomic(raw / 'ami_mirror_unavailable.jsonl', [ | |
| {'asset_id': r['asset_id'], 'reason': rejected.get(r['asset_id'], 'no_single_mirror_segment_contains_full_annotated_sentence')} | |
| for r in planned if r['asset_id'] not in accepted]) | |
| result = {'planned_clips': len(planned), 'materialized_clips': len(records), | |
| 'speakers': len({r['speaker_id'] for r in records}), | |
| 'splits': dict(Counter(r['split'] for r in records)), | |
| 'hours': sum(r['end_seconds'] - r['start_seconds'] for r in records) / 3600, | |
| 'mirror_revision': REVISION, 'no_incomplete_sentence_or_silence_padding': True} | |
| write_json_atomic(raw / 'ami_mirror_materialization.json', result) | |
| print(json.dumps(result, indent=2), flush=True) | |
| if __name__ == '__main__': | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument('--raw-root', required=True, type=Path) | |
| parser.add_argument('--mode', choices=('download', 'materialize'), required=True) | |
| parser.add_argument('--workers', type=int, default=6) | |
| args = parser.parse_args() | |
| if args.mode == 'download': | |
| download(args.raw_root.resolve(), args.workers) | |
| else: | |
| materialize(args.raw_root.resolve()) | |