code/acting_challenge_eval.py

#!/usr/bin/env python3
"""Generate and score the versioned HVSAC challenge matrix for this Talker."""
from __future__ import annotations

import argparse
import hashlib
import io
import json
import math
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tarfile
import time

import numpy as np

SC = Path('/e/scratch/reformo/schuhmann1_moss')
HERE = Path(__file__).resolve().parent
V3 = Path('/e/home/jusers/schuhmann1/jupiter/m2_600m_tts_training/cascade_v3_timed')
M2 = SC / 'code/m2_20k_score'
SMALL = SC / 'code/small_tts'
SCORE = SC / 'out/m2_20k_score'
NB = Path('/e/data1/datasets/playground/mmlaion/schuhmann1/dramabox')
REFERENCE_BANK = SCORE / 'eval/timed_prompts_v3.json'
BENCH = Path('/e/home/jusers/schuhmann1/jupiter/m2_600m_tts_training/acting_challenge_eval_v1')
REFERENCE_BANK_MANIFEST = BENCH / 'reference_bank.jsonl'


def sha(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open('rb') as stream:
        for block in iter(lambda: stream.read(8 << 20), b''):
            digest.update(block)
    return digest.hexdigest()


def encode_mp3_mono(waveform: np.ndarray, sample_rate: int) -> bytes:
    """Encode one float waveform as 128 kb/s mono MP3 using the local LAME wheel."""
    import lameenc
    samples = np.asarray(waveform, dtype=np.float32).reshape(-1)
    assert len(samples) and np.isfinite(samples).all()
    pcm = (np.clip(samples, -1.0, 1.0) * 32767.0).astype('<i2').tobytes()
    encoder = lameenc.Encoder()
    encoder.set_bit_rate(128)
    encoder.set_in_sample_rate(sample_rate)
    encoder.set_channels(1)
    encoder.set_quality(2)
    encoded = bytes(encoder.encode(pcm) + encoder.flush())
    assert encoded and (encoded.startswith(b'ID3') or encoded[:2] in (b'\xff\xfb', b'\xff\xf3'))
    return encoded


def load_checkpoint(model, checkpoint: Path, metadata: dict) -> str:
    import torch
    export = checkpoint / 'model_bf16.pt'
    if export.exists():
        assert metadata.get('export_sha256') in (None, sha(export))
        state = torch.load(export, map_location='cpu', weights_only=True)
        source = 'model_bf16.pt'
    else:
        state_path = checkpoint / 'training_state.pt'
        assert state_path.stat().st_size == metadata['state_bytes']
        assert sha(state_path) == metadata['state_sha256']
        full = torch.load(state_path, map_location='cpu', weights_only=False)
        state = {key[5:]: value for key, value in full['model'].items()
                 if key.startswith('base.')}
        assert state and len(state) == len(full['model'])
        source = 'training_state.pt:model'
        del full
    missing, unexpected = model.load_state_dict(state, strict=True)
    assert not missing and not unexpected
    model.tie_weights()
    del state
    return source


def modes_for(challenge: dict, surface: int, challenge_index: int) -> tuple[str, ...]:
    if surface == 1:
        return ('instruction', 'reference')  # paired no-reference/reference C01 add-on
    if surface == 11:
        return ('reference',)
    if surface in (4, 7):
        return ('instruction',)
    if surface == 10:
        return ('reference',) if str(challenge.get('gemma_mode', '')).startswith('REF_') else ('instruction',)
    if surface in (2, 3, 5, 6, 8, 9):
        return ('instruction', 'reference')[challenge_index % 2:challenge_index % 2 + 1]
    raise ValueError(surface)


def stage_for(step: int) -> str:
    bounds = (('S1', 16265), ('S2', 26284), ('S3', 32337),
              ('S4', 37182), ('S5', 39661), ('S6', 40935),
              ('S7', 41466), ('S8', 41739), ('S9', 41878), ('S10', 41936))
    return next(name for name, end in bounds if step <= end)


def write_reference_archive(codec, bank_rows: list[dict], sample_rate: int,
                            output_dir: Path, torch, decode_fn) -> tuple[Path, Path]:
    """Decode licensed codec references directly into one indexed MP3 TAR."""
    output_dir.mkdir(parents=True, exist_ok=True)
    archive_path = output_dir / 'reference_audio.tar'
    index_path = output_dir / 'reference_audio.index.json'
    bank_digest = sha(REFERENCE_BANK_MANIFEST)
    if archive_path.exists() and index_path.exists():
        existing = json.loads(index_path.read_text())
        assert existing.get('reference_bank_sha256') == bank_digest
        assert existing.get('reference_audio_tar_sha256') == sha(archive_path)
        return archive_path, index_path

    by_id = {str(row['reference_id']): row for row in bank_rows}
    tmp_path = archive_path.with_suffix('.tar.tmp')
    index = {'schema': 'codec-reconstructed-reference-audio-v1',
        'reference_bank': 'reference_bank.jsonl', 'reference_bank_sha256': bank_digest,
        'reference_audio_tar': 'reference_audio.tar', 'references': {}}
    with tarfile.open(tmp_path, 'w') as archive:
        for reference_id, ref in sorted(by_id.items()):
            codes = torch.as_tensor(ref['codec_codes'], dtype=torch.long, device=codec.audio_tokenizer.device)
            waveform, valid = decode_fn(codec, codes, sample_rate)
            assert valid and np.isfinite(waveform).all() and len(waveform)
            encoded = encode_mp3_mono(waveform, sample_rate)
            member = f'{reference_id}.mp3'
            info = tarfile.TarInfo(member)
            info.size, info.mtime = len(encoded), 0
            archive.addfile(info, io.BytesIO(encoded))
            index['references'][reference_id] = {
                'tar': 'reference_audio.tar', 'member': member,
                'codec_reconstructed': True, 'license': ref.get('license'),
                'license_evidence': ref.get('license_evidence'),
                'source_dataset': ref.get('source_dataset'), 'uid': ref.get('uid'),
                'language': ref.get('language'), 'codec_sha256': ref.get('codec_sha256')}
    tmp_path.replace(archive_path)
    index['reference_audio_tar_sha256'] = sha(archive_path)
    tmp_index = index_path.with_suffix('.json.tmp')
    tmp_index.write_text(json.dumps(index, indent=2, sort_keys=True) + '\n')
    tmp_index.replace(index_path)
    return archive_path, index_path


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument('--checkpoint', required=True)
    ap.add_argument('--manifest', default=str(BENCH / 'canary_challenges.jsonl'))
    ap.add_argument('--reference-bank', default=str(REFERENCE_BANK_MANIFEST))
    ap.add_argument('--duration-controls', help='frozen 72-row C01 duration-response manifest')
    ap.add_argument('--output-root', default=str(SC / 'out/m2_600m_ladder/acting_challenge_eval_v1'))
    ap.add_argument('--limit', type=int, help='deterministic prefix for canary')
    ap.add_argument('--dry-run', action='store_true', help='validate manifest/mode/reference inputs only')
    args = ap.parse_args()

    checkpoint = Path(args.checkpoint).resolve()
    metadata = json.loads((checkpoint / 'COMPLETE.json').read_text())
    assert int(metadata['step']) > 0 and metadata['world'] == 128
    assert (checkpoint / 'training_state.pt').stat().st_size == metadata['state_bytes']
    manifest_path = Path(args.manifest).resolve()
    challenges = [json.loads(line) for line in manifest_path.read_text().splitlines() if line.strip()]
    assert challenges and len({r['challenge_id'] for r in challenges}) == len(challenges)
    assert all(list(r['prompts']) == [f'C{i:02d}' for i in range(1, 12)] for r in challenges)
    assert all(len(r['seeds']) == 3 for r in challenges)
    assert all(set(str(seed) for seed in r['seeds']) <= set(r.get('prompts_by_seed', {}))
               for r in challenges), 'seed-specific rendered prompts missing'
    assert all(set(r['prompts']) == set(r['prompts_by_seed'][str(seed)])
               for r in challenges for seed in r['seeds'])
    if args.limit is not None:
        assert 1 <= args.limit <= len(challenges)
        challenges = challenges[:args.limit]
    challenge_index = {row['challenge_id']: i for i, row in enumerate(challenges)}
    duration_controls = []
    if args.duration_controls:
        controls_path = Path(args.duration_controls).resolve()
        control_meta_path = controls_path.with_suffix('.meta.json')
        assert control_meta_path.is_file(), f'missing frozen control metadata {control_meta_path}'
        control_meta = json.loads(control_meta_path.read_text())
        assert control_meta['challenge_manifest_sha256'] == sha(manifest_path), \
            'duration controls were built from a different benchmark manifest'
        all_controls = [json.loads(line) for line in controls_path.read_text().splitlines() if line.strip()]
        assert len(all_controls) == 72 and len({r['task_id'] for r in all_controls}) == 72
        duration_controls = [row for row in all_controls if row['challenge_id'] in challenge_index]
        for control in duration_controls:
            ci = challenge_index[control['challenge_id']]
            challenge = challenges[ci]
            assert control['mode'] in ('instruction', 'reference')
            assert control['surface_base'] == 'C01'
            assert control['canonical_sentences'] == challenge['canonical_sentences']
            assert control['reference_id'] == challenge['reference_policy']['C11']['reference_id']
            assert len(control['duration_targets_s']) == len(challenge['canonical_sentences'])

    bank_path = Path(args.reference_bank).resolve()
    bank_rows = [json.loads(line) for line in bank_path.read_text().splitlines() if line.strip()]
    bank = {str(row.get('reference_id', row.get('ref_uid', row.get('uid')))): row
            for row in bank_rows}
    assert bank, 'pinned reference bank is empty'
    bank_digest = sha(bank_path)

    task_counts = {'instruction': 0, 'reference': 0}
    for ci, challenge in enumerate(challenges):
        policy = challenge.get('reference_policy', {}).get('C11', {})
        pinned_id = policy.get('reference_id') or challenge.get('reference_id')
        assert pinned_id and pinned_id in bank, f"{challenge['challenge_id']} lacks pinned reference asset"
        reference = bank[pinned_id]
        codes = reference.get('codec_codes', reference.get('codes', reference.get('ref_codes')))
        codec_meta = reference.get('codec_contract', reference.get('codec_metadata', reference.get('codec')))
        code_array = np.asarray(codes)
        assert code_array.ndim == 2 and code_array.shape[1] == 12 and len(code_array) > 0
        assert codec_meta, 'reference code/codec contract missing'
        pinned_uid = str(policy.get('reference_uid', reference.get('uid',
                          reference.get('reference_uid', reference.get('ref_uid')))))
        actual_uid = str(reference.get('uid', reference.get('reference_uid', reference.get('ref_uid'))))
        assert pinned_uid == actual_uid
        for surface in range(1, 12):
            for mode in modes_for(challenge, surface, ci):
                task_counts[mode] += len(challenge['seeds'])
    for control in duration_controls:
        task_counts[control['mode']] += 1
    if args.dry_run:
        print(json.dumps({'status': 'PASS', 'challenges': len(challenges),
            'tasks': sum(task_counts.values()), 'tasks_by_mode': task_counts,
            'challenge_sha256': sha(manifest_path), 'reference_bank_sha256': bank_digest,
            'pinned_reference_count': len({
                row.get('reference_policy', {}).get('C11', {}).get('reference_id')
                for row in challenges}),
            'duration_control_rows': len(duration_controls),
            'duration_control_ids': sorted({r['challenge_id'] for r in duration_controls}),
            'selection_role': 'acting benchmark only'}, indent=2))
        return

    import torch
    import torchaudio
    import pyarrow as pa
    import pyarrow.parquet as pq
    import soundfile as sf
    from transformers import AutoTokenizer

    for candidate in (str(HERE), str(V3), str(M2), str(SMALL)):
        while candidate in sys.path:
            sys.path.remove(candidate)
    sys.path[:0] = [str(HERE), str(V3), str(M2), str(SMALL)]
    sys.path[:0] = [str(SC / 'code'), str(NB / 'train2/code/grpo'), str(NB / 'train'),
                    str(SC / 'fastpkgs_sb'), str(NB)]
    from packing import ScorePacker, generated_audio
    import moss_small
    from eval_gen import load_codec_proc, decode
    from large_talker import build_fresh

    rank = int(os.environ.get('SLURM_PROCID', '0'))
    world = int(os.environ.get('SLURM_NTASKS', '1'))
    local = int(os.environ.get('SLURM_LOCALID', '0'))
    assert world in (1, 4)
    device_index = 0 if torch.cuda.device_count() == 1 else local
    torch.cuda.set_device(device_index)
    device = torch.device('cuda', device_index)
    torch.set_num_threads(4)
    schema = json.loads((SCORE / 'score_schema.json').read_text())
    model, config = build_fresh(schema, log=lambda _: None)
    checkpoint_source = load_checkpoint(model, checkpoint, metadata)
    model = model.to(device, dtype=torch.bfloat16).eval()
    _, _, Processor = moss_small.export_classes()
    processor = Processor(tokenizer=AutoTokenizer.from_pretrained(
        moss_small.SFT3, trust_remote_code=True, local_files_only=True),
        audio_tokenizer=None, model_config=config)
    packer = ScorePacker(processor, config, schema)
    codec = load_codec_proc(device)
    sample_rate = int(codec.model_config.sampling_rate)
    reference_archive = reference_index = None
    if rank == 0:
        reference_archive, reference_index = write_reference_archive(
            codec, bank_rows, sample_rate, Path(args.output_root), torch, decode)

    tasks = [(ci, surface, seed, mode, None) for ci, challenge in enumerate(challenges)
             for surface in range(1, 12) for mode in modes_for(challenge, surface, ci)
             for seed in challenge['seeds']]
    tasks.extend((challenge_index[control['challenge_id']], 1, int(control['seed']),
                  control['mode'], control) for control in duration_controls)
    mine = [task for position, task in enumerate(tasks) if position % world == rank]
    generated_rows, waves16 = [], []
    started = time.monotonic()
    for ci, surface, seed, mode, control in mine:
        challenge = challenges[ci]
        key = f'C{surface:02d}'
        output_surface = control['surface'] if control else key
        policy = challenge.get('reference_policy', {}).get('C11', {})
        pinned_reference_id = policy.get('reference_id') or challenge.get('reference_id')
        assert pinned_reference_id, f"{challenge['challenge_id']} has no pinned C11 reference ID"
        assert pinned_reference_id in bank, f'pinned reference {pinned_reference_id} absent from bank'
        reference = bank[pinned_reference_id]
        reference_uid = str(reference.get('uid', reference.get('reference_uid', reference.get('ref_uid'))))
        reference_codes = reference.get('codec_codes', reference.get('codes', reference.get('ref_codes')))
        assert reference_codes, f'pinned reference {pinned_reference_id} has no codec codes'
        reference_language = str(reference.get('language', reference.get('lang', ''))).lower()
        expected_language = 'de' if str(challenge['language']).lower().startswith('de') else 'en'
        assert not reference_language or reference_language.startswith(expected_language)
        pinned_uid = str(policy.get('reference_uid', reference_uid))
        assert pinned_uid == reference_uid
        assert reference_uid != challenge['challenge_id']
        requested_sentence_durations = (control['duration_targets_s'] if control
                                        else challenge['duration_targets_s'])
        target_seconds = float(sum(requested_sentence_durations))
        prompt = (control['prompt'] if control else challenge['prompts_by_seed'][str(seed)][key])
        meta = {'uid': challenge['challenge_id'] + '_' + output_surface,
                'lang': challenge['language'], 'text': challenge['canonical_script'],
                'prompt': prompt,
                'frames': max(2, int(math.ceil((target_seconds + 5.0) * 12.5)))}
        torch.manual_seed(int(seed))
        torch.cuda.manual_seed_all(int(seed))
        example = packer.pack_mode(meta, [], mode,
                                   reference_codes if mode == 'reference' else None,
                                   generation=True)
        batch = packer.collate([example])
        ids = batch['input_ids'].to(device)
        mask = batch['attention_mask'].to(device)
        conditioning = tuple(value.to(device) for value in batch['score_conditioning'])
        before = time.monotonic()
        with torch.inference_mode(), model.generation_scores(conditioning), \
                torch.autocast('cuda', dtype=torch.bfloat16):
            generated = model.generate(input_ids=ids, attention_mask=mask,
                max_new_frames=int(math.ceil(meta['frames'] * 1.6)) + 60,
                do_sample=True, audio_temperature=1., audio_top_p=.95,
                audio_top_k=50, audio_repetition_penalty=1., use_kv_cache=True)
            codes = generated_audio(generated, config).cpu()
        if len(codes) >= 2:
            waveform, canary = decode(codec, codes, sample_rate)
            waveform = np.asarray(waveform, dtype=np.float32)
            assert canary and np.isfinite(waveform).all()
            wave16 = torchaudio.functional.resample(torch.from_numpy(waveform), sample_rate, 16000).numpy()
        else:
            waveform, wave16, canary = np.zeros(0, np.float32), np.zeros(0, np.float32), None
        generated_rows.append({'challenge_id': challenge['challenge_id'], 'surface': output_surface,
            'mode': mode, 'stage': stage_for(int(metadata['step'])),
            'checkpoint_step': int(metadata['step']), 'checkpoint': str(checkpoint),
            'seed': int(seed), 'language': challenge['language'], 'script': challenge['canonical_script'],
            'prompt': prompt,
            'reference_uid': reference_uid if mode == 'reference' else None,
            'reference_assignment_uid': pinned_reference_id,
            'reference_bank_sha256': bank_digest, 'requested_duration_seconds': target_seconds,
            'requested_sentence_durations': [float(x) for x in requested_sentence_durations],
            'duration_control_factor': control['duration_factor'] if control else None,
            'frames': int(len(codes)), 'duration_seconds': float(len(waveform) / sample_rate),
            'canary_ok': canary, 'generation_seconds': time.monotonic() - before,
            'waveform': waveform, 'checkpoint_source': checkpoint_source})
        waves16.append(wave16)
        if len(generated_rows) % 10 == 0:
            print('ACTING_GENERATED', rank, len(generated_rows), flush=True)
    del model, packer, codec
    torch.cuda.empty_cache()

    import asr_parakeet
    import reward
    from annotation_scorer import SentenceScorer, NAMES as SCORER_NAMES
    asr = asr_parakeet.get(device)
    hypotheses = asr.transcribe(waves16)
    scorer = SentenceScorer(device)
    burst_reward = reward.RewardModel(device=str(device), want_asr=False)
    detections = burst_reward.bursts(waves16)
    assert len(SCORER_NAMES) == 99
    raw_names = ['vn:' + name for name in SCORER_NAMES[:57]] + list(SCORER_NAMES[57:])
    assert sum(name.startswith('emo:') for name in raw_names) == 40
    assert sum(name.startswith('vn:') for name in raw_names) == 57
    raw_rows = []
    for wave in waves16:
        if len(wave) < 400:
            raw_rows.append(None)
            continue
        windows = [wave[start:start + 480000] for start in range(0, len(wave), 480000)]
        padded = [np.pad(value, (0, max(0, 8000 - len(value)))) for value in windows]
        raw = scorer.score(padded)
        durations = np.asarray([len(value) for value in windows], dtype=np.float64)
        raw_rows.append(np.average(raw, axis=0, weights=durations))
    score_rows = []
    for row, hypothesis, raw, located in zip(generated_rows, hypotheses, raw_rows, detections):
        target_challenge = next(item for item in challenges
                                if item['challenge_id'] == row['challenge_id'])
        target_bursts = target_challenge.get('vocal_bursts', [])
        requested = [{'label': b['class'], 'sentence_index': int(b['sentence_index']),
                      'onset_location': b['onset_location'],
                      'target_duration_seconds': float(b['target_duration_s'])}
                     for b in target_bursts]
        if raw is None:
            voicenet, empathy, genuineness, blend = {}, {}, None, None
        else:
            voicenet = {name[3:]: float(raw[i]) for i, name in enumerate(raw_names) if name.startswith('vn:')}
            empathy = {name[4:]: float(raw[i]) for i, name in enumerate(raw_names) if name.startswith('emo:')}
            genuineness = float(raw[-2]); blend = float(raw[-1])
        row_ref = row['script']
        import reward as reward_module
        score_rows.append({**{k: row[k] for k in ('challenge_id','surface','mode','seed','language',
            'stage','checkpoint_step','checkpoint','script','prompt','reference_uid',
            'reference_assignment_uid','reference_bank_sha256','requested_duration_seconds',
            'requested_sentence_durations','duration_control_factor','frames',
            'duration_seconds','canary_ok','generation_seconds','checkpoint_source')},
            'hypothesis': hypothesis, 'parakeet_wer': float(reward_module.wer(row_ref, hypothesis)),
            'duration_absolute_error_seconds': abs(row['duration_seconds'] - row['requested_duration_seconds']),
            'duration_relative_error': (row['duration_seconds'] - row['requested_duration_seconds']) /
                                       max(row['requested_duration_seconds'], 1e-6),
            'duration_metric_scope': 'total_clip_diagnostic_only',
            'sentence_duration_metrics_status': 'pending_forced_alignment',
            'duration_response_controls_status': 'pending_72_control_matrix',
            'canonical_sentence_targets_json': json.dumps([
                {'text': sentence, 'target_duration_seconds': float(target)}
                for sentence, target in zip(target_challenge['canonical_sentences'],
                                            row['requested_sentence_durations'])],
                ensure_ascii=False, sort_keys=True),
            'genuineness_0_6': genuineness, 'vocal_burst_blend_0_10': blend,
            'empathic_insight_voice_plus_40_json': json.dumps(empathy, sort_keys=True),
            'voicenet_json': json.dumps(voicenet, sort_keys=True),
            'requested_bursts_json': json.dumps(requested, sort_keys=True),
            'located_bursts_json': json.dumps(located, sort_keys=True),
            'burst_realisation_f1': float(reward_module.burst_realisation(
                [(burst['class'],
                  sum(target_challenge['duration_targets_s'][:max(0, min(
                      int(burst['sentence_index']), len(target_challenge['duration_targets_s'])))] ) +
                  (0.5 * float(target_challenge['duration_targets_s'][int(burst['sentence_index'])])
                   if burst['onset_location'] == 'during_sentence' and
                   int(burst['sentence_index']) < len(target_challenge['duration_targets_s']) else 0.0),
                  float(burst['target_duration_s'])) for burst in target_bursts], located)),
            'audio_seconds': len(waves16[len(score_rows)]) / 16000.0})

    out = Path(args.output_root) / checkpoint.name / ('canary' if args.limit else 'full')
    out.mkdir(parents=True, exist_ok=True)
    tar_path = out / f'audio.rank{rank:03d}.tar'
    tmp_tar = tar_path.with_suffix('.tar.tmp')
    with tarfile.open(tmp_tar, 'w') as archive:
        for row in generated_rows:
            if not len(row['waveform']):
                continue
            payload = encode_mp3_mono(row['waveform'], sample_rate)
            info = tarfile.TarInfo(f"{row['challenge_id']}__{row['surface']}__{row['mode']}__seed{row['seed']}.mp3")
            info.size, info.mtime = len(payload), 0
            archive.addfile(info, io.BytesIO(payload))
    tmp_tar.replace(tar_path)
    parquet_path = out / f'scores.rank{rank:03d}.parquet'
    tmp_parquet = parquet_path.with_suffix('.parquet.tmp')
    pq.write_table(pa.Table.from_pylist(score_rows), tmp_parquet, compression='zstd')
    tmp_parquet.replace(parquet_path)
    proof = {'status': 'PASS', 'rank': rank, 'world': world,
        'checkpoint': str(checkpoint), 'checkpoint_step': int(metadata['step']),
        'challenge_manifest': str(manifest_path), 'challenge_manifest_sha256': sha(manifest_path),
        'reference_bank': str(bank_path), 'reference_bank_sha256': bank_digest,
        'reference_bank_manifest': str(bank_path),
        'reference_bank_manifest_sha256': bank_digest,
        'rows': len(score_rows), 'audio_tar': str(tar_path), 'audio_tar_sha256': sha(tar_path),
        'scores': str(parquet_path), 'scores_sha256': sha(parquet_path),
        'evaluator_sha256': sha(Path(__file__)), 'elapsed_seconds': time.monotonic() - started,
        'browser_audio_format': 'mp3-128k-mono-lameenc',
        'reference_audio_tar': str(reference_archive) if reference_archive else None,
        'reference_audio_tar_sha256': sha(reference_archive) if reference_archive else None,
        'reference_audio_index': str(reference_index) if reference_index else None,
        'reference_audio_index_sha256': sha(reference_index) if reference_index else None,
        'evaluation_completeness': 'INCOMPLETE_PENDING_SENTENCE_ALIGNMENT_AND_72_DURATION_RESPONSE_CONTROLS',
        'publication_ready': False,
        'selection_role': 'acting benchmark only; never checkpoint selection'}
    path = out / f'rank{rank:03d}.PASS.json'
    tmp = path.with_suffix('.json.tmp'); tmp.write_text(json.dumps(proof, indent=2) + '\n'); tmp.replace(path)
    print('ACTING_RANK_PASS', rank, len(score_rows), flush=True)


if __name__ == '__main__':
    main()