File size: 5,002 Bytes
fb0074a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
"""Formal HFD closed-loop inference; never provide command text/GT duration."""
import argparse
import json
import random
import sys
import time
from pathlib import Path
from common import ROOT, WORK, DEFAULT_RUN, sha256, write_json
sys.path[:0]=[str(ROOT),str(ROOT/'src')]
import numpy as np
import torch

def main():
    p=argparse.ArgumentParser()
    p.add_argument('--run',type=Path,default=DEFAULT_RUN)
    p.add_argument('--step',type=int,default=1800000)
    p.add_argument('--cohort',type=Path,default=WORK/'outputs/sessions2048')
    p.add_argument('--output',type=Path,default=WORK/'outputs/session_ckpt_1800000/generated')
    p.add_argument('--shard',type=int,default=0)
    p.add_argument('--shards',type=int,default=1)
    p.add_argument('--limit',type=int,default=0)
    p.add_argument('--unfrozen-smoke',action='store_true',help='Explicitly test partial preparation; never a scored cohort')
    p.add_argument('--selection-manifest',type=Path,help='Immutable preparation snapshot for smoke tests only')
    a=p.parse_args(); a.output.mkdir(parents=True,exist_ok=True)
    manifest_path=a.cohort/('selection_progress.json' if a.unfrozen_smoke else 'manifest.json')
    if a.selection_manifest:
        if not a.unfrozen_smoke:raise ValueError('Selection snapshots are only valid for unscored smoke tests')
        manifest_path=a.selection_manifest
    manifest=json.loads(manifest_path.read_text())
    if a.unfrozen_smoke:
        if not a.limit:raise ValueError('Unfrozen smoke requires an explicit --limit')
        manifest={**manifest['identity'],'rows':manifest['rows']}
    identity=dict(step=a.step,run=str(a.run.resolve()),cohort_sha256=sha256(manifest_path),unfrozen_smoke=a.unfrozen_smoke,
                  config_sha256=sha256(a.run/'run_config.json'),native_speech=manifest.get('schema')=='hfd_bench_session_v3')
    identity_path=a.output/'identity.json'
    if identity_path.exists() and json.loads(identity_path.read_text())!=identity:
        raise ValueError('Refusing mixed checkpoint/cohort outputs')
    if not identity_path.exists():write_json(identity_path,identity)
    selected=manifest['rows'][a.shard::a.shards]
    if a.limit:selected=selected[:a.limit]
    if all((a.output/(r['sample_id']+'.npz')).exists() and (a.output/(r['sample_id']+'.json')).exists() for r in selected):
        print('Already complete',a.shard,flush=True)
        return
    from scripts.infer.unified_overlay import cfg
    from psi.models.interaction_arch.loading import load_swa_checkpoint
    from psi.runtime.swa_rollout import rollout_source
    from psi.runtime.swa_speech import SwaSpeechWorker, schedule_speech
    from concurrent.futures import ThreadPoolExecutor
    torch.set_num_threads(4)
    config=cfg(a.run)
    model=load_swa_checkpoint(a.run,a.step,config,torch.device('cuda:0')).eval()
    speech=None
    for row in selected:
        dest=a.output/(row['sample_id']+'.npz')
        if dest.exists() and dest.with_suffix('.json').exists():continue
        if row.get('source_sha256') and row['source_sha256']!=sha256(a.cohort/row['source_file']):
            raise ValueError('Frozen source content changed')
        started=time.monotonic()
        random.seed(42);np.random.seed(42);torch.manual_seed(42)
        source=torch.load(a.cohort/row['source_file'],map_location='cpu',weights_only=False,mmap=True)
        if identity['native_speech']:
            folder=a.output/row['sample_id'];folder.mkdir(exist_ok=True)
            if speech is None:
                speech=SwaSpeechWorker(model,config.model.model_name_or_path,folder)
            else:
                # Reuse immutable native speech weights, never the previous KV/job.
                speech.output=folder;speech.jobs=[];speech.failures=[]
                speech.pool=ThreadPoolExecutor(max_workers=1,thread_name_prefix='bench-answer')
        try:
            motion,events=rollout_source(model,config,source,speech=speech)
            answers=speech.finish(allow_incomplete=True) if speech else []
        finally:
            if speech:speech.close()
        if len(motion)<row['score_end']:raise ValueError('Incomplete rollout')
        tmp=dest.with_suffix('.tmp.npz')
        np.savez_compressed(tmp,absolute=motion)
        tmp.replace(dest)
        seconds=time.monotonic()-started
        write_json(dest.with_suffix('.json'),dict(source_id=source['source_id'],events=events,
            answers=answers,speech_timeline=schedule_speech(answers),
            speech_failures=speech.failures if speech else [],ground_truth_answer_audio_used=False,
            latency_note='Offline concurrent rollout; wall-clock PCM times are NOT real-time benchmark latency.',
            elapsed_seconds=seconds,peak_gpu_gib=torch.cuda.max_memory_allocated()/2**30,**identity))
        print('GENERATED',row['sample_id'],len(motion),f'{seconds:.2f}s',flush=True)
    write_json(a.output/f'complete_{a.shard}.json',dict(**identity,count=len(selected),limit=a.limit))

if __name__=='__main__':main()