File size: 5,002 Bytes
fb0074a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | """Formal HFD closed-loop inference; never provide command text/GT duration."""
import argparse
import json
import random
import sys
import time
from pathlib import Path
from common import ROOT, WORK, DEFAULT_RUN, sha256, write_json
sys.path[:0]=[str(ROOT),str(ROOT/'src')]
import numpy as np
import torch
def main():
p=argparse.ArgumentParser()
p.add_argument('--run',type=Path,default=DEFAULT_RUN)
p.add_argument('--step',type=int,default=1800000)
p.add_argument('--cohort',type=Path,default=WORK/'outputs/sessions2048')
p.add_argument('--output',type=Path,default=WORK/'outputs/session_ckpt_1800000/generated')
p.add_argument('--shard',type=int,default=0)
p.add_argument('--shards',type=int,default=1)
p.add_argument('--limit',type=int,default=0)
p.add_argument('--unfrozen-smoke',action='store_true',help='Explicitly test partial preparation; never a scored cohort')
p.add_argument('--selection-manifest',type=Path,help='Immutable preparation snapshot for smoke tests only')
a=p.parse_args(); a.output.mkdir(parents=True,exist_ok=True)
manifest_path=a.cohort/('selection_progress.json' if a.unfrozen_smoke else 'manifest.json')
if a.selection_manifest:
if not a.unfrozen_smoke:raise ValueError('Selection snapshots are only valid for unscored smoke tests')
manifest_path=a.selection_manifest
manifest=json.loads(manifest_path.read_text())
if a.unfrozen_smoke:
if not a.limit:raise ValueError('Unfrozen smoke requires an explicit --limit')
manifest={**manifest['identity'],'rows':manifest['rows']}
identity=dict(step=a.step,run=str(a.run.resolve()),cohort_sha256=sha256(manifest_path),unfrozen_smoke=a.unfrozen_smoke,
config_sha256=sha256(a.run/'run_config.json'),native_speech=manifest.get('schema')=='hfd_bench_session_v3')
identity_path=a.output/'identity.json'
if identity_path.exists() and json.loads(identity_path.read_text())!=identity:
raise ValueError('Refusing mixed checkpoint/cohort outputs')
if not identity_path.exists():write_json(identity_path,identity)
selected=manifest['rows'][a.shard::a.shards]
if a.limit:selected=selected[:a.limit]
if all((a.output/(r['sample_id']+'.npz')).exists() and (a.output/(r['sample_id']+'.json')).exists() for r in selected):
print('Already complete',a.shard,flush=True)
return
from scripts.infer.unified_overlay import cfg
from psi.models.interaction_arch.loading import load_swa_checkpoint
from psi.runtime.swa_rollout import rollout_source
from psi.runtime.swa_speech import SwaSpeechWorker, schedule_speech
from concurrent.futures import ThreadPoolExecutor
torch.set_num_threads(4)
config=cfg(a.run)
model=load_swa_checkpoint(a.run,a.step,config,torch.device('cuda:0')).eval()
speech=None
for row in selected:
dest=a.output/(row['sample_id']+'.npz')
if dest.exists() and dest.with_suffix('.json').exists():continue
if row.get('source_sha256') and row['source_sha256']!=sha256(a.cohort/row['source_file']):
raise ValueError('Frozen source content changed')
started=time.monotonic()
random.seed(42);np.random.seed(42);torch.manual_seed(42)
source=torch.load(a.cohort/row['source_file'],map_location='cpu',weights_only=False,mmap=True)
if identity['native_speech']:
folder=a.output/row['sample_id'];folder.mkdir(exist_ok=True)
if speech is None:
speech=SwaSpeechWorker(model,config.model.model_name_or_path,folder)
else:
# Reuse immutable native speech weights, never the previous KV/job.
speech.output=folder;speech.jobs=[];speech.failures=[]
speech.pool=ThreadPoolExecutor(max_workers=1,thread_name_prefix='bench-answer')
try:
motion,events=rollout_source(model,config,source,speech=speech)
answers=speech.finish(allow_incomplete=True) if speech else []
finally:
if speech:speech.close()
if len(motion)<row['score_end']:raise ValueError('Incomplete rollout')
tmp=dest.with_suffix('.tmp.npz')
np.savez_compressed(tmp,absolute=motion)
tmp.replace(dest)
seconds=time.monotonic()-started
write_json(dest.with_suffix('.json'),dict(source_id=source['source_id'],events=events,
answers=answers,speech_timeline=schedule_speech(answers),
speech_failures=speech.failures if speech else [],ground_truth_answer_audio_used=False,
latency_note='Offline concurrent rollout; wall-clock PCM times are NOT real-time benchmark latency.',
elapsed_seconds=seconds,peak_gpu_gib=torch.cuda.max_memory_allocated()/2**30,**identity))
print('GENERATED',row['sample_id'],len(motion),f'{seconds:.2f}s',flush=True)
write_json(a.output/f'complete_{a.shard}.json',dict(**identity,count=len(selected),limit=a.limit))
if __name__=='__main__':main()
|