"""Resume causal-prefix QA review; reuse media/features/eight voices. Output is a PENDING manifest. Loader/GT admission is a separate step; machine review is not human ground truth. Old recent-only receipts cannot be reused. """ import argparse from collections import defaultdict from concurrent.futures import ThreadPoolExecutor import hashlib import json import os from pathlib import Path import re import sys import time ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT/'src')) from psi.data.seed.causal_video_qa import POLICY, candidate_times, prefix_deadlines, point from psi.data.seed.video_overlay import decode_video LEGACY_CODE_SHA256 = '80caad15106ee6b0b78138cb3ce78e606395a7011140a5dda7d978f6ad36f246' BLIND = '''These timestamped frames sample the observed video history in chronological order. Use ONLY events already seen by the question-end time. Earlier observed events ARE allowed evidence, even if no longer visible now. Never invent future events. Respect the question's tense: a question about what is happening NOW must not be answered using a different action that has already ended. Do not rely on audio, subtitles or text overlays. If sampled evidence is insufficient, reject. Return JSON: {"grounded":true/false,"requires_text":true/false,"answer":"brief answer"}. ''' VERIFY = '''Check the reference answer against the timestamped observed history. Past observed frames ARE valid memory; future/unseen events are NOT evidence. Respect the question's tense; past evidence cannot justify an incorrect NOW answer. Do not rely on audio/subtitles/text. Reject insufficient evidence. Return JSON: {"supported":true/false,"equivalent":true/false}. ''' def read(path): return json.loads(Path(path).read_text()) def sha(path): h = hashlib.sha256() with Path(path).open('rb') as f: for chunk in iter(lambda: f.read(2**20), b''): h.update(chunk) return h.hexdigest() def atomic(path, data): path = Path(path); path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(path.suffix+'.partial') with tmp.open('w') as f: json.dump(data, f, ensure_ascii=False); f.flush(); os.fsync(f.fileno()) tmp.replace(path) def parse(text): try: match = re.search(r'\{.*\}', text, re.S) result = json.loads(match[0]) if match else {} return result if isinstance(result, dict) else {} except ValueError: return {} def passed(blind, verify): return (blind.get('grounded') is True and blind.get('requires_text') is False and bool(blind.get('answer')) and verify.get('supported') is True and verify.get('equivalent') is True) def shard_assets(selected, index, count): if count < 1 or not 0 <= index < count: raise ValueError('Invalid audit shard') return [v for v in selected if int(hashlib.sha256( v[0]['asset_id'].encode()).hexdigest(), 16) % count == index] def checkpoint_due(checked, planned, interval): return checked == planned or checked % interval == 0 def decode_prefixes(path, duration, times, max_frames, decoder=decode_video): """Decode the union once, then recover the EXACT old samples for each point.""" import numpy as np deadlines = [prefix_deadlines(t, max_frames=max_frames) for t in times] union = np.unique(np.concatenate(deadlines)) frames, pts = decoder(path, union, end_seconds=duration) return [(frames[np.searchsorted(union, ds)], pts[np.searchsorted(union, ds)]) for ds in deadlines] def ordered_prefetch(items, prepare): """One bounded CPU preparation ahead; never touches CUDA in the thread.""" iterator = iter(items) with ThreadPoolExecutor(max_workers=1, thread_name_prefix='video-prefetch') as pool: item = next(iterator, None) future = pool.submit(prepare, item) if item is not None else None while future is not None: result = future.result() item = next(iterator, None) future = pool.submit(prepare, item) if item is not None else None yield result def job_batches(jobs, batch_size, frame_budget): """Group similarly sized independent prompts; retain indices for scatter.""" batch = []; size = 0 for job in sorted(jobs, key=lambda j: len(j['images'])): n = len(job['images']) if batch and (len(batch) >= batch_size or size + n > frame_budget): yield batch batch = []; size = 0 batch.append(job); size += n if batch: yield batch def compatible_contract(old, current): """Only import the explicitly reviewed predecessor or this exact engine.""" if old.get('code_sha256') not in {LEGACY_CODE_SHA256, current['code_sha256']}: return False keys = ('version', 'source_sha256', 'model', 'config_sha256', 'max_points', 'max_frames', 'sampled_prefix_fps', 'reply_offsets', 'blind_prompt', 'verification_prompt', 'same_model_blind_then_verification', 'machine_review_not_human_GT', 'helper_sha256') return all(old.get(k) == current.get(k) for k in keys) def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--manifest', type=Path, required=True) p.add_argument('--output', type=Path, required=True) p.add_argument('--per-split', type=int, default=24, help='0: all assets') p.add_argument('--max-points', type=int, default=8) p.add_argument('--max-frames', type=int, default=64) p.add_argument('--batch-size', type=int, default=8) p.add_argument('--frame-budget', type=int, default=768) p.add_argument('--assets-per-batch', type=int, default=2) p.add_argument('--limit-assets', type=int, default=0, help='0: all; positive: smoke/benchmark only') p.add_argument('--serial', action='store_true', help='Old scheduling for controlled benchmarks') p.add_argument('--reuse-root', type=Path, help='Read-only completed receipts from predecessor shards') p.add_argument('--num-shards', type=int, default=1) p.add_argument('--shard-index', type=int, default=0) p.add_argument('--checkpoint-every', type=int, default=100, help='Receipts/status are saved per asset; flush pending manifest every N assets') p.add_argument('--model', type=Path, default=ROOT.parents[3]/'team_workspace/public_ckpt/Qwen/Qwen3-VL-8B-Instruct') a = p.parse_args() if a.per_split < 0 or a.max_points < 1 or a.max_frames < 2 or a.batch_size < 1: p.error('Invalid size argument') if a.num_shards < 1 or not 0 <= a.shard_index < a.num_shards or a.checkpoint_every < 1: p.error('Invalid shard/checkpoint argument') if a.frame_budget < a.max_frames or a.assets_per_batch < 1 or a.limit_assets < 0: p.error('Invalid batching argument') a.output.mkdir(parents=True, exist_ok=True) import fcntl lock = (a.output/'review.lock').open('a') fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) source = read(a.manifest) admission = a.manifest.parent/'admission/acceptance.json' original = read(admission) if not original.get('passed') or original['manifest_sha256'] != sha(a.manifest): raise ValueError('Source manifest has not passed admission') by_id = defaultdict(list) for row in source['assets']: if row.get('video_path'): by_id[row['asset_id']].append(row) selected = [] for split in ('train','val'): pool = [v for v in by_id.values() if v[0]['split'] == split and candidate_times(float(v[0]['duration_seconds']), max_points=a.max_points)] pool.sort(key=lambda v: hashlib.sha256(v[0]['asset_id'].encode()).hexdigest()) selected += pool[:a.per_split] if a.per_split else pool selected = shard_assets(selected, a.shard_index, a.num_shards) if a.limit_assets: selected = selected[:a.limit_assets] contract = dict(version=POLICY, source_sha256=sha(a.manifest), model=str(a.model), config_sha256=sha(a.model/'config.json'), max_points=a.max_points, max_frames=a.max_frames, sampled_prefix_fps=2., reply_offsets=[0.,.4,.8], blind_prompt=BLIND, verification_prompt=VERIFY, same_model_blind_then_verification=True, machine_review_not_human_GT=True, per_split=a.per_split, code_sha256=sha(__file__), num_shards=a.num_shards, shard_index=a.shard_index, limit_assets=a.limit_assets, serial=a.serial, batch_size=a.batch_size, frame_budget=a.frame_budget, assets_per_batch=a.assets_per_batch, helper_sha256=sha(ROOT/'src/psi/data/seed/causal_video_qa.py')) cp = a.output/'contract.json' if cp.exists() and read(cp) != contract: raise ValueError('Changed audit contract; use a new output directory') atomic(cp, contract) contract_sha = sha(cp) reuse = {} if a.reuse_root: for old_cp in sorted(a.reuse_root.glob('shard*/contract.json')): old = read(old_cp) if not compatible_contract(old, contract): raise ValueError(f'Incompatible reuse contract: {old_cp}') old_sha = sha(old_cp) for receipt in (old_cp.parent/'reviews').glob('*.json'): if receipt.name in reuse: raise ValueError('Duplicate legacy asset receipt') reuse[receipt.name] = (receipt, old_sha) import torch from transformers import AutoProcessor, Qwen3VLForConditionalGeneration from PIL import Image torch.set_num_threads(2) processor = AutoProcessor.from_pretrained(a.model, local_files_only=True) processor.tokenizer.padding_side = 'left' model = Qwen3VLForConditionalGeneration.from_pretrained(a.model, dtype=torch.bfloat16, attn_implementation='flash_attention_2', local_files_only=True).cuda().eval().requires_grad_(False) def generate(images, prompts): texts = [processor.apply_chat_template([dict(role='user', content= [dict(type='image') for _ in ims]+[dict(type='text',text=prompt)])], tokenize=False, add_generation_prompt=True) for ims,prompt in zip(images,prompts)] batch = processor(text=texts, images=images, return_tensors='pt', padding=True, do_resize=False).to('cuda') with torch.inference_mode(): ids = model.generate(**batch, max_new_tokens=120, do_sample=False) return [parse(s) for s in processor.batch_decode(ids[:,batch['input_ids'].shape[1]:], skip_special_tokens=True)] assets = []; started = time.time(); checked = 0; newly_checked = 0; reused = 0 timings = dict(prepare_seconds=0., blind_seconds=0., verify_seconds=0., batches=0) expected_voices = {f'{s}_{n:02d}_{l}' for s in ('female','male') for n in (1,2) for l in ('en','zh')} def validate(variants): row = variants[0] if {r['voice'] for r in variants} != expected_voices or len(variants) != 8: raise ValueError('Source asset must contain exactly eight voices') if any((r['video_sha256'],r['split'],r['duration_seconds']) != (row['video_sha256'],row['split'],row['duration_seconds']) for r in variants): raise ValueError('Inconsistent voice variants') if sha(row['video_path']) != row['video_sha256']: raise ValueError('Video changed') return row def prepare(group): tick = time.perf_counter(); prepared = [] for variants in group: row = validate(variants) name = hashlib.sha256(row['asset_id'].encode()).hexdigest()+'.json' receipt = a.output/'reviews'/name existing = receipt if receipt.exists() else reuse.get(name, (None, None))[0] if existing: review = read(existing) expected_sha = contract_sha if existing == receipt else reuse[name][1] if (review['contract_sha256'] != expected_sha or review['video_sha256'] != row['video_sha256'] or review['asset_id'] != row['asset_id']): raise ValueError('Stale/misidentified causal audit') if existing != receipt: review = {**review, 'origin_receipt': str(existing), 'origin_contract_sha256': expected_sha, 'contract_sha256': contract_sha} prepared.append(dict(variants=variants, row=row, receipt=receipt, review=review)) continue times = candidate_times(row['duration_seconds'], max_points=a.max_points) if a.serial: prefixes = [decode_video(row['video_path'], prefix_deadlines(t, max_frames=a.max_frames), end_seconds=row['duration_seconds']) for t in times] else: prefixes = decode_prefixes(row['video_path'], row['duration_seconds'], times, a.max_frames) prepared.append(dict(variants=variants, row=row, receipt=receipt, review=None, texts=sorted({(r['question'],r['answer']) for r in variants}), points=[(t, [Image.fromarray(f) for f in frames], pts.tolist()) for t,(frames,pts) in zip(times,prefixes)])) return prepared, time.perf_counter()-tick def persist(item, review, is_new): nonlocal checked, newly_checked, reused row = item['row']; receipt = item['receipt'] if not receipt.exists(): atomic(receipt, review) checked += 1; newly_checked += int(is_new); reused += int(not is_new) if review['accepted']: for variant in item['variants']: new = {**variant, 'earliest_answer_video_seconds':min(p['question_end_seconds'] for p in review['causal_qa_points']), 'causal_qa_points':review['causal_qa_points'], 'live_qa_receipt':str(receipt)} new.pop('live_qa_windows',None) assets.append(new) elapsed = time.time()-started atomic(a.output/'status.json', dict(checked=checked, planned=len(selected), accepted_assets=len(assets)//8, newly_checked=newly_checked, reused=reused, updated_at=time.time(), shard_index=a.shard_index, elapsed_seconds=elapsed, timings=timings, eta_seconds=elapsed/newly_checked*(len(selected)-checked) if newly_checked else None, training_ready=False, phase='reviewing', max_cuda_allocated_bytes=torch.cuda.max_memory_allocated())) print('CAUSAL_REVIEW',row['split'],row['asset_id'],review['accepted'], 'points',len(review['causal_qa_points']),flush=True) if checkpoint_due(checked,len(selected),a.checkpoint_every): atomic(a.output/'overlay_assets.pending.json', {**source,'assets':assets,'provenance':{**source['provenance'], 'live_review_contract':str(cp),'video_timeline':'causal observed prefix; exact question times; full-session coverage'}}) groups = [selected[i:i+a.assets_per_batch] for i in range(0,len(selected),a.assets_per_batch)] prepared_groups = map(prepare, groups) if a.serial else ordered_prefetch(groups, prepare) for prepared, prep_seconds in prepared_groups: timings['prepare_seconds'] += prep_seconds jobs = [] for item in prepared: if item['review'] is not None: persist(item, item['review'], False) continue item['checks'] = [] for t,ims,stamps in item['points']: check = dict(question_end_seconds=t, pts=stamps, texts=item['texts'], blind=[None]*len(item['texts']), verification=[None]*len(item['texts'])) item['checks'].append(check) timing = f'Question ends at {t:.3f}s. Image timestamps (seconds): {stamps}.\n' for j,(q,ans) in enumerate(item['texts']): jobs.append(dict(images=ims, timing=timing, question=q, answer=ans, check=check, index=j)) batches = ([jobs[i:i+2] for i in range(0,len(jobs),2)] if a.serial else job_batches(jobs, a.batch_size, a.frame_budget)) for batch_jobs in batches: tick = time.perf_counter() blind = generate([j['images'] for j in batch_jobs], [BLIND+j['timing']+'Question: '+j['question'] for j in batch_jobs]) timings['blind_seconds'] += time.perf_counter()-tick tick = time.perf_counter() verify = generate([j['images'] for j in batch_jobs], [VERIFY+j['timing']+json.dumps( dict(question=j['question'], reference=j['answer'], blind_answer=b.get('answer','')), ensure_ascii=False) for j,b in zip(batch_jobs,blind)]) timings['verify_seconds'] += time.perf_counter()-tick; timings['batches'] += 1 for job,b,c in zip(batch_jobs,blind,verify): job['check']['blind'][job['index']] = b job['check']['verification'][job['index']] = c for item in prepared: if item['review'] is not None: continue admitted = [] for check in item['checks']: check['accepted'] = all(passed(b,c) for b,c in zip(check['blind'],check['verification'])) if check['accepted']: admitted.append(point(check['question_end_seconds'])) review = dict(asset_id=item['row']['asset_id'], video_sha256=item['row']['video_sha256'], contract_sha256=contract_sha, checks=item['checks'], causal_qa_points=admitted, accepted=bool(admitted)) persist(item, review, True) print('DONE: pending manifest only; loader/GT admission required before training',flush=True) if __name__ == '__main__': main()