Download scripts/data/audit_hfd_live_video_windows.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 17.8 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/audit_hfd_live_video_windows.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/audit_hfd_live_video_windows.py
-
curl -L -o audit_hfd_live_video_windows.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/audit_hfd_live_video_windows.py
17.8 kB
| """Resume causal-prefix QA review; reuse media/features/eight voices. | |
| Output is a PENDING manifest. Loader/GT admission is a separate step; machine | |
| review is not human ground truth. Old recent-only receipts cannot be reused. | |
| """ | |
| import argparse | |
| from collections import defaultdict | |
| from concurrent.futures import ThreadPoolExecutor | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import re | |
| import sys | |
| import time | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT/'src')) | |
| from psi.data.seed.causal_video_qa import POLICY, candidate_times, prefix_deadlines, point | |
| from psi.data.seed.video_overlay import decode_video | |
| LEGACY_CODE_SHA256 = '80caad15106ee6b0b78138cb3ce78e606395a7011140a5dda7d978f6ad36f246' | |
| BLIND = '''These timestamped frames sample the observed video history in chronological order. | |
| Use ONLY events already seen by the question-end time. Earlier observed events | |
| ARE allowed evidence, even if no longer visible now. Never invent future events. | |
| Respect the question's tense: a question about what is happening NOW must not | |
| be answered using a different action that has already ended. Do not rely on | |
| audio, subtitles or text overlays. If sampled evidence is insufficient, reject. | |
| Return JSON: {"grounded":true/false,"requires_text":true/false,"answer":"brief answer"}. | |
| ''' | |
| VERIFY = '''Check the reference answer against the timestamped observed history. | |
| Past observed frames ARE valid memory; future/unseen events are NOT evidence. | |
| Respect the question's tense; past evidence cannot justify an incorrect NOW | |
| answer. Do not rely on audio/subtitles/text. Reject insufficient evidence. | |
| Return JSON: {"supported":true/false,"equivalent":true/false}. | |
| ''' | |
| def read(path): | |
| return json.loads(Path(path).read_text()) | |
| def sha(path): | |
| h = hashlib.sha256() | |
| with Path(path).open('rb') as f: | |
| for chunk in iter(lambda: f.read(2**20), b''): | |
| h.update(chunk) | |
| return h.hexdigest() | |
| def atomic(path, data): | |
| path = Path(path); path.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = path.with_suffix(path.suffix+'.partial') | |
| with tmp.open('w') as f: | |
| json.dump(data, f, ensure_ascii=False); f.flush(); os.fsync(f.fileno()) | |
| tmp.replace(path) | |
| def parse(text): | |
| try: | |
| match = re.search(r'\{.*\}', text, re.S) | |
| result = json.loads(match[0]) if match else {} | |
| return result if isinstance(result, dict) else {} | |
| except ValueError: | |
| return {} | |
| def passed(blind, verify): | |
| return (blind.get('grounded') is True and blind.get('requires_text') is False | |
| and bool(blind.get('answer')) and verify.get('supported') is True | |
| and verify.get('equivalent') is True) | |
| def shard_assets(selected, index, count): | |
| if count < 1 or not 0 <= index < count: | |
| raise ValueError('Invalid audit shard') | |
| return [v for v in selected if int(hashlib.sha256( | |
| v[0]['asset_id'].encode()).hexdigest(), 16) % count == index] | |
| def checkpoint_due(checked, planned, interval): | |
| return checked == planned or checked % interval == 0 | |
| def decode_prefixes(path, duration, times, max_frames, decoder=decode_video): | |
| """Decode the union once, then recover the EXACT old samples for each point.""" | |
| import numpy as np | |
| deadlines = [prefix_deadlines(t, max_frames=max_frames) for t in times] | |
| union = np.unique(np.concatenate(deadlines)) | |
| frames, pts = decoder(path, union, end_seconds=duration) | |
| return [(frames[np.searchsorted(union, ds)], pts[np.searchsorted(union, ds)]) | |
| for ds in deadlines] | |
| def ordered_prefetch(items, prepare): | |
| """One bounded CPU preparation ahead; never touches CUDA in the thread.""" | |
| iterator = iter(items) | |
| with ThreadPoolExecutor(max_workers=1, thread_name_prefix='video-prefetch') as pool: | |
| item = next(iterator, None) | |
| future = pool.submit(prepare, item) if item is not None else None | |
| while future is not None: | |
| result = future.result() | |
| item = next(iterator, None) | |
| future = pool.submit(prepare, item) if item is not None else None | |
| yield result | |
| def job_batches(jobs, batch_size, frame_budget): | |
| """Group similarly sized independent prompts; retain indices for scatter.""" | |
| batch = []; size = 0 | |
| for job in sorted(jobs, key=lambda j: len(j['images'])): | |
| n = len(job['images']) | |
| if batch and (len(batch) >= batch_size or size + n > frame_budget): | |
| yield batch | |
| batch = []; size = 0 | |
| batch.append(job); size += n | |
| if batch: | |
| yield batch | |
| def compatible_contract(old, current): | |
| """Only import the explicitly reviewed predecessor or this exact engine.""" | |
| if old.get('code_sha256') not in {LEGACY_CODE_SHA256, current['code_sha256']}: | |
| return False | |
| keys = ('version', 'source_sha256', 'model', 'config_sha256', 'max_points', | |
| 'max_frames', 'sampled_prefix_fps', 'reply_offsets', 'blind_prompt', | |
| 'verification_prompt', 'same_model_blind_then_verification', | |
| 'machine_review_not_human_GT', 'helper_sha256') | |
| return all(old.get(k) == current.get(k) for k in keys) | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('--manifest', type=Path, required=True) | |
| p.add_argument('--output', type=Path, required=True) | |
| p.add_argument('--per-split', type=int, default=24, help='0: all assets') | |
| p.add_argument('--max-points', type=int, default=8) | |
| p.add_argument('--max-frames', type=int, default=64) | |
| p.add_argument('--batch-size', type=int, default=8) | |
| p.add_argument('--frame-budget', type=int, default=768) | |
| p.add_argument('--assets-per-batch', type=int, default=2) | |
| p.add_argument('--limit-assets', type=int, default=0, help='0: all; positive: smoke/benchmark only') | |
| p.add_argument('--serial', action='store_true', help='Old scheduling for controlled benchmarks') | |
| p.add_argument('--reuse-root', type=Path, help='Read-only completed receipts from predecessor shards') | |
| p.add_argument('--num-shards', type=int, default=1) | |
| p.add_argument('--shard-index', type=int, default=0) | |
| p.add_argument('--checkpoint-every', type=int, default=100, | |
| help='Receipts/status are saved per asset; flush pending manifest every N assets') | |
| p.add_argument('--model', type=Path, default=ROOT.parents[3]/'team_workspace/public_ckpt/Qwen/Qwen3-VL-8B-Instruct') | |
| a = p.parse_args() | |
| if a.per_split < 0 or a.max_points < 1 or a.max_frames < 2 or a.batch_size < 1: | |
| p.error('Invalid size argument') | |
| if a.num_shards < 1 or not 0 <= a.shard_index < a.num_shards or a.checkpoint_every < 1: | |
| p.error('Invalid shard/checkpoint argument') | |
| if a.frame_budget < a.max_frames or a.assets_per_batch < 1 or a.limit_assets < 0: | |
| p.error('Invalid batching argument') | |
| a.output.mkdir(parents=True, exist_ok=True) | |
| import fcntl | |
| lock = (a.output/'review.lock').open('a') | |
| fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| source = read(a.manifest) | |
| admission = a.manifest.parent/'admission/acceptance.json' | |
| original = read(admission) | |
| if not original.get('passed') or original['manifest_sha256'] != sha(a.manifest): | |
| raise ValueError('Source manifest has not passed admission') | |
| by_id = defaultdict(list) | |
| for row in source['assets']: | |
| if row.get('video_path'): | |
| by_id[row['asset_id']].append(row) | |
| selected = [] | |
| for split in ('train','val'): | |
| pool = [v for v in by_id.values() if v[0]['split'] == split | |
| and candidate_times(float(v[0]['duration_seconds']), max_points=a.max_points)] | |
| pool.sort(key=lambda v: hashlib.sha256(v[0]['asset_id'].encode()).hexdigest()) | |
| selected += pool[:a.per_split] if a.per_split else pool | |
| selected = shard_assets(selected, a.shard_index, a.num_shards) | |
| if a.limit_assets: | |
| selected = selected[:a.limit_assets] | |
| contract = dict(version=POLICY, source_sha256=sha(a.manifest), model=str(a.model), | |
| config_sha256=sha(a.model/'config.json'), max_points=a.max_points, | |
| max_frames=a.max_frames, sampled_prefix_fps=2., reply_offsets=[0.,.4,.8], | |
| blind_prompt=BLIND, verification_prompt=VERIFY, | |
| same_model_blind_then_verification=True, machine_review_not_human_GT=True, | |
| per_split=a.per_split, code_sha256=sha(__file__), | |
| num_shards=a.num_shards, shard_index=a.shard_index, | |
| limit_assets=a.limit_assets, serial=a.serial, batch_size=a.batch_size, | |
| frame_budget=a.frame_budget, assets_per_batch=a.assets_per_batch, | |
| helper_sha256=sha(ROOT/'src/psi/data/seed/causal_video_qa.py')) | |
| cp = a.output/'contract.json' | |
| if cp.exists() and read(cp) != contract: | |
| raise ValueError('Changed audit contract; use a new output directory') | |
| atomic(cp, contract) | |
| contract_sha = sha(cp) | |
| reuse = {} | |
| if a.reuse_root: | |
| for old_cp in sorted(a.reuse_root.glob('shard*/contract.json')): | |
| old = read(old_cp) | |
| if not compatible_contract(old, contract): | |
| raise ValueError(f'Incompatible reuse contract: {old_cp}') | |
| old_sha = sha(old_cp) | |
| for receipt in (old_cp.parent/'reviews').glob('*.json'): | |
| if receipt.name in reuse: | |
| raise ValueError('Duplicate legacy asset receipt') | |
| reuse[receipt.name] = (receipt, old_sha) | |
| import torch | |
| from transformers import AutoProcessor, Qwen3VLForConditionalGeneration | |
| from PIL import Image | |
| torch.set_num_threads(2) | |
| processor = AutoProcessor.from_pretrained(a.model, local_files_only=True) | |
| processor.tokenizer.padding_side = 'left' | |
| model = Qwen3VLForConditionalGeneration.from_pretrained(a.model, dtype=torch.bfloat16, | |
| attn_implementation='flash_attention_2', local_files_only=True).cuda().eval().requires_grad_(False) | |
| def generate(images, prompts): | |
| texts = [processor.apply_chat_template([dict(role='user', content= | |
| [dict(type='image') for _ in ims]+[dict(type='text',text=prompt)])], | |
| tokenize=False, add_generation_prompt=True) for ims,prompt in zip(images,prompts)] | |
| batch = processor(text=texts, images=images, return_tensors='pt', padding=True, do_resize=False).to('cuda') | |
| with torch.inference_mode(): | |
| ids = model.generate(**batch, max_new_tokens=120, do_sample=False) | |
| return [parse(s) for s in processor.batch_decode(ids[:,batch['input_ids'].shape[1]:], skip_special_tokens=True)] | |
| assets = []; started = time.time(); checked = 0; newly_checked = 0; reused = 0 | |
| timings = dict(prepare_seconds=0., blind_seconds=0., verify_seconds=0., batches=0) | |
| expected_voices = {f'{s}_{n:02d}_{l}' for s in ('female','male') for n in (1,2) for l in ('en','zh')} | |
| def validate(variants): | |
| row = variants[0] | |
| if {r['voice'] for r in variants} != expected_voices or len(variants) != 8: | |
| raise ValueError('Source asset must contain exactly eight voices') | |
| if any((r['video_sha256'],r['split'],r['duration_seconds']) != | |
| (row['video_sha256'],row['split'],row['duration_seconds']) for r in variants): | |
| raise ValueError('Inconsistent voice variants') | |
| if sha(row['video_path']) != row['video_sha256']: | |
| raise ValueError('Video changed') | |
| return row | |
| def prepare(group): | |
| tick = time.perf_counter(); prepared = [] | |
| for variants in group: | |
| row = validate(variants) | |
| name = hashlib.sha256(row['asset_id'].encode()).hexdigest()+'.json' | |
| receipt = a.output/'reviews'/name | |
| existing = receipt if receipt.exists() else reuse.get(name, (None, None))[0] | |
| if existing: | |
| review = read(existing) | |
| expected_sha = contract_sha if existing == receipt else reuse[name][1] | |
| if (review['contract_sha256'] != expected_sha or | |
| review['video_sha256'] != row['video_sha256'] or | |
| review['asset_id'] != row['asset_id']): | |
| raise ValueError('Stale/misidentified causal audit') | |
| if existing != receipt: | |
| review = {**review, 'origin_receipt': str(existing), | |
| 'origin_contract_sha256': expected_sha, 'contract_sha256': contract_sha} | |
| prepared.append(dict(variants=variants, row=row, receipt=receipt, review=review)) | |
| continue | |
| times = candidate_times(row['duration_seconds'], max_points=a.max_points) | |
| if a.serial: | |
| prefixes = [decode_video(row['video_path'], prefix_deadlines(t, max_frames=a.max_frames), | |
| end_seconds=row['duration_seconds']) for t in times] | |
| else: | |
| prefixes = decode_prefixes(row['video_path'], row['duration_seconds'], times, a.max_frames) | |
| prepared.append(dict(variants=variants, row=row, receipt=receipt, review=None, | |
| texts=sorted({(r['question'],r['answer']) for r in variants}), | |
| points=[(t, [Image.fromarray(f) for f in frames], pts.tolist()) | |
| for t,(frames,pts) in zip(times,prefixes)])) | |
| return prepared, time.perf_counter()-tick | |
| def persist(item, review, is_new): | |
| nonlocal checked, newly_checked, reused | |
| row = item['row']; receipt = item['receipt'] | |
| if not receipt.exists(): | |
| atomic(receipt, review) | |
| checked += 1; newly_checked += int(is_new); reused += int(not is_new) | |
| if review['accepted']: | |
| for variant in item['variants']: | |
| new = {**variant, 'earliest_answer_video_seconds':min(p['question_end_seconds'] for p in review['causal_qa_points']), | |
| 'causal_qa_points':review['causal_qa_points'], 'live_qa_receipt':str(receipt)} | |
| new.pop('live_qa_windows',None) | |
| assets.append(new) | |
| elapsed = time.time()-started | |
| atomic(a.output/'status.json', dict(checked=checked, planned=len(selected), accepted_assets=len(assets)//8, | |
| newly_checked=newly_checked, reused=reused, updated_at=time.time(), shard_index=a.shard_index, | |
| elapsed_seconds=elapsed, timings=timings, | |
| eta_seconds=elapsed/newly_checked*(len(selected)-checked) if newly_checked else None, | |
| training_ready=False, phase='reviewing', max_cuda_allocated_bytes=torch.cuda.max_memory_allocated())) | |
| print('CAUSAL_REVIEW',row['split'],row['asset_id'],review['accepted'], 'points',len(review['causal_qa_points']),flush=True) | |
| if checkpoint_due(checked,len(selected),a.checkpoint_every): | |
| atomic(a.output/'overlay_assets.pending.json', {**source,'assets':assets,'provenance':{**source['provenance'], | |
| 'live_review_contract':str(cp),'video_timeline':'causal observed prefix; exact question times; full-session coverage'}}) | |
| groups = [selected[i:i+a.assets_per_batch] for i in range(0,len(selected),a.assets_per_batch)] | |
| prepared_groups = map(prepare, groups) if a.serial else ordered_prefetch(groups, prepare) | |
| for prepared, prep_seconds in prepared_groups: | |
| timings['prepare_seconds'] += prep_seconds | |
| jobs = [] | |
| for item in prepared: | |
| if item['review'] is not None: | |
| persist(item, item['review'], False) | |
| continue | |
| item['checks'] = [] | |
| for t,ims,stamps in item['points']: | |
| check = dict(question_end_seconds=t, pts=stamps, texts=item['texts'], | |
| blind=[None]*len(item['texts']), verification=[None]*len(item['texts'])) | |
| item['checks'].append(check) | |
| timing = f'Question ends at {t:.3f}s. Image timestamps (seconds): {stamps}.\n' | |
| for j,(q,ans) in enumerate(item['texts']): | |
| jobs.append(dict(images=ims, timing=timing, question=q, answer=ans, check=check, index=j)) | |
| batches = ([jobs[i:i+2] for i in range(0,len(jobs),2)] if a.serial | |
| else job_batches(jobs, a.batch_size, a.frame_budget)) | |
| for batch_jobs in batches: | |
| tick = time.perf_counter() | |
| blind = generate([j['images'] for j in batch_jobs], | |
| [BLIND+j['timing']+'Question: '+j['question'] for j in batch_jobs]) | |
| timings['blind_seconds'] += time.perf_counter()-tick | |
| tick = time.perf_counter() | |
| verify = generate([j['images'] for j in batch_jobs], [VERIFY+j['timing']+json.dumps( | |
| dict(question=j['question'], reference=j['answer'], blind_answer=b.get('answer','')), | |
| ensure_ascii=False) for j,b in zip(batch_jobs,blind)]) | |
| timings['verify_seconds'] += time.perf_counter()-tick; timings['batches'] += 1 | |
| for job,b,c in zip(batch_jobs,blind,verify): | |
| job['check']['blind'][job['index']] = b | |
| job['check']['verification'][job['index']] = c | |
| for item in prepared: | |
| if item['review'] is not None: | |
| continue | |
| admitted = [] | |
| for check in item['checks']: | |
| check['accepted'] = all(passed(b,c) for b,c in zip(check['blind'],check['verification'])) | |
| if check['accepted']: | |
| admitted.append(point(check['question_end_seconds'])) | |
| review = dict(asset_id=item['row']['asset_id'], video_sha256=item['row']['video_sha256'], | |
| contract_sha256=contract_sha, checks=item['checks'], causal_qa_points=admitted, | |
| accepted=bool(admitted)) | |
| persist(item, review, True) | |
| print('DONE: pending manifest only; loader/GT admission required before training',flush=True) | |
| if __name__ == '__main__': | |
| main() | |