code-HFD / scripts /data /audit_hfd_live_video_windows.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
17.8 kB
"""Resume causal-prefix QA review; reuse media/features/eight voices.
Output is a PENDING manifest. Loader/GT admission is a separate step; machine
review is not human ground truth. Old recent-only receipts cannot be reused.
"""
import argparse
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import os
from pathlib import Path
import re
import sys
import time
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT/'src'))
from psi.data.seed.causal_video_qa import POLICY, candidate_times, prefix_deadlines, point
from psi.data.seed.video_overlay import decode_video
LEGACY_CODE_SHA256 = '80caad15106ee6b0b78138cb3ce78e606395a7011140a5dda7d978f6ad36f246'
BLIND = '''These timestamped frames sample the observed video history in chronological order.
Use ONLY events already seen by the question-end time. Earlier observed events
ARE allowed evidence, even if no longer visible now. Never invent future events.
Respect the question's tense: a question about what is happening NOW must not
be answered using a different action that has already ended. Do not rely on
audio, subtitles or text overlays. If sampled evidence is insufficient, reject.
Return JSON: {"grounded":true/false,"requires_text":true/false,"answer":"brief answer"}.
'''
VERIFY = '''Check the reference answer against the timestamped observed history.
Past observed frames ARE valid memory; future/unseen events are NOT evidence.
Respect the question's tense; past evidence cannot justify an incorrect NOW
answer. Do not rely on audio/subtitles/text. Reject insufficient evidence.
Return JSON: {"supported":true/false,"equivalent":true/false}.
'''
def read(path):
return json.loads(Path(path).read_text())
def sha(path):
h = hashlib.sha256()
with Path(path).open('rb') as f:
for chunk in iter(lambda: f.read(2**20), b''):
h.update(chunk)
return h.hexdigest()
def atomic(path, data):
path = Path(path); path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix+'.partial')
with tmp.open('w') as f:
json.dump(data, f, ensure_ascii=False); f.flush(); os.fsync(f.fileno())
tmp.replace(path)
def parse(text):
try:
match = re.search(r'\{.*\}', text, re.S)
result = json.loads(match[0]) if match else {}
return result if isinstance(result, dict) else {}
except ValueError:
return {}
def passed(blind, verify):
return (blind.get('grounded') is True and blind.get('requires_text') is False
and bool(blind.get('answer')) and verify.get('supported') is True
and verify.get('equivalent') is True)
def shard_assets(selected, index, count):
if count < 1 or not 0 <= index < count:
raise ValueError('Invalid audit shard')
return [v for v in selected if int(hashlib.sha256(
v[0]['asset_id'].encode()).hexdigest(), 16) % count == index]
def checkpoint_due(checked, planned, interval):
return checked == planned or checked % interval == 0
def decode_prefixes(path, duration, times, max_frames, decoder=decode_video):
"""Decode the union once, then recover the EXACT old samples for each point."""
import numpy as np
deadlines = [prefix_deadlines(t, max_frames=max_frames) for t in times]
union = np.unique(np.concatenate(deadlines))
frames, pts = decoder(path, union, end_seconds=duration)
return [(frames[np.searchsorted(union, ds)], pts[np.searchsorted(union, ds)])
for ds in deadlines]
def ordered_prefetch(items, prepare):
"""One bounded CPU preparation ahead; never touches CUDA in the thread."""
iterator = iter(items)
with ThreadPoolExecutor(max_workers=1, thread_name_prefix='video-prefetch') as pool:
item = next(iterator, None)
future = pool.submit(prepare, item) if item is not None else None
while future is not None:
result = future.result()
item = next(iterator, None)
future = pool.submit(prepare, item) if item is not None else None
yield result
def job_batches(jobs, batch_size, frame_budget):
"""Group similarly sized independent prompts; retain indices for scatter."""
batch = []; size = 0
for job in sorted(jobs, key=lambda j: len(j['images'])):
n = len(job['images'])
if batch and (len(batch) >= batch_size or size + n > frame_budget):
yield batch
batch = []; size = 0
batch.append(job); size += n
if batch:
yield batch
def compatible_contract(old, current):
"""Only import the explicitly reviewed predecessor or this exact engine."""
if old.get('code_sha256') not in {LEGACY_CODE_SHA256, current['code_sha256']}:
return False
keys = ('version', 'source_sha256', 'model', 'config_sha256', 'max_points',
'max_frames', 'sampled_prefix_fps', 'reply_offsets', 'blind_prompt',
'verification_prompt', 'same_model_blind_then_verification',
'machine_review_not_human_GT', 'helper_sha256')
return all(old.get(k) == current.get(k) for k in keys)
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--manifest', type=Path, required=True)
p.add_argument('--output', type=Path, required=True)
p.add_argument('--per-split', type=int, default=24, help='0: all assets')
p.add_argument('--max-points', type=int, default=8)
p.add_argument('--max-frames', type=int, default=64)
p.add_argument('--batch-size', type=int, default=8)
p.add_argument('--frame-budget', type=int, default=768)
p.add_argument('--assets-per-batch', type=int, default=2)
p.add_argument('--limit-assets', type=int, default=0, help='0: all; positive: smoke/benchmark only')
p.add_argument('--serial', action='store_true', help='Old scheduling for controlled benchmarks')
p.add_argument('--reuse-root', type=Path, help='Read-only completed receipts from predecessor shards')
p.add_argument('--num-shards', type=int, default=1)
p.add_argument('--shard-index', type=int, default=0)
p.add_argument('--checkpoint-every', type=int, default=100,
help='Receipts/status are saved per asset; flush pending manifest every N assets')
p.add_argument('--model', type=Path, default=ROOT.parents[3]/'team_workspace/public_ckpt/Qwen/Qwen3-VL-8B-Instruct')
a = p.parse_args()
if a.per_split < 0 or a.max_points < 1 or a.max_frames < 2 or a.batch_size < 1:
p.error('Invalid size argument')
if a.num_shards < 1 or not 0 <= a.shard_index < a.num_shards or a.checkpoint_every < 1:
p.error('Invalid shard/checkpoint argument')
if a.frame_budget < a.max_frames or a.assets_per_batch < 1 or a.limit_assets < 0:
p.error('Invalid batching argument')
a.output.mkdir(parents=True, exist_ok=True)
import fcntl
lock = (a.output/'review.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
source = read(a.manifest)
admission = a.manifest.parent/'admission/acceptance.json'
original = read(admission)
if not original.get('passed') or original['manifest_sha256'] != sha(a.manifest):
raise ValueError('Source manifest has not passed admission')
by_id = defaultdict(list)
for row in source['assets']:
if row.get('video_path'):
by_id[row['asset_id']].append(row)
selected = []
for split in ('train','val'):
pool = [v for v in by_id.values() if v[0]['split'] == split
and candidate_times(float(v[0]['duration_seconds']), max_points=a.max_points)]
pool.sort(key=lambda v: hashlib.sha256(v[0]['asset_id'].encode()).hexdigest())
selected += pool[:a.per_split] if a.per_split else pool
selected = shard_assets(selected, a.shard_index, a.num_shards)
if a.limit_assets:
selected = selected[:a.limit_assets]
contract = dict(version=POLICY, source_sha256=sha(a.manifest), model=str(a.model),
config_sha256=sha(a.model/'config.json'), max_points=a.max_points,
max_frames=a.max_frames, sampled_prefix_fps=2., reply_offsets=[0.,.4,.8],
blind_prompt=BLIND, verification_prompt=VERIFY,
same_model_blind_then_verification=True, machine_review_not_human_GT=True,
per_split=a.per_split, code_sha256=sha(__file__),
num_shards=a.num_shards, shard_index=a.shard_index,
limit_assets=a.limit_assets, serial=a.serial, batch_size=a.batch_size,
frame_budget=a.frame_budget, assets_per_batch=a.assets_per_batch,
helper_sha256=sha(ROOT/'src/psi/data/seed/causal_video_qa.py'))
cp = a.output/'contract.json'
if cp.exists() and read(cp) != contract:
raise ValueError('Changed audit contract; use a new output directory')
atomic(cp, contract)
contract_sha = sha(cp)
reuse = {}
if a.reuse_root:
for old_cp in sorted(a.reuse_root.glob('shard*/contract.json')):
old = read(old_cp)
if not compatible_contract(old, contract):
raise ValueError(f'Incompatible reuse contract: {old_cp}')
old_sha = sha(old_cp)
for receipt in (old_cp.parent/'reviews').glob('*.json'):
if receipt.name in reuse:
raise ValueError('Duplicate legacy asset receipt')
reuse[receipt.name] = (receipt, old_sha)
import torch
from transformers import AutoProcessor, Qwen3VLForConditionalGeneration
from PIL import Image
torch.set_num_threads(2)
processor = AutoProcessor.from_pretrained(a.model, local_files_only=True)
processor.tokenizer.padding_side = 'left'
model = Qwen3VLForConditionalGeneration.from_pretrained(a.model, dtype=torch.bfloat16,
attn_implementation='flash_attention_2', local_files_only=True).cuda().eval().requires_grad_(False)
def generate(images, prompts):
texts = [processor.apply_chat_template([dict(role='user', content=
[dict(type='image') for _ in ims]+[dict(type='text',text=prompt)])],
tokenize=False, add_generation_prompt=True) for ims,prompt in zip(images,prompts)]
batch = processor(text=texts, images=images, return_tensors='pt', padding=True, do_resize=False).to('cuda')
with torch.inference_mode():
ids = model.generate(**batch, max_new_tokens=120, do_sample=False)
return [parse(s) for s in processor.batch_decode(ids[:,batch['input_ids'].shape[1]:], skip_special_tokens=True)]
assets = []; started = time.time(); checked = 0; newly_checked = 0; reused = 0
timings = dict(prepare_seconds=0., blind_seconds=0., verify_seconds=0., batches=0)
expected_voices = {f'{s}_{n:02d}_{l}' for s in ('female','male') for n in (1,2) for l in ('en','zh')}
def validate(variants):
row = variants[0]
if {r['voice'] for r in variants} != expected_voices or len(variants) != 8:
raise ValueError('Source asset must contain exactly eight voices')
if any((r['video_sha256'],r['split'],r['duration_seconds']) !=
(row['video_sha256'],row['split'],row['duration_seconds']) for r in variants):
raise ValueError('Inconsistent voice variants')
if sha(row['video_path']) != row['video_sha256']:
raise ValueError('Video changed')
return row
def prepare(group):
tick = time.perf_counter(); prepared = []
for variants in group:
row = validate(variants)
name = hashlib.sha256(row['asset_id'].encode()).hexdigest()+'.json'
receipt = a.output/'reviews'/name
existing = receipt if receipt.exists() else reuse.get(name, (None, None))[0]
if existing:
review = read(existing)
expected_sha = contract_sha if existing == receipt else reuse[name][1]
if (review['contract_sha256'] != expected_sha or
review['video_sha256'] != row['video_sha256'] or
review['asset_id'] != row['asset_id']):
raise ValueError('Stale/misidentified causal audit')
if existing != receipt:
review = {**review, 'origin_receipt': str(existing),
'origin_contract_sha256': expected_sha, 'contract_sha256': contract_sha}
prepared.append(dict(variants=variants, row=row, receipt=receipt, review=review))
continue
times = candidate_times(row['duration_seconds'], max_points=a.max_points)
if a.serial:
prefixes = [decode_video(row['video_path'], prefix_deadlines(t, max_frames=a.max_frames),
end_seconds=row['duration_seconds']) for t in times]
else:
prefixes = decode_prefixes(row['video_path'], row['duration_seconds'], times, a.max_frames)
prepared.append(dict(variants=variants, row=row, receipt=receipt, review=None,
texts=sorted({(r['question'],r['answer']) for r in variants}),
points=[(t, [Image.fromarray(f) for f in frames], pts.tolist())
for t,(frames,pts) in zip(times,prefixes)]))
return prepared, time.perf_counter()-tick
def persist(item, review, is_new):
nonlocal checked, newly_checked, reused
row = item['row']; receipt = item['receipt']
if not receipt.exists():
atomic(receipt, review)
checked += 1; newly_checked += int(is_new); reused += int(not is_new)
if review['accepted']:
for variant in item['variants']:
new = {**variant, 'earliest_answer_video_seconds':min(p['question_end_seconds'] for p in review['causal_qa_points']),
'causal_qa_points':review['causal_qa_points'], 'live_qa_receipt':str(receipt)}
new.pop('live_qa_windows',None)
assets.append(new)
elapsed = time.time()-started
atomic(a.output/'status.json', dict(checked=checked, planned=len(selected), accepted_assets=len(assets)//8,
newly_checked=newly_checked, reused=reused, updated_at=time.time(), shard_index=a.shard_index,
elapsed_seconds=elapsed, timings=timings,
eta_seconds=elapsed/newly_checked*(len(selected)-checked) if newly_checked else None,
training_ready=False, phase='reviewing', max_cuda_allocated_bytes=torch.cuda.max_memory_allocated()))
print('CAUSAL_REVIEW',row['split'],row['asset_id'],review['accepted'], 'points',len(review['causal_qa_points']),flush=True)
if checkpoint_due(checked,len(selected),a.checkpoint_every):
atomic(a.output/'overlay_assets.pending.json', {**source,'assets':assets,'provenance':{**source['provenance'],
'live_review_contract':str(cp),'video_timeline':'causal observed prefix; exact question times; full-session coverage'}})
groups = [selected[i:i+a.assets_per_batch] for i in range(0,len(selected),a.assets_per_batch)]
prepared_groups = map(prepare, groups) if a.serial else ordered_prefetch(groups, prepare)
for prepared, prep_seconds in prepared_groups:
timings['prepare_seconds'] += prep_seconds
jobs = []
for item in prepared:
if item['review'] is not None:
persist(item, item['review'], False)
continue
item['checks'] = []
for t,ims,stamps in item['points']:
check = dict(question_end_seconds=t, pts=stamps, texts=item['texts'],
blind=[None]*len(item['texts']), verification=[None]*len(item['texts']))
item['checks'].append(check)
timing = f'Question ends at {t:.3f}s. Image timestamps (seconds): {stamps}.\n'
for j,(q,ans) in enumerate(item['texts']):
jobs.append(dict(images=ims, timing=timing, question=q, answer=ans, check=check, index=j))
batches = ([jobs[i:i+2] for i in range(0,len(jobs),2)] if a.serial
else job_batches(jobs, a.batch_size, a.frame_budget))
for batch_jobs in batches:
tick = time.perf_counter()
blind = generate([j['images'] for j in batch_jobs],
[BLIND+j['timing']+'Question: '+j['question'] for j in batch_jobs])
timings['blind_seconds'] += time.perf_counter()-tick
tick = time.perf_counter()
verify = generate([j['images'] for j in batch_jobs], [VERIFY+j['timing']+json.dumps(
dict(question=j['question'], reference=j['answer'], blind_answer=b.get('answer','')),
ensure_ascii=False) for j,b in zip(batch_jobs,blind)])
timings['verify_seconds'] += time.perf_counter()-tick; timings['batches'] += 1
for job,b,c in zip(batch_jobs,blind,verify):
job['check']['blind'][job['index']] = b
job['check']['verification'][job['index']] = c
for item in prepared:
if item['review'] is not None:
continue
admitted = []
for check in item['checks']:
check['accepted'] = all(passed(b,c) for b,c in zip(check['blind'],check['verification']))
if check['accepted']:
admitted.append(point(check['question_end_seconds']))
review = dict(asset_id=item['row']['asset_id'], video_sha256=item['row']['video_sha256'],
contract_sha256=contract_sha, checks=item['checks'], causal_qa_points=admitted,
accepted=bool(admitted))
persist(item, review, True)
print('DONE: pending manifest only; loader/GT admission required before training',flush=True)
if __name__ == '__main__':
main()