code-HFD / scripts /data /encode_native_visual.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
4.31 kB
#!/usr/bin/env python3
"""Build a standalone visual .pt, or verify byte-exact tensor parity only."""
import argparse
import hashlib
import json
from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'src'))
import numpy as np
import torch
from psi.data.seed.native_visual import encode_frames, request_frames
from psi.data.seed.video_overlay import FORMAT, camera_plan, decode_video
def encode_media(model, processor, path, kind, duration=None):
request = dict(path=str(path), kind=kind, duration=duration, index=0, count=5)
if kind == 'image':
return {str(n): encode_frames(model, processor, request_frames({**request, 'count': n})).cpu()
for n in (3, 4, 5)}
ticks, groups = camera_plan(float(duration))
frames, pts = decode_video(path, ticks, end_seconds=float(duration))
return dict(format=FORMAT, duration_seconds=float(duration), frame_pts=pts.tolist(),
packets=[encode_frames(model, processor, frames[g]).cpu() for g in groups],
tail={str(n): encode_frames(model, processor, np.repeat(frames[-1:], n, axis=0)).cpu()
for n in (3, 4, 5)})
def tensors(payload):
if 'packets' in payload:
return payload['packets'] + [payload['tail'][str(n)] for n in (3, 4, 5)]
return [payload[str(n)] for n in (3, 4, 5)]
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--input', type=Path, required=True)
p.add_argument('--kind', choices=('image', 'video'), required=True)
p.add_argument('--duration', type=float, help='Required for video; use manifest duration_seconds')
p.add_argument('--base-model', type=Path, required=True)
p.add_argument('--output', type=Path)
p.add_argument('--reference', type=Path, help='Require torch.equal against an existing cache')
a = p.parse_args()
if not a.output and not a.reference:
p.error('Specify --output or --reference')
if a.output and a.output.exists():
p.error('Refusing to overwrite existing features')
if a.kind == 'video' and (a.duration is None or a.duration <= .2):
p.error('--duration is required for video')
from transformers import Qwen2_5OmniProcessor, Qwen2_5OmniThinkerForConditionalGeneration
torch.set_num_threads(2)
model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained(str(a.base_model),
dtype=torch.bfloat16, attn_implementation='flash_attention_2', local_files_only=True).cuda().eval().requires_grad_(False)
processor = Qwen2_5OmniProcessor.from_pretrained(str(a.base_model), local_files_only=True)
value = encode_media(model, processor, a.input, a.kind, a.duration)
if a.reference:
reference = torch.load(a.reference, map_location='cpu', weights_only=False, mmap=True)
actual, expected = tensors(value), tensors(reference)
if len(actual) != len(expected) or not all(x.dtype == y.dtype and torch.equal(x, y) for x,y in zip(actual, expected)):
raise RuntimeError('Native features are NOT exactly equal to the reference; no output published')
if a.kind == 'video' and value['frame_pts'] != reference['frame_pts']:
raise RuntimeError('Native frame timestamps differ from reference')
print(json.dumps(dict(exact_equal=True, packets=len(actual))), flush=True)
if a.output:
sha = hashlib.sha256()
with a.input.open('rb') as f:
for block in iter(lambda: f.read(4 << 20), b''): sha.update(block)
value['native_provenance'] = dict(base_model=str(a.base_model.resolve()), media_sha256=sha.hexdigest(),
kind=a.kind, camera_fps=8, side=224, semantic_stride=.4, vision_packet_batch=1)
a.output.parent.mkdir(parents=True, exist_ok=True)
import tempfile
with tempfile.NamedTemporaryFile(dir=a.output.parent, suffix='.partial.pt', delete=False) as f:
temporary = Path(f.name)
try:
torch.save(value, temporary)
# Exclusive creation protects an existing output even if two workers race.
import os
os.link(temporary, a.output)
finally:
temporary.unlink(missing_ok=True)
print(a.output)
if __name__ == '__main__':
main()