File size: 4,306 Bytes
9fc9873
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
#!/usr/bin/env python3
"""Build a standalone visual .pt, or verify byte-exact tensor parity only."""
import argparse
import hashlib
import json
from pathlib import Path
import sys

ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'src'))
import numpy as np
import torch
from psi.data.seed.native_visual import encode_frames, request_frames
from psi.data.seed.video_overlay import FORMAT, camera_plan, decode_video


def encode_media(model, processor, path, kind, duration=None):
    request = dict(path=str(path), kind=kind, duration=duration, index=0, count=5)
    if kind == 'image':
        return {str(n): encode_frames(model, processor, request_frames({**request, 'count': n})).cpu()
                for n in (3, 4, 5)}
    ticks, groups = camera_plan(float(duration))
    frames, pts = decode_video(path, ticks, end_seconds=float(duration))
    return dict(format=FORMAT, duration_seconds=float(duration), frame_pts=pts.tolist(),
                packets=[encode_frames(model, processor, frames[g]).cpu() for g in groups],
                tail={str(n): encode_frames(model, processor, np.repeat(frames[-1:], n, axis=0)).cpu()
                      for n in (3, 4, 5)})


def tensors(payload):
    if 'packets' in payload:
        return payload['packets'] + [payload['tail'][str(n)] for n in (3, 4, 5)]
    return [payload[str(n)] for n in (3, 4, 5)]


def main():
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--input', type=Path, required=True)
    p.add_argument('--kind', choices=('image', 'video'), required=True)
    p.add_argument('--duration', type=float, help='Required for video; use manifest duration_seconds')
    p.add_argument('--base-model', type=Path, required=True)
    p.add_argument('--output', type=Path)
    p.add_argument('--reference', type=Path, help='Require torch.equal against an existing cache')
    a = p.parse_args()
    if not a.output and not a.reference:
        p.error('Specify --output or --reference')
    if a.output and a.output.exists():
        p.error('Refusing to overwrite existing features')
    if a.kind == 'video' and (a.duration is None or a.duration <= .2):
        p.error('--duration is required for video')
    from transformers import Qwen2_5OmniProcessor, Qwen2_5OmniThinkerForConditionalGeneration
    torch.set_num_threads(2)
    model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained(str(a.base_model),
        dtype=torch.bfloat16, attn_implementation='flash_attention_2', local_files_only=True).cuda().eval().requires_grad_(False)
    processor = Qwen2_5OmniProcessor.from_pretrained(str(a.base_model), local_files_only=True)
    value = encode_media(model, processor, a.input, a.kind, a.duration)
    if a.reference:
        reference = torch.load(a.reference, map_location='cpu', weights_only=False, mmap=True)
        actual, expected = tensors(value), tensors(reference)
        if len(actual) != len(expected) or not all(x.dtype == y.dtype and torch.equal(x, y) for x,y in zip(actual, expected)):
            raise RuntimeError('Native features are NOT exactly equal to the reference; no output published')
        if a.kind == 'video' and value['frame_pts'] != reference['frame_pts']:
            raise RuntimeError('Native frame timestamps differ from reference')
        print(json.dumps(dict(exact_equal=True, packets=len(actual))), flush=True)
    if a.output:
        sha = hashlib.sha256()
        with a.input.open('rb') as f:
            for block in iter(lambda: f.read(4 << 20), b''): sha.update(block)
        value['native_provenance'] = dict(base_model=str(a.base_model.resolve()), media_sha256=sha.hexdigest(),
            kind=a.kind, camera_fps=8, side=224, semantic_stride=.4, vision_packet_batch=1)
        a.output.parent.mkdir(parents=True, exist_ok=True)
        import tempfile
        with tempfile.NamedTemporaryFile(dir=a.output.parent, suffix='.partial.pt', delete=False) as f:
            temporary = Path(f.name)
        try:
            torch.save(value, temporary)
            # Exclusive creation protects an existing output even if two workers race.
            import os
            os.link(temporary, a.output)
        finally:
            temporary.unlink(missing_ok=True)
        print(a.output)


if __name__ == '__main__':
    main()