File size: 8,979 Bytes
24ef38b
 
 
 
 
 
 
 
 
 
 
602f9e5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
da9de03
 
 
24ef38b
 
 
 
da9de03
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24ef38b
 
 
 
 
 
 
 
 
 
 
 
a416f24
24ef38b
 
 
602f9e5
 
24ef38b
 
 
 
 
 
 
 
 
b5e6cd5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24ef38b
 
 
 
 
a416f24
 
 
 
 
 
 
24ef38b
a416f24
24ef38b
a416f24
 
 
 
 
 
 
 
602f9e5
 
 
 
 
 
 
24ef38b
 
 
b5e6cd5
 
 
 
 
 
 
 
 
 
 
 
 
da9de03
24ef38b
da9de03
 
 
 
 
 
 
b5e6cd5
 
 
 
 
 
 
 
 
24ef38b
 
 
 
602f9e5
 
24ef38b
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
# Keyframe sampling for real video understanding (NINA-style): pull a handful of
# evenly-spaced frames from a video and return them as base64 JPEG data URLs, ready
# to hand to the vision model alongside any speech transcript. Uses PyAV (pulled in
# transitively by faster-whisper) so there's no extra dependency. Best-effort: returns
# [] on any failure rather than raising.
import base64
import io

from config import VIDEO_FRAME_MAX_PX, VIDEO_MAX_FRAMES


def _signature(image) -> tuple:
    """A coarse greyscale thumbnail, used only to spot frames that are the SAME picture.

    A video whose sampled points all land on one frame - a static screen recording, or
    any clip whose keyframes are sparse - used to send that frame several times over at
    full token cost. This catches that.

    The comparison is exact on purpose. A fuzzy threshold was tried and measured against
    a 60-second recording with six distinct sections: at every threshold loose enough to
    be useful it also dropped real section changes, because text on a dark slide averages
    away once the picture is small. Losing content from the Note to save tokens is the
    wrong trade, so only identical signatures are treated as duplicates.
    """
    return tuple(image.convert("L").resize((32, 32)).getdata())


def _shrink(image):
    """Downscale to the size the vision model gets anyway. Done at pick time so a
    frame held in memory costs a few hundred kilobytes, not a full 4K bitmap."""
    w, h = image.size
    scale = VIDEO_FRAME_MAX_PX / max(w, h)
    if scale < 1:
        image = image.resize((max(1, int(w * scale)), max(1, int(h * scale))))
    return image


class Reservoir:
    """At most `cap` evenly thinned items from a stream of unknown length.

    The no-duration path used to keep every 15th decoded frame as a full PIL image
    until the stream ended: 4500 bitmaps for a 45-minute clip at 25 fps, several
    gigabytes on a Space with 16. Here the stride doubles each time the reservoir is
    full and every other kept item is dropped, so memory is bounded by `cap` while the
    sample stays evenly spread over whatever has been decoded so far."""

    def __init__(self, cap: int = 64, step: int = 15):
        self.cap, self.step, self.idx = cap, step, 0
        self.items: list = []

    def offer(self, make_item) -> None:
        """`make_item()` is only called for frames that are kept."""
        if self.idx % self.step == 0:
            self.items.append(make_item())
            if len(self.items) > self.cap:
                self.items = self.items[::2]
                self.step *= 2
        self.idx += 1


def _to_data_url(image) -> str:
    image = _shrink(image)
    if image.mode != "RGB":
        image = image.convert("RGB")
    buf = io.BytesIO()
    image.save(buf, format="JPEG", quality=85)
    return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()


def sample_frames(path: str, max_frames: int = VIDEO_MAX_FRAMES) -> list[tuple[float, str]]:
    """Return [(timestamp_seconds, data_url), …] sampled evenly across the video."""
    try:
        import av
    except Exception as e:
        print(f"[LUNA] frames: PyAV unavailable ({e}) - video frames skipped.")
        return []

    out: list[tuple[float, str]] = []
    sigs: set = set()
    skipped_same = 0
    try:
        container = av.open(path)
        vstream = container.streams.video[0]
        vstream.thread_type = "AUTO"
        tb = vstream.time_base
        duration = float(vstream.duration * tb) if vstream.duration else None
        if not duration and container.duration:
            duration = float(container.duration) / 1_000_000.0  # AV_TIME_BASE

        if not duration or duration <= 0:
            # No duration in the metadata (fragmented MP4, some WebM, a few recorders).
            # Rather than fall back to sampling the opening and calling it the video,
            # demux the stream WITHOUT decoding to find the last timestamp. Packets
            # carry their pts and skipping the decode makes this cheap, and it puts us
            # back on the evenly-spaced path below.
            # On its OWN handle. Demuxing runs the stream to the end, and a raw
            # elementary stream cannot seek back, so probing through the container we
            # are about to decode from left it at EOF and the sampler returned nothing.
            try:
                probe = av.open(path)
                try:
                    pv = probe.streams.video[0]
                    ptb = pv.time_base
                    last = 0.0
                    for packet in probe.demux(pv):
                        if packet.pts is not None:
                            last = max(last, float(packet.pts * ptb))
                    if last > 0:
                        duration = last
                        print(f"[LUNA] frames: no duration in metadata, measured "
                              f"{last:.1f}s by demuxing.")
                finally:
                    probe.close()
            except Exception:
                pass

        if duration and duration > 0:
            # Evenly spaced timestamps; seek to each and grab the next decoded frame.
            times = [duration * (i + 1) / (max_frames + 1) for i in range(max_frames)]
            for t in times:
                try:
                    # Seeking backward lands on the keyframe BEFORE t, not on t. Taking
                    # the first frame decoded after that seek therefore returns the
                    # keyframe, and on a video with a sparse GOP - a screen recording,
                    # a short clip - every seek lands on the SAME keyframe. That sent
                    # the vision model several copies of one picture, each labelled with
                    # a different time it had never seen. Decode forward to reach t, and
                    # report the timestamp of the frame actually used.
                    container.seek(int(t / tb), stream=vstream, any_frame=False, backward=True)
                    picked = None
                    for frame in container.decode(vstream):
                        picked = frame
                        if frame.pts is not None and float(frame.pts * tb) >= t:
                            break
                    if picked is None:
                        continue
                    actual = float(picked.pts * tb) if picked.pts is not None else t
                    if out and abs(actual - out[-1][0]) < 0.05:
                        continue        # the decoder had nothing new to give at this point
                    img = picked.to_image()
                    sig = _signature(img)
                    if sig in sigs:
                        skipped_same += 1
                        continue
                    sigs.add(sig)
                    out.append((round(actual, 1), _to_data_url(img)))
                except Exception:
                    continue
        else:
            # Still no idea how long it is, so spread the sample over the whole stream
            # by keeping the LAST max_frames of evenly thinned frames: a fixed "every
            # 30th frame, stop at eight" covered the first ten seconds of a 25fps clip
            # and reported it as a view of the video. Decoding runs to the end either
            # way; only which frames are kept changes.
            # A raw elementary stream carries no timestamps at all, so pts is None on
            # every frame. Requiring one here returned an empty list and the video was
            # reported unreadable; the frame index over the stream's nominal rate is a
            # perfectly good estimate, and better than no video at all.
            try:
                fps = float(vstream.average_rate) if vstream.average_rate else 0.0
            except Exception:
                fps = 0.0
            res = Reservoir()
            for frame in container.decode(vstream):
                idx = res.idx
                if frame.pts is not None:
                    ts = float(frame.pts * tb)
                else:
                    ts = idx / fps if fps > 0 else 0.0
                res.offer(lambda fr=frame, t=ts: (t, _shrink(fr.to_image())))
            picked_all = res.items
            if picked_all:
                stride = max(1, len(picked_all) // max_frames)
                for ts, img in picked_all[::stride][:max_frames]:
                    sig = _signature(img)
                    if sig in sigs:
                        skipped_same += 1
                        continue
                    sigs.add(sig)
                    out.append((round(ts, 1), _to_data_url(img)))
        container.close()
    except Exception as e:
        print(f"[LUNA] frames: sampling failed for {path} ({type(e).__name__}: {e}).")
        return out
    print(f"[LUNA] frames: sampled {len(out)} keyframe(s)"
          + (f", {skipped_same} near-duplicate(s) dropped" if skipped_same else "") + ".")
    return out