Spaces:
Paused
Paused
File size: 8,979 Bytes
24ef38b 602f9e5 da9de03 24ef38b da9de03 24ef38b a416f24 24ef38b 602f9e5 24ef38b b5e6cd5 24ef38b a416f24 24ef38b a416f24 24ef38b a416f24 602f9e5 24ef38b b5e6cd5 da9de03 24ef38b da9de03 b5e6cd5 24ef38b 602f9e5 24ef38b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 | # Keyframe sampling for real video understanding (NINA-style): pull a handful of
# evenly-spaced frames from a video and return them as base64 JPEG data URLs, ready
# to hand to the vision model alongside any speech transcript. Uses PyAV (pulled in
# transitively by faster-whisper) so there's no extra dependency. Best-effort: returns
# [] on any failure rather than raising.
import base64
import io
from config import VIDEO_FRAME_MAX_PX, VIDEO_MAX_FRAMES
def _signature(image) -> tuple:
"""A coarse greyscale thumbnail, used only to spot frames that are the SAME picture.
A video whose sampled points all land on one frame - a static screen recording, or
any clip whose keyframes are sparse - used to send that frame several times over at
full token cost. This catches that.
The comparison is exact on purpose. A fuzzy threshold was tried and measured against
a 60-second recording with six distinct sections: at every threshold loose enough to
be useful it also dropped real section changes, because text on a dark slide averages
away once the picture is small. Losing content from the Note to save tokens is the
wrong trade, so only identical signatures are treated as duplicates.
"""
return tuple(image.convert("L").resize((32, 32)).getdata())
def _shrink(image):
"""Downscale to the size the vision model gets anyway. Done at pick time so a
frame held in memory costs a few hundred kilobytes, not a full 4K bitmap."""
w, h = image.size
scale = VIDEO_FRAME_MAX_PX / max(w, h)
if scale < 1:
image = image.resize((max(1, int(w * scale)), max(1, int(h * scale))))
return image
class Reservoir:
"""At most `cap` evenly thinned items from a stream of unknown length.
The no-duration path used to keep every 15th decoded frame as a full PIL image
until the stream ended: 4500 bitmaps for a 45-minute clip at 25 fps, several
gigabytes on a Space with 16. Here the stride doubles each time the reservoir is
full and every other kept item is dropped, so memory is bounded by `cap` while the
sample stays evenly spread over whatever has been decoded so far."""
def __init__(self, cap: int = 64, step: int = 15):
self.cap, self.step, self.idx = cap, step, 0
self.items: list = []
def offer(self, make_item) -> None:
"""`make_item()` is only called for frames that are kept."""
if self.idx % self.step == 0:
self.items.append(make_item())
if len(self.items) > self.cap:
self.items = self.items[::2]
self.step *= 2
self.idx += 1
def _to_data_url(image) -> str:
image = _shrink(image)
if image.mode != "RGB":
image = image.convert("RGB")
buf = io.BytesIO()
image.save(buf, format="JPEG", quality=85)
return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()
def sample_frames(path: str, max_frames: int = VIDEO_MAX_FRAMES) -> list[tuple[float, str]]:
"""Return [(timestamp_seconds, data_url), …] sampled evenly across the video."""
try:
import av
except Exception as e:
print(f"[LUNA] frames: PyAV unavailable ({e}) - video frames skipped.")
return []
out: list[tuple[float, str]] = []
sigs: set = set()
skipped_same = 0
try:
container = av.open(path)
vstream = container.streams.video[0]
vstream.thread_type = "AUTO"
tb = vstream.time_base
duration = float(vstream.duration * tb) if vstream.duration else None
if not duration and container.duration:
duration = float(container.duration) / 1_000_000.0 # AV_TIME_BASE
if not duration or duration <= 0:
# No duration in the metadata (fragmented MP4, some WebM, a few recorders).
# Rather than fall back to sampling the opening and calling it the video,
# demux the stream WITHOUT decoding to find the last timestamp. Packets
# carry their pts and skipping the decode makes this cheap, and it puts us
# back on the evenly-spaced path below.
# On its OWN handle. Demuxing runs the stream to the end, and a raw
# elementary stream cannot seek back, so probing through the container we
# are about to decode from left it at EOF and the sampler returned nothing.
try:
probe = av.open(path)
try:
pv = probe.streams.video[0]
ptb = pv.time_base
last = 0.0
for packet in probe.demux(pv):
if packet.pts is not None:
last = max(last, float(packet.pts * ptb))
if last > 0:
duration = last
print(f"[LUNA] frames: no duration in metadata, measured "
f"{last:.1f}s by demuxing.")
finally:
probe.close()
except Exception:
pass
if duration and duration > 0:
# Evenly spaced timestamps; seek to each and grab the next decoded frame.
times = [duration * (i + 1) / (max_frames + 1) for i in range(max_frames)]
for t in times:
try:
# Seeking backward lands on the keyframe BEFORE t, not on t. Taking
# the first frame decoded after that seek therefore returns the
# keyframe, and on a video with a sparse GOP - a screen recording,
# a short clip - every seek lands on the SAME keyframe. That sent
# the vision model several copies of one picture, each labelled with
# a different time it had never seen. Decode forward to reach t, and
# report the timestamp of the frame actually used.
container.seek(int(t / tb), stream=vstream, any_frame=False, backward=True)
picked = None
for frame in container.decode(vstream):
picked = frame
if frame.pts is not None and float(frame.pts * tb) >= t:
break
if picked is None:
continue
actual = float(picked.pts * tb) if picked.pts is not None else t
if out and abs(actual - out[-1][0]) < 0.05:
continue # the decoder had nothing new to give at this point
img = picked.to_image()
sig = _signature(img)
if sig in sigs:
skipped_same += 1
continue
sigs.add(sig)
out.append((round(actual, 1), _to_data_url(img)))
except Exception:
continue
else:
# Still no idea how long it is, so spread the sample over the whole stream
# by keeping the LAST max_frames of evenly thinned frames: a fixed "every
# 30th frame, stop at eight" covered the first ten seconds of a 25fps clip
# and reported it as a view of the video. Decoding runs to the end either
# way; only which frames are kept changes.
# A raw elementary stream carries no timestamps at all, so pts is None on
# every frame. Requiring one here returned an empty list and the video was
# reported unreadable; the frame index over the stream's nominal rate is a
# perfectly good estimate, and better than no video at all.
try:
fps = float(vstream.average_rate) if vstream.average_rate else 0.0
except Exception:
fps = 0.0
res = Reservoir()
for frame in container.decode(vstream):
idx = res.idx
if frame.pts is not None:
ts = float(frame.pts * tb)
else:
ts = idx / fps if fps > 0 else 0.0
res.offer(lambda fr=frame, t=ts: (t, _shrink(fr.to_image())))
picked_all = res.items
if picked_all:
stride = max(1, len(picked_all) // max_frames)
for ts, img in picked_all[::stride][:max_frames]:
sig = _signature(img)
if sig in sigs:
skipped_same += 1
continue
sigs.add(sig)
out.append((round(ts, 1), _to_data_url(img)))
container.close()
except Exception as e:
print(f"[LUNA] frames: sampling failed for {path} ({type(e).__name__}: {e}).")
return out
print(f"[LUNA] frames: sampled {len(out)} keyframe(s)"
+ (f", {skipped_same} near-duplicate(s) dropped" if skipped_same else "") + ".")
return out
|