# Keyframe sampling for real video understanding (NINA-style): pull a handful of # evenly-spaced frames from a video and return them as base64 JPEG data URLs, ready # to hand to the vision model alongside any speech transcript. Uses PyAV (pulled in # transitively by faster-whisper) so there's no extra dependency. Best-effort: returns # [] on any failure rather than raising. import base64 import io from config import VIDEO_FRAME_MAX_PX, VIDEO_MAX_FRAMES def _signature(image) -> tuple: """A coarse greyscale thumbnail, used only to spot frames that are the SAME picture. A video whose sampled points all land on one frame - a static screen recording, or any clip whose keyframes are sparse - used to send that frame several times over at full token cost. This catches that. The comparison is exact on purpose. A fuzzy threshold was tried and measured against a 60-second recording with six distinct sections: at every threshold loose enough to be useful it also dropped real section changes, because text on a dark slide averages away once the picture is small. Losing content from the Note to save tokens is the wrong trade, so only identical signatures are treated as duplicates. """ return tuple(image.convert("L").resize((32, 32)).getdata()) def _shrink(image): """Downscale to the size the vision model gets anyway. Done at pick time so a frame held in memory costs a few hundred kilobytes, not a full 4K bitmap.""" w, h = image.size scale = VIDEO_FRAME_MAX_PX / max(w, h) if scale < 1: image = image.resize((max(1, int(w * scale)), max(1, int(h * scale)))) return image class Reservoir: """At most `cap` evenly thinned items from a stream of unknown length. The no-duration path used to keep every 15th decoded frame as a full PIL image until the stream ended: 4500 bitmaps for a 45-minute clip at 25 fps, several gigabytes on a Space with 16. Here the stride doubles each time the reservoir is full and every other kept item is dropped, so memory is bounded by `cap` while the sample stays evenly spread over whatever has been decoded so far.""" def __init__(self, cap: int = 64, step: int = 15): self.cap, self.step, self.idx = cap, step, 0 self.items: list = [] def offer(self, make_item) -> None: """`make_item()` is only called for frames that are kept.""" if self.idx % self.step == 0: self.items.append(make_item()) if len(self.items) > self.cap: self.items = self.items[::2] self.step *= 2 self.idx += 1 def _to_data_url(image) -> str: image = _shrink(image) if image.mode != "RGB": image = image.convert("RGB") buf = io.BytesIO() image.save(buf, format="JPEG", quality=85) return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode() def sample_frames(path: str, max_frames: int = VIDEO_MAX_FRAMES) -> list[tuple[float, str]]: """Return [(timestamp_seconds, data_url), …] sampled evenly across the video.""" try: import av except Exception as e: print(f"[LUNA] frames: PyAV unavailable ({e}) - video frames skipped.") return [] out: list[tuple[float, str]] = [] sigs: set = set() skipped_same = 0 try: container = av.open(path) vstream = container.streams.video[0] vstream.thread_type = "AUTO" tb = vstream.time_base duration = float(vstream.duration * tb) if vstream.duration else None if not duration and container.duration: duration = float(container.duration) / 1_000_000.0 # AV_TIME_BASE if not duration or duration <= 0: # No duration in the metadata (fragmented MP4, some WebM, a few recorders). # Rather than fall back to sampling the opening and calling it the video, # demux the stream WITHOUT decoding to find the last timestamp. Packets # carry their pts and skipping the decode makes this cheap, and it puts us # back on the evenly-spaced path below. # On its OWN handle. Demuxing runs the stream to the end, and a raw # elementary stream cannot seek back, so probing through the container we # are about to decode from left it at EOF and the sampler returned nothing. try: probe = av.open(path) try: pv = probe.streams.video[0] ptb = pv.time_base last = 0.0 for packet in probe.demux(pv): if packet.pts is not None: last = max(last, float(packet.pts * ptb)) if last > 0: duration = last print(f"[LUNA] frames: no duration in metadata, measured " f"{last:.1f}s by demuxing.") finally: probe.close() except Exception: pass if duration and duration > 0: # Evenly spaced timestamps; seek to each and grab the next decoded frame. times = [duration * (i + 1) / (max_frames + 1) for i in range(max_frames)] for t in times: try: # Seeking backward lands on the keyframe BEFORE t, not on t. Taking # the first frame decoded after that seek therefore returns the # keyframe, and on a video with a sparse GOP - a screen recording, # a short clip - every seek lands on the SAME keyframe. That sent # the vision model several copies of one picture, each labelled with # a different time it had never seen. Decode forward to reach t, and # report the timestamp of the frame actually used. container.seek(int(t / tb), stream=vstream, any_frame=False, backward=True) picked = None for frame in container.decode(vstream): picked = frame if frame.pts is not None and float(frame.pts * tb) >= t: break if picked is None: continue actual = float(picked.pts * tb) if picked.pts is not None else t if out and abs(actual - out[-1][0]) < 0.05: continue # the decoder had nothing new to give at this point img = picked.to_image() sig = _signature(img) if sig in sigs: skipped_same += 1 continue sigs.add(sig) out.append((round(actual, 1), _to_data_url(img))) except Exception: continue else: # Still no idea how long it is, so spread the sample over the whole stream # by keeping the LAST max_frames of evenly thinned frames: a fixed "every # 30th frame, stop at eight" covered the first ten seconds of a 25fps clip # and reported it as a view of the video. Decoding runs to the end either # way; only which frames are kept changes. # A raw elementary stream carries no timestamps at all, so pts is None on # every frame. Requiring one here returned an empty list and the video was # reported unreadable; the frame index over the stream's nominal rate is a # perfectly good estimate, and better than no video at all. try: fps = float(vstream.average_rate) if vstream.average_rate else 0.0 except Exception: fps = 0.0 res = Reservoir() for frame in container.decode(vstream): idx = res.idx if frame.pts is not None: ts = float(frame.pts * tb) else: ts = idx / fps if fps > 0 else 0.0 res.offer(lambda fr=frame, t=ts: (t, _shrink(fr.to_image()))) picked_all = res.items if picked_all: stride = max(1, len(picked_all) // max_frames) for ts, img in picked_all[::stride][:max_frames]: sig = _signature(img) if sig in sigs: skipped_same += 1 continue sigs.add(sig) out.append((round(ts, 1), _to_data_url(img))) container.close() except Exception as e: print(f"[LUNA] frames: sampling failed for {path} ({type(e).__name__}: {e}).") return out print(f"[LUNA] frames: sampled {len(out)} keyframe(s)" + (f", {skipped_same} near-duplicate(s) dropped" if skipped_same else "") + ".") return out