Spaces:
Paused
Paused
Malina
Claude Fable 5.1
Scoped tokens, bounded archives and frames, build deadline, circuit breaker, honest note caps
da9de03 Download frames.py from NextNeural/LUNA: direct link, hf CLI and curl.
- Browser
- Download file 8.98 kB
-
https://huggingface.co/spaces/NextNeural/LUNA/resolve/main/frames.py
- Command line
-
hf download hf://spaces/NextNeural/LUNA/frames.py
-
curl -L -o frames.py https://huggingface.co/spaces/NextNeural/LUNA/resolve/main/frames.py
8.98 kB
| # Keyframe sampling for real video understanding (NINA-style): pull a handful of | |
| # evenly-spaced frames from a video and return them as base64 JPEG data URLs, ready | |
| # to hand to the vision model alongside any speech transcript. Uses PyAV (pulled in | |
| # transitively by faster-whisper) so there's no extra dependency. Best-effort: returns | |
| # [] on any failure rather than raising. | |
| import base64 | |
| import io | |
| from config import VIDEO_FRAME_MAX_PX, VIDEO_MAX_FRAMES | |
| def _signature(image) -> tuple: | |
| """A coarse greyscale thumbnail, used only to spot frames that are the SAME picture. | |
| A video whose sampled points all land on one frame - a static screen recording, or | |
| any clip whose keyframes are sparse - used to send that frame several times over at | |
| full token cost. This catches that. | |
| The comparison is exact on purpose. A fuzzy threshold was tried and measured against | |
| a 60-second recording with six distinct sections: at every threshold loose enough to | |
| be useful it also dropped real section changes, because text on a dark slide averages | |
| away once the picture is small. Losing content from the Note to save tokens is the | |
| wrong trade, so only identical signatures are treated as duplicates. | |
| """ | |
| return tuple(image.convert("L").resize((32, 32)).getdata()) | |
| def _shrink(image): | |
| """Downscale to the size the vision model gets anyway. Done at pick time so a | |
| frame held in memory costs a few hundred kilobytes, not a full 4K bitmap.""" | |
| w, h = image.size | |
| scale = VIDEO_FRAME_MAX_PX / max(w, h) | |
| if scale < 1: | |
| image = image.resize((max(1, int(w * scale)), max(1, int(h * scale)))) | |
| return image | |
| class Reservoir: | |
| """At most `cap` evenly thinned items from a stream of unknown length. | |
| The no-duration path used to keep every 15th decoded frame as a full PIL image | |
| until the stream ended: 4500 bitmaps for a 45-minute clip at 25 fps, several | |
| gigabytes on a Space with 16. Here the stride doubles each time the reservoir is | |
| full and every other kept item is dropped, so memory is bounded by `cap` while the | |
| sample stays evenly spread over whatever has been decoded so far.""" | |
| def __init__(self, cap: int = 64, step: int = 15): | |
| self.cap, self.step, self.idx = cap, step, 0 | |
| self.items: list = [] | |
| def offer(self, make_item) -> None: | |
| """`make_item()` is only called for frames that are kept.""" | |
| if self.idx % self.step == 0: | |
| self.items.append(make_item()) | |
| if len(self.items) > self.cap: | |
| self.items = self.items[::2] | |
| self.step *= 2 | |
| self.idx += 1 | |
| def _to_data_url(image) -> str: | |
| image = _shrink(image) | |
| if image.mode != "RGB": | |
| image = image.convert("RGB") | |
| buf = io.BytesIO() | |
| image.save(buf, format="JPEG", quality=85) | |
| return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode() | |
| def sample_frames(path: str, max_frames: int = VIDEO_MAX_FRAMES) -> list[tuple[float, str]]: | |
| """Return [(timestamp_seconds, data_url), …] sampled evenly across the video.""" | |
| try: | |
| import av | |
| except Exception as e: | |
| print(f"[LUNA] frames: PyAV unavailable ({e}) - video frames skipped.") | |
| return [] | |
| out: list[tuple[float, str]] = [] | |
| sigs: set = set() | |
| skipped_same = 0 | |
| try: | |
| container = av.open(path) | |
| vstream = container.streams.video[0] | |
| vstream.thread_type = "AUTO" | |
| tb = vstream.time_base | |
| duration = float(vstream.duration * tb) if vstream.duration else None | |
| if not duration and container.duration: | |
| duration = float(container.duration) / 1_000_000.0 # AV_TIME_BASE | |
| if not duration or duration <= 0: | |
| # No duration in the metadata (fragmented MP4, some WebM, a few recorders). | |
| # Rather than fall back to sampling the opening and calling it the video, | |
| # demux the stream WITHOUT decoding to find the last timestamp. Packets | |
| # carry their pts and skipping the decode makes this cheap, and it puts us | |
| # back on the evenly-spaced path below. | |
| # On its OWN handle. Demuxing runs the stream to the end, and a raw | |
| # elementary stream cannot seek back, so probing through the container we | |
| # are about to decode from left it at EOF and the sampler returned nothing. | |
| try: | |
| probe = av.open(path) | |
| try: | |
| pv = probe.streams.video[0] | |
| ptb = pv.time_base | |
| last = 0.0 | |
| for packet in probe.demux(pv): | |
| if packet.pts is not None: | |
| last = max(last, float(packet.pts * ptb)) | |
| if last > 0: | |
| duration = last | |
| print(f"[LUNA] frames: no duration in metadata, measured " | |
| f"{last:.1f}s by demuxing.") | |
| finally: | |
| probe.close() | |
| except Exception: | |
| pass | |
| if duration and duration > 0: | |
| # Evenly spaced timestamps; seek to each and grab the next decoded frame. | |
| times = [duration * (i + 1) / (max_frames + 1) for i in range(max_frames)] | |
| for t in times: | |
| try: | |
| # Seeking backward lands on the keyframe BEFORE t, not on t. Taking | |
| # the first frame decoded after that seek therefore returns the | |
| # keyframe, and on a video with a sparse GOP - a screen recording, | |
| # a short clip - every seek lands on the SAME keyframe. That sent | |
| # the vision model several copies of one picture, each labelled with | |
| # a different time it had never seen. Decode forward to reach t, and | |
| # report the timestamp of the frame actually used. | |
| container.seek(int(t / tb), stream=vstream, any_frame=False, backward=True) | |
| picked = None | |
| for frame in container.decode(vstream): | |
| picked = frame | |
| if frame.pts is not None and float(frame.pts * tb) >= t: | |
| break | |
| if picked is None: | |
| continue | |
| actual = float(picked.pts * tb) if picked.pts is not None else t | |
| if out and abs(actual - out[-1][0]) < 0.05: | |
| continue # the decoder had nothing new to give at this point | |
| img = picked.to_image() | |
| sig = _signature(img) | |
| if sig in sigs: | |
| skipped_same += 1 | |
| continue | |
| sigs.add(sig) | |
| out.append((round(actual, 1), _to_data_url(img))) | |
| except Exception: | |
| continue | |
| else: | |
| # Still no idea how long it is, so spread the sample over the whole stream | |
| # by keeping the LAST max_frames of evenly thinned frames: a fixed "every | |
| # 30th frame, stop at eight" covered the first ten seconds of a 25fps clip | |
| # and reported it as a view of the video. Decoding runs to the end either | |
| # way; only which frames are kept changes. | |
| # A raw elementary stream carries no timestamps at all, so pts is None on | |
| # every frame. Requiring one here returned an empty list and the video was | |
| # reported unreadable; the frame index over the stream's nominal rate is a | |
| # perfectly good estimate, and better than no video at all. | |
| try: | |
| fps = float(vstream.average_rate) if vstream.average_rate else 0.0 | |
| except Exception: | |
| fps = 0.0 | |
| res = Reservoir() | |
| for frame in container.decode(vstream): | |
| idx = res.idx | |
| if frame.pts is not None: | |
| ts = float(frame.pts * tb) | |
| else: | |
| ts = idx / fps if fps > 0 else 0.0 | |
| res.offer(lambda fr=frame, t=ts: (t, _shrink(fr.to_image()))) | |
| picked_all = res.items | |
| if picked_all: | |
| stride = max(1, len(picked_all) // max_frames) | |
| for ts, img in picked_all[::stride][:max_frames]: | |
| sig = _signature(img) | |
| if sig in sigs: | |
| skipped_same += 1 | |
| continue | |
| sigs.add(sig) | |
| out.append((round(ts, 1), _to_data_url(img))) | |
| container.close() | |
| except Exception as e: | |
| print(f"[LUNA] frames: sampling failed for {path} ({type(e).__name__}: {e}).") | |
| return out | |
| print(f"[LUNA] frames: sampled {len(out)} keyframe(s)" | |
| + (f", {skipped_same} near-duplicate(s) dropped" if skipped_same else "") + ".") | |
| return out | |