Spaces:
Paused
Czas klatki byl ucinany w dol i publikowany w Notatce
Browse filesframes.py zapisuje prawdziwy czas prezentacji zdekodowanej klatki z
dokladnoscia do dziesiatej sekundy, a synthesis.py renderowal go przez int(),
czyli UCINAL: klatka z 1,7 s szla do modelu jako "Frame at 1s", zawsze za
wczesnie, nawet o sekunde. Model cytuje te etykiety w notatkach, a uzytkownik
czyta je w gotowej Notatce, wiec blad byl publikowany.
Przy dluzszym nagraniu sam format przestawal byc czasem: "3725s" to godzina
i dwie minuty, tylko nikt tego tak nie czyta. Ponizej minuty jest teraz
"1.7s", powyzej "1:02:05".
Sprawdzone na prawdziwym modelu, trzyslajdowa prezentacja: przed zmiana model
pisal "Frame at 1s / 5s / 10s", po zmianie cytuje "1.7s", "5s i 6.7s",
"10s i 11.7s", czyli dokladnie te klatki, ktore dostal.
Naprawione tez wlasne niedopatrzenie z poprzedniego commita: werdykt "nie
udalo sie odczytac wideo (brak klatek i mowy)" oznaczylem jako trwaly, a
dokladnie tak samo wyglada z tego miejsca brak PyAV albo Whisper, ktory sie
nie zaladowal. Na Space z chwilowo niedostepnym narzedziem dobre nagranie
zostaloby zapamietane jako nieczytelne na caly czas zycia wpisu. Oba werdykty
medialne nie sa juz zapamietywane, i nic na tym nie tracimy, bo drozsza
polowe - transkrypcje - trzyma juz cache kluczowany samym plikiem.
Zweryfikowane przy okazji, dzialaja: skanowany PDF (7/7 faktow odczytanych
przez model widzenia, 3 s), napisy z kontenera (4 kwestie w 0,06 s zamiast
okolo 40 s Whisperem, pelne polskie znaki), deduplikacja klatek (5 unikalnych,
3 odrzucone), krok "Am I right?" w obu jezykach.
Testy: 43 -> 47.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
- pipeline.py +12 -2
- synthesis.py +19 -1
- test_luna.py +48 -1
|
@@ -106,12 +106,22 @@ def _process_one(prompt: str, path: str, name: str) -> dict:
|
|
| 106 |
elif transcript:
|
| 107 |
m = map_text(prompt, name, cat, transcript)
|
| 108 |
else:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
return _skip(path, prompt, name, cat,
|
| 110 |
-
"the video couldn't be read (no frames or speech)"
|
|
|
|
| 111 |
else: # audio
|
| 112 |
transcript = _transcribe(info["payload"])
|
| 113 |
if not transcript:
|
| 114 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
m = map_text(prompt, name, cat, transcript)
|
| 116 |
else: # text
|
| 117 |
m = map_text(prompt, name, cat, info["payload"])
|
|
|
|
| 106 |
elif transcript:
|
| 107 |
m = map_text(prompt, name, cat, transcript)
|
| 108 |
else:
|
| 109 |
+
# NOT remembered. "No frames or speech" is exactly what a missing
|
| 110 |
+
# PyAV or a Whisper that failed to load also looks like from here,
|
| 111 |
+
# and caching that would mark a perfectly good recording unreadable
|
| 112 |
+
# for the life of the entry. Nothing is lost by re-checking: the
|
| 113 |
+
# transcript cache above already holds the expensive half.
|
| 114 |
return _skip(path, prompt, name, cat,
|
| 115 |
+
"the video couldn't be read (no frames or speech)",
|
| 116 |
+
cacheable=False)
|
| 117 |
else: # audio
|
| 118 |
transcript = _transcribe(info["payload"])
|
| 119 |
if not transcript:
|
| 120 |
+
# Same reasoning: an empty transcript can mean a silent file or a
|
| 121 |
+
# Whisper that never loaded, and only one of those is the file's
|
| 122 |
+
# fault. The transcript cache already saves the second pass.
|
| 123 |
+
return _skip(path, prompt, name, cat, "no speech could be transcribed",
|
| 124 |
+
cacheable=False)
|
| 125 |
m = map_text(prompt, name, cat, transcript)
|
| 126 |
else: # text
|
| 127 |
m = map_text(prompt, name, cat, info["payload"])
|
|
@@ -251,6 +251,24 @@ def map_image(prompt: str, name: str, data_url: str) -> dict:
|
|
| 251 |
"error": "the vision model could not be reached"}
|
| 252 |
|
| 253 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 254 |
def map_video(prompt: str, name: str, transcript: str, frames: list[tuple]) -> dict:
|
| 255 |
"""Real video understanding (NINA-style): the VLM sees sampled keyframes + the
|
| 256 |
speech transcript and extracts the request-relevant content. Works on silent
|
|
@@ -267,7 +285,7 @@ def map_video(prompt: str, name: str, transcript: str, frames: list[tuple]) -> d
|
|
| 267 |
if transcript:
|
| 268 |
user.append({"type": "text", "text": "Speech transcript:\n" + transcript[:8000]})
|
| 269 |
for ts, url in frames:
|
| 270 |
-
user.append({"type": "text", "text": f"Frame at {
|
| 271 |
user.append({"type": "image_url", "image_url": {"url": url}})
|
| 272 |
try:
|
| 273 |
out = _map_json(VLM_MODEL, [
|
|
|
|
| 251 |
"error": "the vision model could not be reached"}
|
| 252 |
|
| 253 |
|
| 254 |
+
def _stamp(ts: float) -> str:
|
| 255 |
+
"""A frame's time as a person would write it.
|
| 256 |
+
|
| 257 |
+
frames.py goes to the trouble of recording the decoded frame's real presentation
|
| 258 |
+
time to a tenth of a second, and this used to render it with int(), which TRUNCATES:
|
| 259 |
+
a frame at 1.7s was announced as "1s", always early, by up to a second. The model
|
| 260 |
+
quotes these labels back in its notes and the user reads them in the Note, so the
|
| 261 |
+
error was being published. Past a minute, seconds alone stop being a time anyone can
|
| 262 |
+
use - "3725s" is an hour into a recording - so it switches to h:mm:ss.
|
| 263 |
+
"""
|
| 264 |
+
ts = max(0.0, float(ts or 0.0))
|
| 265 |
+
if ts < 60:
|
| 266 |
+
return f"{ts:.1f}s".replace(".0s", "s")
|
| 267 |
+
m, sec = divmod(int(round(ts)), 60)
|
| 268 |
+
h, m = divmod(m, 60)
|
| 269 |
+
return f"{h}:{m:02d}:{sec:02d}" if h else f"{m}:{sec:02d}"
|
| 270 |
+
|
| 271 |
+
|
| 272 |
def map_video(prompt: str, name: str, transcript: str, frames: list[tuple]) -> dict:
|
| 273 |
"""Real video understanding (NINA-style): the VLM sees sampled keyframes + the
|
| 274 |
speech transcript and extracts the request-relevant content. Works on silent
|
|
|
|
| 285 |
if transcript:
|
| 286 |
user.append({"type": "text", "text": "Speech transcript:\n" + transcript[:8000]})
|
| 287 |
for ts, url in frames:
|
| 288 |
+
user.append({"type": "text", "text": f"Frame at {_stamp(ts)}:"})
|
| 289 |
user.append({"type": "image_url", "image_url": {"url": url}})
|
| 290 |
try:
|
| 291 |
out = _map_json(VLM_MODEL, [
|
|
@@ -12,7 +12,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
| 12 |
|
| 13 |
from extract import _looks_binary, _word_density, category_of, extract # noqa: E402
|
| 14 |
from pipeline import _resolve, _validate # noqa: E402
|
| 15 |
-
from synthesis import (_fix_filenames, _fold, _parse_json,
|
| 16 |
_reduce_user, detect_language) # noqa: E402
|
| 17 |
|
| 18 |
|
|
@@ -410,3 +410,50 @@ def test_fingerprint_follows_content_not_path():
|
|
| 410 |
with open(b, "ab") as fh:
|
| 411 |
fh.write(b"roznica")
|
| 412 |
assert cache.fingerprint(a) != cache.fingerprint(b)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
from extract import _looks_binary, _word_density, category_of, extract # noqa: E402
|
| 14 |
from pipeline import _resolve, _validate # noqa: E402
|
| 15 |
+
from synthesis import (_fix_filenames, _fold, _parse_json, _stamp, # noqa: E402
|
| 16 |
_reduce_user, detect_language) # noqa: E402
|
| 17 |
|
| 18 |
|
|
|
|
| 410 |
with open(b, "ab") as fh:
|
| 411 |
fh.write(b"roznica")
|
| 412 |
assert cache.fingerprint(a) != cache.fingerprint(b)
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
def test_a_media_verdict_is_never_remembered():
|
| 416 |
+
""""No frames or speech" is also what a missing PyAV looks like from here."""
|
| 417 |
+
import cache, pipeline
|
| 418 |
+
with tempfile.TemporaryDirectory() as d:
|
| 419 |
+
old, cache.CACHE_DIR = cache.CACHE_DIR, d
|
| 420 |
+
f = os.path.join(d, "film.mp4")
|
| 421 |
+
with open(f, "wb") as fh:
|
| 422 |
+
fh.write(b"udawane wideo" * 200)
|
| 423 |
+
try:
|
| 424 |
+
pipeline._skip(f, "p", "film.mp4", "video",
|
| 425 |
+
"the video couldn't be read (no frames or speech)",
|
| 426 |
+
cacheable=False)
|
| 427 |
+
assert cache.load(f, "p") is None
|
| 428 |
+
pipeline._skip(f, "p", "film.mp4", "audio",
|
| 429 |
+
"no speech could be transcribed", cacheable=False)
|
| 430 |
+
assert cache.load(f, "p") is None
|
| 431 |
+
finally:
|
| 432 |
+
cache.CACHE_DIR = old
|
| 433 |
+
|
| 434 |
+
|
| 435 |
+
# --- how a frame's time is written ---------------------------------------------
|
| 436 |
+
|
| 437 |
+
def test_a_frame_time_is_not_truncated():
|
| 438 |
+
"""int() always rounded down: a frame at 1.7s was published as "1s"."""
|
| 439 |
+
assert _stamp(1.7) == "1.7s"
|
| 440 |
+
assert _stamp(9.9) == "9.9s"
|
| 441 |
+
assert _stamp(0.4) == "0.4s"
|
| 442 |
+
|
| 443 |
+
|
| 444 |
+
def test_whole_seconds_lose_the_decimal():
|
| 445 |
+
assert _stamp(42.0) == "42s"
|
| 446 |
+
assert _stamp(0) == "0s"
|
| 447 |
+
|
| 448 |
+
|
| 449 |
+
def test_past_a_minute_it_becomes_a_clock():
|
| 450 |
+
assert _stamp(60) == "1:00"
|
| 451 |
+
assert _stamp(125) == "2:05"
|
| 452 |
+
assert _stamp(599) == "9:59"
|
| 453 |
+
assert _stamp(3725) == "1:02:05" # "3725s" is not a time anyone can use
|
| 454 |
+
assert _stamp(7384.6) == "2:03:05"
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def test_a_nonsense_time_does_not_raise():
|
| 458 |
+
assert _stamp(-5) == "0s"
|
| 459 |
+
assert _stamp(None) == "0s"
|