Malina Claude Opus 5 (1M context) commited on
Commit
e4a00dc
·
1 Parent(s): 1a8a270

Czas klatki byl ucinany w dol i publikowany w Notatce

Browse files

frames.py zapisuje prawdziwy czas prezentacji zdekodowanej klatki z
dokladnoscia do dziesiatej sekundy, a synthesis.py renderowal go przez int(),
czyli UCINAL: klatka z 1,7 s szla do modelu jako "Frame at 1s", zawsze za
wczesnie, nawet o sekunde. Model cytuje te etykiety w notatkach, a uzytkownik
czyta je w gotowej Notatce, wiec blad byl publikowany.

Przy dluzszym nagraniu sam format przestawal byc czasem: "3725s" to godzina
i dwie minuty, tylko nikt tego tak nie czyta. Ponizej minuty jest teraz
"1.7s", powyzej "1:02:05".

Sprawdzone na prawdziwym modelu, trzyslajdowa prezentacja: przed zmiana model
pisal "Frame at 1s / 5s / 10s", po zmianie cytuje "1.7s", "5s i 6.7s",
"10s i 11.7s", czyli dokladnie te klatki, ktore dostal.

Naprawione tez wlasne niedopatrzenie z poprzedniego commita: werdykt "nie
udalo sie odczytac wideo (brak klatek i mowy)" oznaczylem jako trwaly, a
dokladnie tak samo wyglada z tego miejsca brak PyAV albo Whisper, ktory sie
nie zaladowal. Na Space z chwilowo niedostepnym narzedziem dobre nagranie
zostaloby zapamietane jako nieczytelne na caly czas zycia wpisu. Oba werdykty
medialne nie sa juz zapamietywane, i nic na tym nie tracimy, bo drozsza
polowe - transkrypcje - trzyma juz cache kluczowany samym plikiem.

Zweryfikowane przy okazji, dzialaja: skanowany PDF (7/7 faktow odczytanych
przez model widzenia, 3 s), napisy z kontenera (4 kwestie w 0,06 s zamiast
okolo 40 s Whisperem, pelne polskie znaki), deduplikacja klatek (5 unikalnych,
3 odrzucone), krok "Am I right?" w obu jezykach.

Testy: 43 -> 47.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Files changed (3) hide show
  1. pipeline.py +12 -2
  2. synthesis.py +19 -1
  3. test_luna.py +48 -1
pipeline.py CHANGED
@@ -106,12 +106,22 @@ def _process_one(prompt: str, path: str, name: str) -> dict:
106
  elif transcript:
107
  m = map_text(prompt, name, cat, transcript)
108
  else:
 
 
 
 
 
109
  return _skip(path, prompt, name, cat,
110
- "the video couldn't be read (no frames or speech)")
 
111
  else: # audio
112
  transcript = _transcribe(info["payload"])
113
  if not transcript:
114
- return _skip(path, prompt, name, cat, "no speech could be transcribed")
 
 
 
 
115
  m = map_text(prompt, name, cat, transcript)
116
  else: # text
117
  m = map_text(prompt, name, cat, info["payload"])
 
106
  elif transcript:
107
  m = map_text(prompt, name, cat, transcript)
108
  else:
109
+ # NOT remembered. "No frames or speech" is exactly what a missing
110
+ # PyAV or a Whisper that failed to load also looks like from here,
111
+ # and caching that would mark a perfectly good recording unreadable
112
+ # for the life of the entry. Nothing is lost by re-checking: the
113
+ # transcript cache above already holds the expensive half.
114
  return _skip(path, prompt, name, cat,
115
+ "the video couldn't be read (no frames or speech)",
116
+ cacheable=False)
117
  else: # audio
118
  transcript = _transcribe(info["payload"])
119
  if not transcript:
120
+ # Same reasoning: an empty transcript can mean a silent file or a
121
+ # Whisper that never loaded, and only one of those is the file's
122
+ # fault. The transcript cache already saves the second pass.
123
+ return _skip(path, prompt, name, cat, "no speech could be transcribed",
124
+ cacheable=False)
125
  m = map_text(prompt, name, cat, transcript)
126
  else: # text
127
  m = map_text(prompt, name, cat, info["payload"])
synthesis.py CHANGED
@@ -251,6 +251,24 @@ def map_image(prompt: str, name: str, data_url: str) -> dict:
251
  "error": "the vision model could not be reached"}
252
 
253
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
254
  def map_video(prompt: str, name: str, transcript: str, frames: list[tuple]) -> dict:
255
  """Real video understanding (NINA-style): the VLM sees sampled keyframes + the
256
  speech transcript and extracts the request-relevant content. Works on silent
@@ -267,7 +285,7 @@ def map_video(prompt: str, name: str, transcript: str, frames: list[tuple]) -> d
267
  if transcript:
268
  user.append({"type": "text", "text": "Speech transcript:\n" + transcript[:8000]})
269
  for ts, url in frames:
270
- user.append({"type": "text", "text": f"Frame at {int(ts)}s:"})
271
  user.append({"type": "image_url", "image_url": {"url": url}})
272
  try:
273
  out = _map_json(VLM_MODEL, [
 
251
  "error": "the vision model could not be reached"}
252
 
253
 
254
+ def _stamp(ts: float) -> str:
255
+ """A frame's time as a person would write it.
256
+
257
+ frames.py goes to the trouble of recording the decoded frame's real presentation
258
+ time to a tenth of a second, and this used to render it with int(), which TRUNCATES:
259
+ a frame at 1.7s was announced as "1s", always early, by up to a second. The model
260
+ quotes these labels back in its notes and the user reads them in the Note, so the
261
+ error was being published. Past a minute, seconds alone stop being a time anyone can
262
+ use - "3725s" is an hour into a recording - so it switches to h:mm:ss.
263
+ """
264
+ ts = max(0.0, float(ts or 0.0))
265
+ if ts < 60:
266
+ return f"{ts:.1f}s".replace(".0s", "s")
267
+ m, sec = divmod(int(round(ts)), 60)
268
+ h, m = divmod(m, 60)
269
+ return f"{h}:{m:02d}:{sec:02d}" if h else f"{m}:{sec:02d}"
270
+
271
+
272
  def map_video(prompt: str, name: str, transcript: str, frames: list[tuple]) -> dict:
273
  """Real video understanding (NINA-style): the VLM sees sampled keyframes + the
274
  speech transcript and extracts the request-relevant content. Works on silent
 
285
  if transcript:
286
  user.append({"type": "text", "text": "Speech transcript:\n" + transcript[:8000]})
287
  for ts, url in frames:
288
+ user.append({"type": "text", "text": f"Frame at {_stamp(ts)}:"})
289
  user.append({"type": "image_url", "image_url": {"url": url}})
290
  try:
291
  out = _map_json(VLM_MODEL, [
test_luna.py CHANGED
@@ -12,7 +12,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
12
 
13
  from extract import _looks_binary, _word_density, category_of, extract # noqa: E402
14
  from pipeline import _resolve, _validate # noqa: E402
15
- from synthesis import (_fix_filenames, _fold, _parse_json, # noqa: E402
16
  _reduce_user, detect_language) # noqa: E402
17
 
18
 
@@ -410,3 +410,50 @@ def test_fingerprint_follows_content_not_path():
410
  with open(b, "ab") as fh:
411
  fh.write(b"roznica")
412
  assert cache.fingerprint(a) != cache.fingerprint(b)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
 
13
  from extract import _looks_binary, _word_density, category_of, extract # noqa: E402
14
  from pipeline import _resolve, _validate # noqa: E402
15
+ from synthesis import (_fix_filenames, _fold, _parse_json, _stamp, # noqa: E402
16
  _reduce_user, detect_language) # noqa: E402
17
 
18
 
 
410
  with open(b, "ab") as fh:
411
  fh.write(b"roznica")
412
  assert cache.fingerprint(a) != cache.fingerprint(b)
413
+
414
+
415
+ def test_a_media_verdict_is_never_remembered():
416
+ """"No frames or speech" is also what a missing PyAV looks like from here."""
417
+ import cache, pipeline
418
+ with tempfile.TemporaryDirectory() as d:
419
+ old, cache.CACHE_DIR = cache.CACHE_DIR, d
420
+ f = os.path.join(d, "film.mp4")
421
+ with open(f, "wb") as fh:
422
+ fh.write(b"udawane wideo" * 200)
423
+ try:
424
+ pipeline._skip(f, "p", "film.mp4", "video",
425
+ "the video couldn't be read (no frames or speech)",
426
+ cacheable=False)
427
+ assert cache.load(f, "p") is None
428
+ pipeline._skip(f, "p", "film.mp4", "audio",
429
+ "no speech could be transcribed", cacheable=False)
430
+ assert cache.load(f, "p") is None
431
+ finally:
432
+ cache.CACHE_DIR = old
433
+
434
+
435
+ # --- how a frame's time is written ---------------------------------------------
436
+
437
+ def test_a_frame_time_is_not_truncated():
438
+ """int() always rounded down: a frame at 1.7s was published as "1s"."""
439
+ assert _stamp(1.7) == "1.7s"
440
+ assert _stamp(9.9) == "9.9s"
441
+ assert _stamp(0.4) == "0.4s"
442
+
443
+
444
+ def test_whole_seconds_lose_the_decimal():
445
+ assert _stamp(42.0) == "42s"
446
+ assert _stamp(0) == "0s"
447
+
448
+
449
+ def test_past_a_minute_it_becomes_a_clock():
450
+ assert _stamp(60) == "1:00"
451
+ assert _stamp(125) == "2:05"
452
+ assert _stamp(599) == "9:59"
453
+ assert _stamp(3725) == "1:02:05" # "3725s" is not a time anyone can use
454
+ assert _stamp(7384.6) == "2:03:05"
455
+
456
+
457
+ def test_a_nonsense_time_does_not_raise():
458
+ assert _stamp(-5) == "0s"
459
+ assert _stamp(None) == "0s"