Malina Claude Opus 5 (1M context) commited on
Commit
eac204b
·
1 Parent(s): cadec28

Transkrypcja byla jedyna trescia, ktora szla do modelu bez ograniczenia

Browse files

Kazdy inny typ pliku przechodzi przez extract(), gdzie obowiazuje
PER_FILE_CHARS. Transkrypcja nie: idzie prosto z Whispera albo ze sciezki
napisow do map_text, z pominieciem tej drogi. Trzy godziny napisow to grubo
ponad sto tysiecy znakow, czyli daleko poza to, co model MAP przyjmie w
jednym wywolaniu, a awaria wygladalaby na kapryszacy model, nie na zbyt duze
zapytanie. Ograniczenie siedzi teraz w _map_user, wiec zaden wywolujacy go
nie obejdzie, i jest powiedziane wprost w zapytaniu, zeby model nie udawal,
ze przeczytal plik, ktorego widzial tylko poczatek.

Przy okazji dwie rzeczy po stronie napisow:

- Napisy nie podlegaly limitowi TRANSCRIBE_MAX_SECONDS, ktory obowiazuje
Whispera. Ta sama robota sluchala innej reguly zaleznie tylko od tego, skad
wzielismy slowa. Teraz obie konczą w tym samym miejscu.
- Napisy przewijane (rodzina eia_608 szczegolnie) powtarzaja te sama linijke
przez kolejne kwestie, gdy tekst sie przesuwa. Kazda kopia byla placona w
tokenach. Scalane sa tylko powtorzenia SASIEDNIE, bo refren wracajacy
pozniej to tresc, nie artefakt.

Testy: 60 -> 67.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Files changed (3) hide show
  1. subtitles.py +22 -3
  2. synthesis.py +15 -1
  3. test_luna.py +66 -0
subtitles.py CHANGED
@@ -18,6 +18,8 @@ import shutil
18
  import subprocess
19
  import tempfile
20
 
 
 
21
  TEXT_CODECS = {"subrip", "srt", "ass", "ssa", "mov_text", "webvtt", "text", "eia_608", "subviewer"}
22
 
23
  _SRT_TIME = re.compile(r"(\d+):(\d+):(\d+)[,.](\d+)\s*-->\s*(\d+):(\d+):(\d+)[,.](\d+)")
@@ -40,18 +42,35 @@ def _first_text_track(path: str):
40
  return None
41
 
42
 
 
 
 
 
 
43
  def _parse_srt(text: str) -> list[str]:
44
- lines = []
 
 
 
 
 
 
 
 
 
45
  for block in re.split(r"\n\s*\n", text.strip()):
46
- if not _SRT_TIME.search(block):
 
47
  continue
 
 
48
  rows = block.splitlines()
49
  ti = next((i for i, ln in enumerate(rows) if "-->" in ln), 0)
50
  txt = " ".join(ln.strip() for ln in rows[ti + 1:] if ln.strip())
51
  txt = re.sub(r"<[^>]+>", "", txt) # styling tags
52
  txt = re.sub(r"\{[^}]+\}", "", txt) # ASS override blocks
53
  txt = txt.strip()
54
- if txt:
55
  lines.append(txt)
56
  return lines
57
 
 
18
  import subprocess
19
  import tempfile
20
 
21
+ from config import TRANSCRIBE_MAX_SECONDS
22
+
23
  TEXT_CODECS = {"subrip", "srt", "ass", "ssa", "mov_text", "webvtt", "text", "eia_608", "subviewer"}
24
 
25
  _SRT_TIME = re.compile(r"(\d+):(\d+):(\d+)[,.](\d+)\s*-->\s*(\d+):(\d+):(\d+)[,.](\d+)")
 
42
  return None
43
 
44
 
45
+ def _start_seconds(match) -> float:
46
+ h, m, sec, frac = match.group(1), match.group(2), match.group(3), match.group(4)
47
+ return int(h) * 3600 + int(m) * 60 + int(sec) + int(frac) / (10 ** len(frac))
48
+
49
+
50
  def _parse_srt(text: str) -> list[str]:
51
+ """Cues as plain lines, in order.
52
+
53
+ Two things the raw track does that the transcript should not inherit. Rolling
54
+ captions (the eia_608 family especially) repeat the same line across consecutive
55
+ cues as the text scrolls, which would pad the transcript with duplicates and pay
56
+ tokens for them. And Whisper stops at TRANSCRIBE_MAX_SECONDS, while a subtitle
57
+ track had no cap at all: a three hour film handed over its whole script, which is
58
+ a different rule for the same job depending only on how the words were obtained.
59
+ """
60
+ lines: list[str] = []
61
  for block in re.split(r"\n\s*\n", text.strip()):
62
+ m = _SRT_TIME.search(block)
63
+ if not m:
64
  continue
65
+ if _start_seconds(m) > TRANSCRIBE_MAX_SECONDS:
66
+ break
67
  rows = block.splitlines()
68
  ti = next((i for i, ln in enumerate(rows) if "-->" in ln), 0)
69
  txt = " ".join(ln.strip() for ln in rows[ti + 1:] if ln.strip())
70
  txt = re.sub(r"<[^>]+>", "", txt) # styling tags
71
  txt = re.sub(r"\{[^}]+\}", "", txt) # ASS override blocks
72
  txt = txt.strip()
73
+ if txt and txt != (lines[-1] if lines else None):
74
  lines.append(txt)
75
  return lines
76
 
synthesis.py CHANGED
@@ -7,6 +7,7 @@ import re
7
  import unicodedata
8
 
9
  from config import (
 
10
  DEBUG, MAP_MAX_TOKENS, MAP_MODEL, MAP_TEMPERATURE, REDUCE_BUDGET,
11
  MAP_MAX_TOKENS_RETRY, REDUCE_MAX_TOKENS, REDUCE_MODEL, REDUCE_TEMPERATURE, VLM_MODEL,
12
  )
@@ -228,10 +229,23 @@ def _map_json(model: str, messages: list, label: str) -> dict:
228
 
229
 
230
  def _map_user(prompt: str, name: str, category: str, content: str) -> str:
 
 
 
 
 
 
 
 
 
 
 
 
 
231
  return (
232
  f'Build request: "{prompt}"\n\n'
233
  f"File: {name} (type: {category})\n"
234
- f"--- file content ---\n{content}\n--- end ---\n\n"
235
  "Extract this file's content AND grade its relevance. Return the strict JSON "
236
  "described in the system message."
237
  )
 
7
  import unicodedata
8
 
9
  from config import (
10
+ PER_FILE_CHARS,
11
  DEBUG, MAP_MAX_TOKENS, MAP_MODEL, MAP_TEMPERATURE, REDUCE_BUDGET,
12
  MAP_MAX_TOKENS_RETRY, REDUCE_MAX_TOKENS, REDUCE_MODEL, REDUCE_TEMPERATURE, VLM_MODEL,
13
  )
 
229
 
230
 
231
  def _map_user(prompt: str, name: str, category: str, content: str) -> str:
232
+ # The cap lives here rather than only at the call sites, because not every caller
233
+ # comes through extract(): a transcript goes straight from Whisper or from the
234
+ # container's subtitle track into map_text, uncapped. Three hours of subtitles is
235
+ # well over a hundred thousand characters, which is far past what the MAP model
236
+ # will accept in one call, and the failure would have looked like a flaky model
237
+ # rather than an oversized prompt. Said out loud, too, so the model does not claim
238
+ # to have read a file it only saw the front of.
239
+ content = content or ""
240
+ cut = ""
241
+ if len(content) > PER_FILE_CHARS:
242
+ content = content[:PER_FILE_CHARS]
243
+ cut = ("\n\n[This file is longer than what fits in one pass. The text above is "
244
+ "its opening; say so in the notes rather than implying the file ends here.]")
245
  return (
246
  f'Build request: "{prompt}"\n\n'
247
  f"File: {name} (type: {category})\n"
248
+ f"--- file content ---\n{content}{cut}\n--- end ---\n\n"
249
  "Extract this file's content AND grade its relevance. Return the strict JSON "
250
  "described in the system message."
251
  )
test_luna.py CHANGED
@@ -640,3 +640,69 @@ def test_every_category_the_space_can_return_has_a_label_in_the_wizard():
640
  labelled = set(re.findall(r"([a-z]+)\s*:\s*\[", cat.group(1)))
641
  used = set(extract._EXT.values()) | {"other"}
642
  assert not (used - labelled), f"no label for: {sorted(used - labelled)}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
640
  labelled = set(re.findall(r"([a-z]+)\s*:\s*\[", cat.group(1)))
641
  used = set(extract._EXT.values()) | {"other"}
642
  assert not (used - labelled), f"no label for: {sorted(used - labelled)}"
643
+
644
+
645
+ # --- subtitles from the container ----------------------------------------------
646
+
647
+ def test_rolling_captions_are_not_repeated_into_the_transcript():
648
+ """A scrolling caption track repeats the same line across cues as it rolls, and
649
+ every copy would have been paid for in tokens."""
650
+ from subtitles import _parse_srt
651
+ srt = ("1\n00:00:01,000 --> 00:00:03,000\nDzien dobry.\n\n"
652
+ "2\n00:00:03,000 --> 00:00:05,000\nDzien dobry.\n\n"
653
+ "3\n00:00:05,000 --> 00:00:07,000\nZaczynamy.\n")
654
+ assert _parse_srt(srt) == ["Dzien dobry.", "Zaczynamy."]
655
+
656
+
657
+ def test_a_line_that_comes_back_later_is_kept():
658
+ """Only CONSECUTIVE repeats are the rolling artefact; a refrain is content."""
659
+ from subtitles import _parse_srt
660
+ srt = ("1\n00:00:01,000 --> 00:00:03,000\nRefren.\n\n"
661
+ "2\n00:00:03,000 --> 00:00:05,000\nZwrotka.\n\n"
662
+ "3\n00:00:05,000 --> 00:00:07,000\nRefren.\n")
663
+ assert _parse_srt(srt) == ["Refren.", "Zwrotka.", "Refren."]
664
+
665
+
666
+ def test_subtitles_stop_at_the_same_point_whisper_does():
667
+ """A three hour film used to hand over its whole script, while Whisper stopped at
668
+ thirty minutes: the same job obeyed a different rule depending only on how the
669
+ words were obtained."""
670
+ from subtitles import _parse_srt
671
+ from config import TRANSCRIBE_MAX_SECONDS
672
+ late = TRANSCRIBE_MAX_SECONDS + 120
673
+ h, rest = divmod(int(late), 3600)
674
+ m, sec = divmod(rest, 60)
675
+ srt = ("1\n00:00:01,000 --> 00:00:03,000\nNa poczatku.\n\n"
676
+ f"2\n{h:02d}:{m:02d}:{sec:02d},000 --> {h:02d}:{m:02d}:{sec + 2:02d},000\nPo limicie.\n")
677
+ out = _parse_srt(srt)
678
+ assert out == ["Na poczatku."]
679
+
680
+
681
+ def test_styling_and_override_blocks_are_stripped():
682
+ from subtitles import _parse_srt
683
+ srt = ("1\n00:00:01,000 --> 00:00:03,000\n<i>Kursywa</i> i {\\an8}pozycja\n")
684
+ assert _parse_srt(srt) == ["Kursywa i pozycja"]
685
+
686
+
687
+ def test_a_cue_time_is_read_with_its_fraction():
688
+ from subtitles import _SRT_TIME, _start_seconds
689
+ assert _start_seconds(_SRT_TIME.search("00:01:02,500 --> 00:01:05,000")) == 62.5
690
+ assert _start_seconds(_SRT_TIME.search("01:00:00.250 --> 01:00:02,000")) == 3600.25
691
+
692
+
693
+ # --- nothing may reach the model uncapped --------------------------------------
694
+
695
+ def test_an_oversized_transcript_is_cut_before_it_reaches_the_model():
696
+ """A transcript goes straight from Whisper or the subtitle track into map_text,
697
+ never through extract(), so it never met the per-file cap."""
698
+ from synthesis import _map_user
699
+ from config import PER_FILE_CHARS
700
+ out = _map_user("p", "film.srt", "video", "x" * (PER_FILE_CHARS + 90000))
701
+ assert len(out) < PER_FILE_CHARS + 900
702
+ assert "longer than" in out # and the model is told, so it cannot pretend
703
+
704
+
705
+ def test_a_normal_file_is_passed_through_untouched():
706
+ from synthesis import _map_user
707
+ out = _map_user("p", "a.txt", "text", "krotka tresc pliku")
708
+ assert "krotka tresc pliku" in out and "longer than" not in out