Spaces:
Paused
Transkrypcja byla jedyna trescia, ktora szla do modelu bez ograniczenia
Browse filesKazdy inny typ pliku przechodzi przez extract(), gdzie obowiazuje
PER_FILE_CHARS. Transkrypcja nie: idzie prosto z Whispera albo ze sciezki
napisow do map_text, z pominieciem tej drogi. Trzy godziny napisow to grubo
ponad sto tysiecy znakow, czyli daleko poza to, co model MAP przyjmie w
jednym wywolaniu, a awaria wygladalaby na kapryszacy model, nie na zbyt duze
zapytanie. Ograniczenie siedzi teraz w _map_user, wiec zaden wywolujacy go
nie obejdzie, i jest powiedziane wprost w zapytaniu, zeby model nie udawal,
ze przeczytal plik, ktorego widzial tylko poczatek.
Przy okazji dwie rzeczy po stronie napisow:
- Napisy nie podlegaly limitowi TRANSCRIBE_MAX_SECONDS, ktory obowiazuje
Whispera. Ta sama robota sluchala innej reguly zaleznie tylko od tego, skad
wzielismy slowa. Teraz obie konczą w tym samym miejscu.
- Napisy przewijane (rodzina eia_608 szczegolnie) powtarzaja te sama linijke
przez kolejne kwestie, gdy tekst sie przesuwa. Kazda kopia byla placona w
tokenach. Scalane sa tylko powtorzenia SASIEDNIE, bo refren wracajacy
pozniej to tresc, nie artefakt.
Testy: 60 -> 67.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
- subtitles.py +22 -3
- synthesis.py +15 -1
- test_luna.py +66 -0
|
@@ -18,6 +18,8 @@ import shutil
|
|
| 18 |
import subprocess
|
| 19 |
import tempfile
|
| 20 |
|
|
|
|
|
|
|
| 21 |
TEXT_CODECS = {"subrip", "srt", "ass", "ssa", "mov_text", "webvtt", "text", "eia_608", "subviewer"}
|
| 22 |
|
| 23 |
_SRT_TIME = re.compile(r"(\d+):(\d+):(\d+)[,.](\d+)\s*-->\s*(\d+):(\d+):(\d+)[,.](\d+)")
|
|
@@ -40,18 +42,35 @@ def _first_text_track(path: str):
|
|
| 40 |
return None
|
| 41 |
|
| 42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
def _parse_srt(text: str) -> list[str]:
|
| 44 |
-
lines
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
for block in re.split(r"\n\s*\n", text.strip()):
|
| 46 |
-
|
|
|
|
| 47 |
continue
|
|
|
|
|
|
|
| 48 |
rows = block.splitlines()
|
| 49 |
ti = next((i for i, ln in enumerate(rows) if "-->" in ln), 0)
|
| 50 |
txt = " ".join(ln.strip() for ln in rows[ti + 1:] if ln.strip())
|
| 51 |
txt = re.sub(r"<[^>]+>", "", txt) # styling tags
|
| 52 |
txt = re.sub(r"\{[^}]+\}", "", txt) # ASS override blocks
|
| 53 |
txt = txt.strip()
|
| 54 |
-
if txt:
|
| 55 |
lines.append(txt)
|
| 56 |
return lines
|
| 57 |
|
|
|
|
| 18 |
import subprocess
|
| 19 |
import tempfile
|
| 20 |
|
| 21 |
+
from config import TRANSCRIBE_MAX_SECONDS
|
| 22 |
+
|
| 23 |
TEXT_CODECS = {"subrip", "srt", "ass", "ssa", "mov_text", "webvtt", "text", "eia_608", "subviewer"}
|
| 24 |
|
| 25 |
_SRT_TIME = re.compile(r"(\d+):(\d+):(\d+)[,.](\d+)\s*-->\s*(\d+):(\d+):(\d+)[,.](\d+)")
|
|
|
|
| 42 |
return None
|
| 43 |
|
| 44 |
|
| 45 |
+
def _start_seconds(match) -> float:
|
| 46 |
+
h, m, sec, frac = match.group(1), match.group(2), match.group(3), match.group(4)
|
| 47 |
+
return int(h) * 3600 + int(m) * 60 + int(sec) + int(frac) / (10 ** len(frac))
|
| 48 |
+
|
| 49 |
+
|
| 50 |
def _parse_srt(text: str) -> list[str]:
|
| 51 |
+
"""Cues as plain lines, in order.
|
| 52 |
+
|
| 53 |
+
Two things the raw track does that the transcript should not inherit. Rolling
|
| 54 |
+
captions (the eia_608 family especially) repeat the same line across consecutive
|
| 55 |
+
cues as the text scrolls, which would pad the transcript with duplicates and pay
|
| 56 |
+
tokens for them. And Whisper stops at TRANSCRIBE_MAX_SECONDS, while a subtitle
|
| 57 |
+
track had no cap at all: a three hour film handed over its whole script, which is
|
| 58 |
+
a different rule for the same job depending only on how the words were obtained.
|
| 59 |
+
"""
|
| 60 |
+
lines: list[str] = []
|
| 61 |
for block in re.split(r"\n\s*\n", text.strip()):
|
| 62 |
+
m = _SRT_TIME.search(block)
|
| 63 |
+
if not m:
|
| 64 |
continue
|
| 65 |
+
if _start_seconds(m) > TRANSCRIBE_MAX_SECONDS:
|
| 66 |
+
break
|
| 67 |
rows = block.splitlines()
|
| 68 |
ti = next((i for i, ln in enumerate(rows) if "-->" in ln), 0)
|
| 69 |
txt = " ".join(ln.strip() for ln in rows[ti + 1:] if ln.strip())
|
| 70 |
txt = re.sub(r"<[^>]+>", "", txt) # styling tags
|
| 71 |
txt = re.sub(r"\{[^}]+\}", "", txt) # ASS override blocks
|
| 72 |
txt = txt.strip()
|
| 73 |
+
if txt and txt != (lines[-1] if lines else None):
|
| 74 |
lines.append(txt)
|
| 75 |
return lines
|
| 76 |
|
|
@@ -7,6 +7,7 @@ import re
|
|
| 7 |
import unicodedata
|
| 8 |
|
| 9 |
from config import (
|
|
|
|
| 10 |
DEBUG, MAP_MAX_TOKENS, MAP_MODEL, MAP_TEMPERATURE, REDUCE_BUDGET,
|
| 11 |
MAP_MAX_TOKENS_RETRY, REDUCE_MAX_TOKENS, REDUCE_MODEL, REDUCE_TEMPERATURE, VLM_MODEL,
|
| 12 |
)
|
|
@@ -228,10 +229,23 @@ def _map_json(model: str, messages: list, label: str) -> dict:
|
|
| 228 |
|
| 229 |
|
| 230 |
def _map_user(prompt: str, name: str, category: str, content: str) -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
return (
|
| 232 |
f'Build request: "{prompt}"\n\n'
|
| 233 |
f"File: {name} (type: {category})\n"
|
| 234 |
-
f"--- file content ---\n{content}\n--- end ---\n\n"
|
| 235 |
"Extract this file's content AND grade its relevance. Return the strict JSON "
|
| 236 |
"described in the system message."
|
| 237 |
)
|
|
|
|
| 7 |
import unicodedata
|
| 8 |
|
| 9 |
from config import (
|
| 10 |
+
PER_FILE_CHARS,
|
| 11 |
DEBUG, MAP_MAX_TOKENS, MAP_MODEL, MAP_TEMPERATURE, REDUCE_BUDGET,
|
| 12 |
MAP_MAX_TOKENS_RETRY, REDUCE_MAX_TOKENS, REDUCE_MODEL, REDUCE_TEMPERATURE, VLM_MODEL,
|
| 13 |
)
|
|
|
|
| 229 |
|
| 230 |
|
| 231 |
def _map_user(prompt: str, name: str, category: str, content: str) -> str:
|
| 232 |
+
# The cap lives here rather than only at the call sites, because not every caller
|
| 233 |
+
# comes through extract(): a transcript goes straight from Whisper or from the
|
| 234 |
+
# container's subtitle track into map_text, uncapped. Three hours of subtitles is
|
| 235 |
+
# well over a hundred thousand characters, which is far past what the MAP model
|
| 236 |
+
# will accept in one call, and the failure would have looked like a flaky model
|
| 237 |
+
# rather than an oversized prompt. Said out loud, too, so the model does not claim
|
| 238 |
+
# to have read a file it only saw the front of.
|
| 239 |
+
content = content or ""
|
| 240 |
+
cut = ""
|
| 241 |
+
if len(content) > PER_FILE_CHARS:
|
| 242 |
+
content = content[:PER_FILE_CHARS]
|
| 243 |
+
cut = ("\n\n[This file is longer than what fits in one pass. The text above is "
|
| 244 |
+
"its opening; say so in the notes rather than implying the file ends here.]")
|
| 245 |
return (
|
| 246 |
f'Build request: "{prompt}"\n\n'
|
| 247 |
f"File: {name} (type: {category})\n"
|
| 248 |
+
f"--- file content ---\n{content}{cut}\n--- end ---\n\n"
|
| 249 |
"Extract this file's content AND grade its relevance. Return the strict JSON "
|
| 250 |
"described in the system message."
|
| 251 |
)
|
|
@@ -640,3 +640,69 @@ def test_every_category_the_space_can_return_has_a_label_in_the_wizard():
|
|
| 640 |
labelled = set(re.findall(r"([a-z]+)\s*:\s*\[", cat.group(1)))
|
| 641 |
used = set(extract._EXT.values()) | {"other"}
|
| 642 |
assert not (used - labelled), f"no label for: {sorted(used - labelled)}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 640 |
labelled = set(re.findall(r"([a-z]+)\s*:\s*\[", cat.group(1)))
|
| 641 |
used = set(extract._EXT.values()) | {"other"}
|
| 642 |
assert not (used - labelled), f"no label for: {sorted(used - labelled)}"
|
| 643 |
+
|
| 644 |
+
|
| 645 |
+
# --- subtitles from the container ----------------------------------------------
|
| 646 |
+
|
| 647 |
+
def test_rolling_captions_are_not_repeated_into_the_transcript():
|
| 648 |
+
"""A scrolling caption track repeats the same line across cues as it rolls, and
|
| 649 |
+
every copy would have been paid for in tokens."""
|
| 650 |
+
from subtitles import _parse_srt
|
| 651 |
+
srt = ("1\n00:00:01,000 --> 00:00:03,000\nDzien dobry.\n\n"
|
| 652 |
+
"2\n00:00:03,000 --> 00:00:05,000\nDzien dobry.\n\n"
|
| 653 |
+
"3\n00:00:05,000 --> 00:00:07,000\nZaczynamy.\n")
|
| 654 |
+
assert _parse_srt(srt) == ["Dzien dobry.", "Zaczynamy."]
|
| 655 |
+
|
| 656 |
+
|
| 657 |
+
def test_a_line_that_comes_back_later_is_kept():
|
| 658 |
+
"""Only CONSECUTIVE repeats are the rolling artefact; a refrain is content."""
|
| 659 |
+
from subtitles import _parse_srt
|
| 660 |
+
srt = ("1\n00:00:01,000 --> 00:00:03,000\nRefren.\n\n"
|
| 661 |
+
"2\n00:00:03,000 --> 00:00:05,000\nZwrotka.\n\n"
|
| 662 |
+
"3\n00:00:05,000 --> 00:00:07,000\nRefren.\n")
|
| 663 |
+
assert _parse_srt(srt) == ["Refren.", "Zwrotka.", "Refren."]
|
| 664 |
+
|
| 665 |
+
|
| 666 |
+
def test_subtitles_stop_at_the_same_point_whisper_does():
|
| 667 |
+
"""A three hour film used to hand over its whole script, while Whisper stopped at
|
| 668 |
+
thirty minutes: the same job obeyed a different rule depending only on how the
|
| 669 |
+
words were obtained."""
|
| 670 |
+
from subtitles import _parse_srt
|
| 671 |
+
from config import TRANSCRIBE_MAX_SECONDS
|
| 672 |
+
late = TRANSCRIBE_MAX_SECONDS + 120
|
| 673 |
+
h, rest = divmod(int(late), 3600)
|
| 674 |
+
m, sec = divmod(rest, 60)
|
| 675 |
+
srt = ("1\n00:00:01,000 --> 00:00:03,000\nNa poczatku.\n\n"
|
| 676 |
+
f"2\n{h:02d}:{m:02d}:{sec:02d},000 --> {h:02d}:{m:02d}:{sec + 2:02d},000\nPo limicie.\n")
|
| 677 |
+
out = _parse_srt(srt)
|
| 678 |
+
assert out == ["Na poczatku."]
|
| 679 |
+
|
| 680 |
+
|
| 681 |
+
def test_styling_and_override_blocks_are_stripped():
|
| 682 |
+
from subtitles import _parse_srt
|
| 683 |
+
srt = ("1\n00:00:01,000 --> 00:00:03,000\n<i>Kursywa</i> i {\\an8}pozycja\n")
|
| 684 |
+
assert _parse_srt(srt) == ["Kursywa i pozycja"]
|
| 685 |
+
|
| 686 |
+
|
| 687 |
+
def test_a_cue_time_is_read_with_its_fraction():
|
| 688 |
+
from subtitles import _SRT_TIME, _start_seconds
|
| 689 |
+
assert _start_seconds(_SRT_TIME.search("00:01:02,500 --> 00:01:05,000")) == 62.5
|
| 690 |
+
assert _start_seconds(_SRT_TIME.search("01:00:00.250 --> 01:00:02,000")) == 3600.25
|
| 691 |
+
|
| 692 |
+
|
| 693 |
+
# --- nothing may reach the model uncapped --------------------------------------
|
| 694 |
+
|
| 695 |
+
def test_an_oversized_transcript_is_cut_before_it_reaches_the_model():
|
| 696 |
+
"""A transcript goes straight from Whisper or the subtitle track into map_text,
|
| 697 |
+
never through extract(), so it never met the per-file cap."""
|
| 698 |
+
from synthesis import _map_user
|
| 699 |
+
from config import PER_FILE_CHARS
|
| 700 |
+
out = _map_user("p", "film.srt", "video", "x" * (PER_FILE_CHARS + 90000))
|
| 701 |
+
assert len(out) < PER_FILE_CHARS + 900
|
| 702 |
+
assert "longer than" in out # and the model is told, so it cannot pretend
|
| 703 |
+
|
| 704 |
+
|
| 705 |
+
def test_a_normal_file_is_passed_through_untouched():
|
| 706 |
+
from synthesis import _map_user
|
| 707 |
+
out = _map_user("p", "a.txt", "text", "krotka tresc pliku")
|
| 708 |
+
assert "krotka tresc pliku" in out and "longer than" not in out
|