Spaces:
Paused
Paused
Malina Claude Opus 4.8 commited on
Commit ·
d70b247
1
Parent(s): 61c7718
LUNA: binary PL/EN language detection + forceful directive (no topic-language drift)
Browse files- synthesis.py +18 -21
synthesis.py
CHANGED
|
@@ -28,10 +28,17 @@ _PL_HINTS = frozenset({
|
|
| 28 |
"wszystkich", "pliki", "plik", "calosc", "całość", "ogolny", "ogólny", "przeglad",
|
| 29 |
"przegląd", "stworz", "stwórz", "zbuduj", "napisz", "raport", "notatka", "notatke",
|
| 30 |
"temat", "dokumenty", "dokument", "czego", "oraz", "dla", "jak", "czy",
|
|
|
|
|
|
|
| 31 |
})
|
| 32 |
|
| 33 |
|
| 34 |
def detect_language(text: str) -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
t = (text or "").strip()
|
| 36 |
if not t:
|
| 37 |
return "English"
|
|
@@ -40,18 +47,6 @@ def detect_language(text: str) -> str:
|
|
| 40 |
words = re.findall(r"[\wąćęłńóśźż]+", t.lower())
|
| 41 |
if any(w in _PL_HINTS for w in words):
|
| 42 |
return "Polish"
|
| 43 |
-
# langdetect is unreliable on very short text (e.g. "Summarize all files" → Italian),
|
| 44 |
-
# so only trust it when the prompt is long enough; otherwise default to English.
|
| 45 |
-
if len(t) >= 25:
|
| 46 |
-
try:
|
| 47 |
-
from langdetect import detect, DetectorFactory
|
| 48 |
-
DetectorFactory.seed = 0
|
| 49 |
-
code = detect(t).lower()
|
| 50 |
-
name = _LANG_NAMES.get(code) or _LANG_NAMES.get(code.split("-")[0])
|
| 51 |
-
if name:
|
| 52 |
-
return name
|
| 53 |
-
except Exception:
|
| 54 |
-
pass
|
| 55 |
return "English"
|
| 56 |
|
| 57 |
|
|
@@ -194,17 +189,18 @@ def map_image(prompt: str, name: str, data_url: str, global_mode: bool = False)
|
|
| 194 |
|
| 195 |
|
| 196 |
# ── REDUCE ──────────────────────────────────────────────────────────────────
|
| 197 |
-
def _reduce_system() -> str:
|
| 198 |
return (
|
| 199 |
"You are LUNA, an expert research synthesizer. From extracted notes drawn from a "
|
| 200 |
"set of source files, you write ONE clean, structured Note that fulfils the user's "
|
| 201 |
"build request.\n\n"
|
| 202 |
-
"LANGUAGE — Write the ENTIRE Note (title, headings, body, Sources — everything)
|
| 203 |
-
"
|
| 204 |
-
"
|
| 205 |
-
"
|
| 206 |
-
"
|
| 207 |
-
"
|
|
|
|
| 208 |
"GROUNDING — Use ONLY the provided notes. Never invent facts, figures or sources. "
|
| 209 |
"If the material is thin or doesn't really cover the request, say so honestly rather "
|
| 210 |
"than padding.\n\n"
|
|
@@ -263,8 +259,9 @@ def reduce_stream(prompt: str, kept: list[dict], skipped: list[dict], fmt: str =
|
|
| 263 |
"""Stream the final Note from the strong model. Yields the accumulated Markdown.
|
| 264 |
The model mirrors the request's language itself (more reliable than langdetect,
|
| 265 |
which misfires on short prompts)."""
|
|
|
|
| 266 |
if DEBUG:
|
| 267 |
-
print(f"[LUNA] REDUCE — {len(kept)} kept, {len(skipped)} skipped")
|
| 268 |
if not kept:
|
| 269 |
yield (
|
| 270 |
f"# {prompt.strip()[:80]}\n\n"
|
|
@@ -283,7 +280,7 @@ def reduce_stream(prompt: str, kept: list[dict], skipped: list[dict], fmt: str =
|
|
| 283 |
stream = llm_client.chat_completion(
|
| 284 |
model=REDUCE_MODEL,
|
| 285 |
messages=[
|
| 286 |
-
{"role": "system", "content": _reduce_system()},
|
| 287 |
{"role": "user", "content": _reduce_user(prompt, kept, skipped, fmt)},
|
| 288 |
],
|
| 289 |
max_tokens=REDUCE_MAX_TOKENS,
|
|
|
|
| 28 |
"wszystkich", "pliki", "plik", "calosc", "całość", "ogolny", "ogólny", "przeglad",
|
| 29 |
"przegląd", "stworz", "stwórz", "zbuduj", "napisz", "raport", "notatka", "notatke",
|
| 30 |
"temat", "dokumenty", "dokument", "czego", "oraz", "dla", "jak", "czy",
|
| 31 |
+
"zrob", "zrób", "wygeneruj", "utworz", "utwórz", "sporzadz", "sporządź", "wypisz",
|
| 32 |
+
"wytłumacz", "wytlumacz", "koloseum", "zdjęcia", "obrazek", "obraz",
|
| 33 |
})
|
| 34 |
|
| 35 |
|
| 36 |
def detect_language(text: str) -> str:
|
| 37 |
+
"""Binary PL/EN detection for this PL/EN product. We deliberately do NOT use
|
| 38 |
+
langdetect or trust the LLM's own guess: both misfire on short prompts (e.g.
|
| 39 |
+
"Summarize all files" → Italian) or drift to the topic's language ("Colosseum"
|
| 40 |
+
→ French). Polish is detected from diacritics or common Polish words; everything
|
| 41 |
+
else defaults to English."""
|
| 42 |
t = (text or "").strip()
|
| 43 |
if not t:
|
| 44 |
return "English"
|
|
|
|
| 47 |
words = re.findall(r"[\wąćęłńóśźż]+", t.lower())
|
| 48 |
if any(w in _PL_HINTS for w in words):
|
| 49 |
return "Polish"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
return "English"
|
| 51 |
|
| 52 |
|
|
|
|
| 189 |
|
| 190 |
|
| 191 |
# ── REDUCE ──────────────────────────────────────────────────────────────────
|
| 192 |
+
def _reduce_system(language: str) -> str:
|
| 193 |
return (
|
| 194 |
"You are LUNA, an expert research synthesizer. From extracted notes drawn from a "
|
| 195 |
"set of source files, you write ONE clean, structured Note that fulfils the user's "
|
| 196 |
"build request.\n\n"
|
| 197 |
+
f"LANGUAGE — Write the ENTIRE Note (title, headings, body, Sources — everything) STRICTLY "
|
| 198 |
+
f"in {language}. Under no circumstances use any other language: NOT the language of the "
|
| 199 |
+
f"source files, NOT the language associated with the topic or with any place/landmark "
|
| 200 |
+
f"named (e.g. do not switch to French/Italian/Spanish just because the subject is "
|
| 201 |
+
f"European), and NOT the language of any formatting notes. If the source material is in "
|
| 202 |
+
f"another language, translate the relevant content into {language}. Keep proper names, "
|
| 203 |
+
f"numbers, brands and direct quotes as they appear.\n\n"
|
| 204 |
"GROUNDING — Use ONLY the provided notes. Never invent facts, figures or sources. "
|
| 205 |
"If the material is thin or doesn't really cover the request, say so honestly rather "
|
| 206 |
"than padding.\n\n"
|
|
|
|
| 259 |
"""Stream the final Note from the strong model. Yields the accumulated Markdown.
|
| 260 |
The model mirrors the request's language itself (more reliable than langdetect,
|
| 261 |
which misfires on short prompts)."""
|
| 262 |
+
language = detect_language(prompt)
|
| 263 |
if DEBUG:
|
| 264 |
+
print(f"[LUNA] REDUCE — {len(kept)} kept, {len(skipped)} skipped, lang={language}")
|
| 265 |
if not kept:
|
| 266 |
yield (
|
| 267 |
f"# {prompt.strip()[:80]}\n\n"
|
|
|
|
| 280 |
stream = llm_client.chat_completion(
|
| 281 |
model=REDUCE_MODEL,
|
| 282 |
messages=[
|
| 283 |
+
{"role": "system", "content": _reduce_system(language)},
|
| 284 |
{"role": "user", "content": _reduce_user(prompt, kept, skipped, fmt)},
|
| 285 |
],
|
| 286 |
max_tokens=REDUCE_MAX_TOKENS,
|