Malina Claude Opus 4.8 commited on
Commit
d70b247
·
1 Parent(s): 61c7718

LUNA: binary PL/EN language detection + forceful directive (no topic-language drift)

Browse files
Files changed (1) hide show
  1. synthesis.py +18 -21
synthesis.py CHANGED
@@ -28,10 +28,17 @@ _PL_HINTS = frozenset({
28
  "wszystkich", "pliki", "plik", "calosc", "całość", "ogolny", "ogólny", "przeglad",
29
  "przegląd", "stworz", "stwórz", "zbuduj", "napisz", "raport", "notatka", "notatke",
30
  "temat", "dokumenty", "dokument", "czego", "oraz", "dla", "jak", "czy",
 
 
31
  })
32
 
33
 
34
  def detect_language(text: str) -> str:
 
 
 
 
 
35
  t = (text or "").strip()
36
  if not t:
37
  return "English"
@@ -40,18 +47,6 @@ def detect_language(text: str) -> str:
40
  words = re.findall(r"[\wąćęłńóśźż]+", t.lower())
41
  if any(w in _PL_HINTS for w in words):
42
  return "Polish"
43
- # langdetect is unreliable on very short text (e.g. "Summarize all files" → Italian),
44
- # so only trust it when the prompt is long enough; otherwise default to English.
45
- if len(t) >= 25:
46
- try:
47
- from langdetect import detect, DetectorFactory
48
- DetectorFactory.seed = 0
49
- code = detect(t).lower()
50
- name = _LANG_NAMES.get(code) or _LANG_NAMES.get(code.split("-")[0])
51
- if name:
52
- return name
53
- except Exception:
54
- pass
55
  return "English"
56
 
57
 
@@ -194,17 +189,18 @@ def map_image(prompt: str, name: str, data_url: str, global_mode: bool = False)
194
 
195
 
196
  # ── REDUCE ──────────────────────────────────────────────────────────────────
197
- def _reduce_system() -> str:
198
  return (
199
  "You are LUNA, an expert research synthesizer. From extracted notes drawn from a "
200
  "set of source files, you write ONE clean, structured Note that fulfils the user's "
201
  "build request.\n\n"
202
- "LANGUAGE — Write the ENTIRE Note (title, headings, body, Sources — everything) in the "
203
- "SAME language as the user's build request shown in quotes in the next message. Identify "
204
- "that language yourself from the request text and mirror it exactly. Do NOT switch to the "
205
- "language of the source files, and treat any English formatting/length notes as "
206
- "meta-instructions that do NOT affect the output language. Keep names, numbers, brands, "
207
- "titles and direct quotes exactly as they appear.\n\n"
 
208
  "GROUNDING — Use ONLY the provided notes. Never invent facts, figures or sources. "
209
  "If the material is thin or doesn't really cover the request, say so honestly rather "
210
  "than padding.\n\n"
@@ -263,8 +259,9 @@ def reduce_stream(prompt: str, kept: list[dict], skipped: list[dict], fmt: str =
263
  """Stream the final Note from the strong model. Yields the accumulated Markdown.
264
  The model mirrors the request's language itself (more reliable than langdetect,
265
  which misfires on short prompts)."""
 
266
  if DEBUG:
267
- print(f"[LUNA] REDUCE — {len(kept)} kept, {len(skipped)} skipped")
268
  if not kept:
269
  yield (
270
  f"# {prompt.strip()[:80]}\n\n"
@@ -283,7 +280,7 @@ def reduce_stream(prompt: str, kept: list[dict], skipped: list[dict], fmt: str =
283
  stream = llm_client.chat_completion(
284
  model=REDUCE_MODEL,
285
  messages=[
286
- {"role": "system", "content": _reduce_system()},
287
  {"role": "user", "content": _reduce_user(prompt, kept, skipped, fmt)},
288
  ],
289
  max_tokens=REDUCE_MAX_TOKENS,
 
28
  "wszystkich", "pliki", "plik", "calosc", "całość", "ogolny", "ogólny", "przeglad",
29
  "przegląd", "stworz", "stwórz", "zbuduj", "napisz", "raport", "notatka", "notatke",
30
  "temat", "dokumenty", "dokument", "czego", "oraz", "dla", "jak", "czy",
31
+ "zrob", "zrób", "wygeneruj", "utworz", "utwórz", "sporzadz", "sporządź", "wypisz",
32
+ "wytłumacz", "wytlumacz", "koloseum", "zdjęcia", "obrazek", "obraz",
33
  })
34
 
35
 
36
  def detect_language(text: str) -> str:
37
+ """Binary PL/EN detection for this PL/EN product. We deliberately do NOT use
38
+ langdetect or trust the LLM's own guess: both misfire on short prompts (e.g.
39
+ "Summarize all files" → Italian) or drift to the topic's language ("Colosseum"
40
+ → French). Polish is detected from diacritics or common Polish words; everything
41
+ else defaults to English."""
42
  t = (text or "").strip()
43
  if not t:
44
  return "English"
 
47
  words = re.findall(r"[\wąćęłńóśźż]+", t.lower())
48
  if any(w in _PL_HINTS for w in words):
49
  return "Polish"
 
 
 
 
 
 
 
 
 
 
 
 
50
  return "English"
51
 
52
 
 
189
 
190
 
191
  # ── REDUCE ──────────────────────────────────────────────────────────────────
192
+ def _reduce_system(language: str) -> str:
193
  return (
194
  "You are LUNA, an expert research synthesizer. From extracted notes drawn from a "
195
  "set of source files, you write ONE clean, structured Note that fulfils the user's "
196
  "build request.\n\n"
197
+ f"LANGUAGE — Write the ENTIRE Note (title, headings, body, Sources — everything) STRICTLY "
198
+ f"in {language}. Under no circumstances use any other language: NOT the language of the "
199
+ f"source files, NOT the language associated with the topic or with any place/landmark "
200
+ f"named (e.g. do not switch to French/Italian/Spanish just because the subject is "
201
+ f"European), and NOT the language of any formatting notes. If the source material is in "
202
+ f"another language, translate the relevant content into {language}. Keep proper names, "
203
+ f"numbers, brands and direct quotes as they appear.\n\n"
204
  "GROUNDING — Use ONLY the provided notes. Never invent facts, figures or sources. "
205
  "If the material is thin or doesn't really cover the request, say so honestly rather "
206
  "than padding.\n\n"
 
259
  """Stream the final Note from the strong model. Yields the accumulated Markdown.
260
  The model mirrors the request's language itself (more reliable than langdetect,
261
  which misfires on short prompts)."""
262
+ language = detect_language(prompt)
263
  if DEBUG:
264
+ print(f"[LUNA] REDUCE — {len(kept)} kept, {len(skipped)} skipped, lang={language}")
265
  if not kept:
266
  yield (
267
  f"# {prompt.strip()[:80]}\n\n"
 
280
  stream = llm_client.chat_completion(
281
  model=REDUCE_MODEL,
282
  messages=[
283
+ {"role": "system", "content": _reduce_system(language)},
284
  {"role": "user", "content": _reduce_user(prompt, kept, skipped, fmt)},
285
  ],
286
  max_tokens=REDUCE_MAX_TOKENS,