File size: 4,927 Bytes
1d697a7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
"""Language of the user's prompt. Lives in its own module so it can be imported (and
tested) without the model stack behind synthesis.py."""
import re


_POLISH_CHARS  = frozenset("ąćęłńóśźżĄĆĘŁŃÓŚŹŻ")
_GERMAN_CHARS  = frozenset("äöüÄÖÜß")
_FRENCH_CHARS  = frozenset("àâæçéèêëîïôœùûüÿÀÂÆÇÉÈÊËÎÏÔŒÙÛÜŸ")
_SPANISH_CHARS = frozenset("áéíóúüñ¡¿ÁÉÍÓÚÜÑ")

# ISO-639-1 → English name, for the languages langdetect distinguishes that matter
# to NINA's users. Anything else falls back to the system prompt's "same language
# as the question" instruction, which Qwen handles well on its own.
_LANG_NAMES = {
    "en": "English", "pl": "Polish", "de": "German", "fr": "French",
    "es": "Spanish", "it": "Italian", "pt": "Portuguese", "nl": "Dutch",
    "ru": "Russian", "uk": "Ukrainian", "cs": "Czech", "sk": "Slovak",
    "sv": "Swedish", "no": "Norwegian", "da": "Danish", "fi": "Finnish",
    "tr": "Turkish", "ro": "Romanian", "hu": "Hungarian", "el": "Greek",
    "bg": "Bulgarian", "hr": "Croatian", "lt": "Lithuanian", "lv": "Latvian",
    "et": "Estonian", "sl": "Slovenian", "ja": "Japanese", "ko": "Korean",
    "zh": "Chinese", "zh-cn": "Chinese", "zh-tw": "Chinese", "ar": "Arabic",
    "he": "Hebrew", "hi": "Hindi", "id": "Indonesian", "vi": "Vietnamese",
    "th": "Thai", "ca": "Catalan",
}


# Function words, the signal that survives a Pole typing without diacritics. A closed
# set, so it never needs maintaining, and far more signal per word than topic words.
_PL_WORDS = frozenset({
    "w", "we", "z", "ze", "na", "do", "od", "po", "za", "o", "u", "i", "a", "ale", "oraz",
    "nie", "tak", "jest", "sa", "byl", "byla", "bylo", "byc", "ma", "maja", "mam", "mowi",
    "to", "ten", "ta", "te", "tego", "tej", "tych", "tym", "temu", "czy", "co", "kto",
    "jak", "ile", "gdzie", "kiedy", "dlaczego", "ktory", "ktora", "ktore", "ktorych",
    "sie", "sobie", "jego", "jej", "ich", "mnie", "ciebie", "ty", "ja", "wlasciwie",
    "dla", "przez", "pod", "nad", "bez", "przy", "wedlug", "miedzy", "wszystko", "razy",
    "tylko", "juz", "jeszcze", "bardzo", "wiec", "zeby", "aby", "jesli", "tam", "tutaj",
    "film", "filmie", "nagranie", "nagraniu", "wideo", "powiedz", "pokaz", "streszcz",
})
_EN_WORDS = frozenset({
    "the", "a", "an", "and", "or", "but", "of", "in", "on", "at", "to", "for", "from",
    "with", "without", "by", "about", "into", "over", "under", "between", "again",
    "is", "are", "was", "were", "be", "been", "being", "has", "have", "had", "does",
    "did", "can", "could", "will", "would", "should", "may", "might", "your", "you",
    "this", "that", "these", "those", "it", "its", "they", "them", "their", "there",
    "what", "which", "who", "whom", "whose", "when", "where", "why", "how", "many",
    "all", "any", "some", "every", "each", "only", "just", "also", "not", "no",
    "video", "clip", "say", "says", "said", "show", "shows", "tell", "summarize",
})
# langdetect is confident and wrong on short text. Measured on real follow-ups:
# "co tam robisz" -> Portuguese, "ile lat ma ten prelegent?" -> Catalan. A follow-up in
# a chat IS short, so its verdict is only taken on longer text and only when the
# function words do not already settle it.
_LANGDETECT_MIN_CHARS = 60


def detect_language(text: str) -> str:
    """Best-effort language of the user's prompt.

    Order matters. Polish diacritics settle it outright. Then Polish against English on
    function words, which is what a Pole typing without accents still gives us. Only a
    longer prompt that neither of those settles is handed to langdetect, because on a
    short one it answers with a language nobody in this product is writing in, and the
    prompt that carries that answer tells the model to reply in it."""
    t = (text or "").strip()
    if not t:
        return "English"
    if any(c in t for c in _POLISH_CHARS):
        return "Polish"

    words = re.findall(r"[a-ząćęłńóśźż]+", t.lower())
    pl = sum(1 for w in words if w in _PL_WORDS)
    en = sum(1 for w in words if w in _EN_WORDS)
    if pl != en:
        return "Polish" if pl > en else "English"

    if len(t) >= _LANGDETECT_MIN_CHARS:
        try:
            from langdetect import detect, DetectorFactory
            DetectorFactory.seed = 0
            code = detect(t).lower()
            name = _LANG_NAMES.get(code) or _LANG_NAMES.get(code.split("-")[0])
            if name:
                return name
        except Exception:
            pass

    if any(c in t for c in _GERMAN_CHARS):
        return "German"
    if any(c in t for c in _FRENCH_CHARS):
        return "French"
    if any(c in t for c in _SPANISH_CHARS):
        return "Spanish"
    # Polish-only letter sequences English does not produce, then English.
    if re.search(r"(cz|sz|rz|dz|ch)\w", " ".join(words)) and not en:
        return "Polish"
    return "English"