Spaces:
Paused
Paused
Malina
Claude Fable 5.1
v0.90: scoped single-use tokens, hardened link fetch, 45-minute cap, whole-video digest, one classification call, no OCR
1d697a7 Download lang.py from NextNeural/NINA: direct link, hf CLI and curl.
- Browser
- Download file 4.93 kB
-
https://huggingface.co/spaces/NextNeural/NINA/resolve/main/lang.py
- Command line
-
hf download hf://spaces/NextNeural/NINA/lang.py
-
curl -L -o lang.py https://huggingface.co/spaces/NextNeural/NINA/resolve/main/lang.py
4.93 kB
| """Language of the user's prompt. Lives in its own module so it can be imported (and | |
| tested) without the model stack behind synthesis.py.""" | |
| import re | |
| _POLISH_CHARS = frozenset("ąćęłńóśźżĄĆĘŁŃÓŚŹŻ") | |
| _GERMAN_CHARS = frozenset("äöüÄÖÜß") | |
| _FRENCH_CHARS = frozenset("àâæçéèêëîïôœùûüÿÀÂÆÇÉÈÊËÎÏÔŒÙÛÜŸ") | |
| _SPANISH_CHARS = frozenset("áéíóúüñ¡¿ÁÉÍÓÚÜÑ") | |
| # ISO-639-1 → English name, for the languages langdetect distinguishes that matter | |
| # to NINA's users. Anything else falls back to the system prompt's "same language | |
| # as the question" instruction, which Qwen handles well on its own. | |
| _LANG_NAMES = { | |
| "en": "English", "pl": "Polish", "de": "German", "fr": "French", | |
| "es": "Spanish", "it": "Italian", "pt": "Portuguese", "nl": "Dutch", | |
| "ru": "Russian", "uk": "Ukrainian", "cs": "Czech", "sk": "Slovak", | |
| "sv": "Swedish", "no": "Norwegian", "da": "Danish", "fi": "Finnish", | |
| "tr": "Turkish", "ro": "Romanian", "hu": "Hungarian", "el": "Greek", | |
| "bg": "Bulgarian", "hr": "Croatian", "lt": "Lithuanian", "lv": "Latvian", | |
| "et": "Estonian", "sl": "Slovenian", "ja": "Japanese", "ko": "Korean", | |
| "zh": "Chinese", "zh-cn": "Chinese", "zh-tw": "Chinese", "ar": "Arabic", | |
| "he": "Hebrew", "hi": "Hindi", "id": "Indonesian", "vi": "Vietnamese", | |
| "th": "Thai", "ca": "Catalan", | |
| } | |
| # Function words, the signal that survives a Pole typing without diacritics. A closed | |
| # set, so it never needs maintaining, and far more signal per word than topic words. | |
| _PL_WORDS = frozenset({ | |
| "w", "we", "z", "ze", "na", "do", "od", "po", "za", "o", "u", "i", "a", "ale", "oraz", | |
| "nie", "tak", "jest", "sa", "byl", "byla", "bylo", "byc", "ma", "maja", "mam", "mowi", | |
| "to", "ten", "ta", "te", "tego", "tej", "tych", "tym", "temu", "czy", "co", "kto", | |
| "jak", "ile", "gdzie", "kiedy", "dlaczego", "ktory", "ktora", "ktore", "ktorych", | |
| "sie", "sobie", "jego", "jej", "ich", "mnie", "ciebie", "ty", "ja", "wlasciwie", | |
| "dla", "przez", "pod", "nad", "bez", "przy", "wedlug", "miedzy", "wszystko", "razy", | |
| "tylko", "juz", "jeszcze", "bardzo", "wiec", "zeby", "aby", "jesli", "tam", "tutaj", | |
| "film", "filmie", "nagranie", "nagraniu", "wideo", "powiedz", "pokaz", "streszcz", | |
| }) | |
| _EN_WORDS = frozenset({ | |
| "the", "a", "an", "and", "or", "but", "of", "in", "on", "at", "to", "for", "from", | |
| "with", "without", "by", "about", "into", "over", "under", "between", "again", | |
| "is", "are", "was", "were", "be", "been", "being", "has", "have", "had", "does", | |
| "did", "can", "could", "will", "would", "should", "may", "might", "your", "you", | |
| "this", "that", "these", "those", "it", "its", "they", "them", "their", "there", | |
| "what", "which", "who", "whom", "whose", "when", "where", "why", "how", "many", | |
| "all", "any", "some", "every", "each", "only", "just", "also", "not", "no", | |
| "video", "clip", "say", "says", "said", "show", "shows", "tell", "summarize", | |
| }) | |
| # langdetect is confident and wrong on short text. Measured on real follow-ups: | |
| # "co tam robisz" -> Portuguese, "ile lat ma ten prelegent?" -> Catalan. A follow-up in | |
| # a chat IS short, so its verdict is only taken on longer text and only when the | |
| # function words do not already settle it. | |
| _LANGDETECT_MIN_CHARS = 60 | |
| def detect_language(text: str) -> str: | |
| """Best-effort language of the user's prompt. | |
| Order matters. Polish diacritics settle it outright. Then Polish against English on | |
| function words, which is what a Pole typing without accents still gives us. Only a | |
| longer prompt that neither of those settles is handed to langdetect, because on a | |
| short one it answers with a language nobody in this product is writing in, and the | |
| prompt that carries that answer tells the model to reply in it.""" | |
| t = (text or "").strip() | |
| if not t: | |
| return "English" | |
| if any(c in t for c in _POLISH_CHARS): | |
| return "Polish" | |
| words = re.findall(r"[a-ząćęłńóśźż]+", t.lower()) | |
| pl = sum(1 for w in words if w in _PL_WORDS) | |
| en = sum(1 for w in words if w in _EN_WORDS) | |
| if pl != en: | |
| return "Polish" if pl > en else "English" | |
| if len(t) >= _LANGDETECT_MIN_CHARS: | |
| try: | |
| from langdetect import detect, DetectorFactory | |
| DetectorFactory.seed = 0 | |
| code = detect(t).lower() | |
| name = _LANG_NAMES.get(code) or _LANG_NAMES.get(code.split("-")[0]) | |
| if name: | |
| return name | |
| except Exception: | |
| pass | |
| if any(c in t for c in _GERMAN_CHARS): | |
| return "German" | |
| if any(c in t for c in _FRENCH_CHARS): | |
| return "French" | |
| if any(c in t for c in _SPANISH_CHARS): | |
| return "Spanish" | |
| # Polish-only letter sequences English does not produce, then English. | |
| if re.search(r"(cz|sz|rz|dz|ch)\w", " ".join(words)) and not en: | |
| return "Polish" | |
| return "English" | |