File size: 2,954 Bytes
3932148 80cecae 13015af 80cecae 3932148 80cecae 3932148 80cecae 3932148 80cecae 3932148 80cecae 13015af 80cecae 3932148 80cecae 3932148 1038b14 80cecae 1038b14 3932148 80cecae 3932148 80cecae | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | # utils.py
# Core helper tools for character scripting, text tokenization estimates,
# model complexity routing, and speech-delivery formatting overrides.
import re
from typing import List, Dict, Any
from language_config import (
classify_arabic_script,
detect_roman_urdu,
DEVANAGARI,
GURMUKHI,
has_arabic_script,
normalize_language,
SUPPORTED_LANGUAGES,
URDU_SHARED_PHRASES,
)
def needs70B(prompt: str, detected_language: str) -> bool:
"""
Determine if the prompt requires the 70B model based on complexity.
Routes low-tier requests to 8B models to preserve real-time call performance.
"""
complex_keywords = [
"analyze", "calculate", "reasoning", "explain step by step",
"complex", "complain", "refund", "manager",
]
requires_complex_reasoning = any(keyword in prompt.lower() for keyword in complex_keywords)
return requires_complex_reasoning or normalize_language(detected_language) == "ur"
def detect_language_from_content(text: str) -> str:
"""
Detect language from text using script rules. Returns only 'en' or 'ur'.
Hindi (Devanagari) and Punjabi (Gurmukhi) are rerouted to 'ur'.
"""
if not text or not text.strip():
return "en"
if DEVANAGARI.search(text) or GURMUKHI.search(text):
return "ur"
if has_arabic_script(text):
return "ur"
if detect_roman_urdu(text):
return "ur"
text_lower = text.lower()
for phrase in URDU_SHARED_PHRASES:
if phrase.lower() in text_lower:
return "ur"
return "en"
def get_arabic_clarification_flag(text: str) -> bool:
"""Return True when Arabic script input needs a clarification prompt."""
return classify_arabic_script(text) == "needs_clarification"
def estimate_tokens_from_text(text: str) -> int:
"""Rough token estimator based on character count splits."""
return max(1, (len(text) + 3) // 4)
def estimate_payload_tokens(payload: List[Dict[str, Any]]) -> int:
"""Aggregate token runtime footings for structural context bounds."""
total = 0
for p in payload:
try:
total += estimate_tokens_from_text(str(p.get("content", "")))
except Exception:
pass
return total
def strip_formatting(text: str) -> str:
"""
Remove code fences, inline blocks, markdown structures, headers,
parenthetical bracket groups, and extra spacing elements to format
raw strings cleanly for TTS processing.
"""
if not text:
return text
patterns = [
r"```[\s\S]*?```",
r"`[^`]*`",
r"!\[[^\]]*\]\([^\)]*\)",
r"\[[^\]]*\]\([^\)]*\)",
r"\(.*?\)",
r"(^|\n)#{1,6}\s+",
r"\*\*([^*]+)\*\*",
r"\*([^*]+)\*",
r"__([^_]+)__",
r"_([^_]+)_",
r"\s{2,}",
]
for pat in patterns:
text = re.sub(pat, " ", text, flags=re.MULTILINE)
return text.strip()
|