File size: 2,360 Bytes
80300e5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
import re
from .tokenizer import normalize_text
from .infer import generate, bad_text

FALLBACK='ใ†ใพใ็ญ”ใˆใ‚‰ใ‚Œใพใ›ใ‚“ใงใ—ใŸใ€‚่ณชๅ•ใ‚’็Ÿญใ่จ€ใ„ๆ›ใˆใฆใใ ใ•ใ„ใ€‚'
TOO_LONG='่ณชๅ•ใŒ้•ทใ™ใŽใพใ™ใ€‚็Ÿญใๅˆ†ใ‘ใฆๅ…ฅๅŠ›ใ—ใฆใใ ใ•ใ„ใ€‚'

# Small deterministic tool path. This is algorithmic, not a table of fixed answers.
_CALC_PATTERNS=[
    (re.compile(r'^\s*([+-]?\d{1,5})\s*([+\-*ร—])\s*([+-]?\d{1,5})\s*(?:ใฏ|=)?\s*[?๏ผŸ]?\s*$'), None),
    (re.compile(r'^\s*([+-]?\d{1,5})\s*(ใŸใ™|ใฒใ|ใ‹ใ‘ใ‚‹)\s*([+-]?\d{1,5})\s*(?:ใจ|ใฏ)?\s*[?๏ผŸ]?\s*$'), None),
]

def try_calculate(prompt:str):
    s=normalize_text(prompt)
    for pat,_ in _CALC_PATTERNS:
        m=pat.fullmatch(s)
        if not m: continue
        a=int(m.group(1)); op=m.group(2); b=int(m.group(3))
        if op in ('+','ใŸใ™'): r=a+b
        elif op in ('-','ใฒใ'): r=a-b
        elif op in ('*','ร—','ใ‹ใ‘ใ‚‹'): r=a*b
        else: return None
        # keep device-side integer formatting simple and bounded
        if not (-2147483648 <= r <= 2147483647): return '่จˆ็ฎ—็ตๆžœใŒๆ‰ฑใˆใ‚‹็ฏ„ๅ›ฒใ‚’่ถ…ใˆใฆใ„ใพใ™ใ€‚'
        return f'{r}ใงใ™ใ€‚'
    return None

def _clean_output(text:str):
    text=normalize_text(text)
    if bad_text(text): return FALLBACK
    try:
        text.encode('utf-8','strict').decode('utf-8','strict')
    except Exception:
        return FALLBACK
    # If a generation ran long and contains a complete Japanese sentence, keep the complete prefix.
    if len(text)>72 and 'ใ€‚' in text:
        text=text[:text.rfind('ใ€‚')+1]
    # Long non-terminated fragments are safer as a fallback than as broken prose.
    if len(text)>24 and text[-1] not in 'ใ€‚๏ผ๏ผŸ?!':
        cut=max(text.rfind('ใ€‚'),text.rfind('๏ผ'),text.rfind('๏ผŸ'))
        if cut>=6: text=text[:cut+1]
        else: return FALLBACK
    return text or FALLBACK

def answer(model,tok,prompt,max_new=64,temperature=0.0,top_k=8):
    s=normalize_text(prompt)
    if not s: return '่ณชๅ•ใ‚’ๅ…ฅๅŠ›ใ—ใฆใใ ใ•ใ„ใ€‚'
    # Unicode character count limit; byte-fallback may use more tokens internally.
    if len(s)>160: return TOO_LONG
    calc=try_calculate(s)
    if calc is not None: return calc
    text,_=generate(model,tok,s,max_new=max_new,temperature=temperature,top_k=top_k,confidence_fallback=True)
    return _clean_output(text)