Download tetu_code/serve.py from tetu-AI/Tetu-Code-2: direct link, hf CLI and curl.
- Browser
- Download file 23.7 kB
-
https://huggingface.co/tetu-AI/Tetu-Code-2/resolve/main/tetu_code/serve.py
- Command line
-
hf download hf://tetu-AI/Tetu-Code-2/tetu_code/serve.py
-
curl -L -o serve.py https://huggingface.co/tetu-AI/Tetu-Code-2/resolve/main/tetu_code/serve.py
23.7 kB
| # Têtu Code 2 — inference code (tetu-AI). License: Apache 2.0, see README.md. | |
| import argparse | |
| import collections | |
| import hashlib | |
| import ntpath | |
| import os | |
| import posixpath | |
| from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | |
| import json | |
| import re | |
| import threading | |
| import time | |
| import uuid | |
| from tetu_code import fmt | |
| def project_instructions(system): | |
| out = [] | |
| for m in re.finditer('Instructions from: ([^\\n]+)\\n(.*?)(?=\\nInstructions from: |\\n<available_skills>|\\Z)', system, re.S): | |
| out.append(f'Instructions ({m.group(1).strip()}) :\n{m.group(2).strip()[:4000]}') | |
| return '\n\n'.join(out) | |
| def split_think(s): | |
| if not s.startswith('<|think|>'): | |
| return ('', s) | |
| inner, sep, rest = s[len('<|think|>'):].partition('<|/think|>') | |
| return (inner, rest if sep else '') | |
| def last_call(body): | |
| for m in reversed(body.get('messages', [])): | |
| if m.get('role') == 'assistant' and m.get('tool_calls'): | |
| f = m['tool_calls'][-1].get('function', {}) | |
| try: | |
| return (f.get('name'), json.loads(f.get('arguments') or '{}')) | |
| except ValueError: | |
| return None | |
| return None | |
| def degenerate(toks, max_period=256): | |
| n = len(toks) | |
| for p in range(1, max_period + 1): | |
| L = max(4 * p, 160) | |
| if L > n: | |
| break | |
| if toks[-1] == toks[-1 - p] and toks[n - L + p:] == toks[n - L:n - p]: | |
| return n - L + p | |
| return 0 | |
| def recent_calls(body, k=4): | |
| out = [] | |
| for m in reversed(body.get('messages', [])): | |
| for c in reversed(m.get('tool_calls') or []) if m.get('role') == 'assistant' else []: | |
| f = c.get('function', {}) | |
| try: | |
| out.append((f.get('name'), json.loads(f.get('arguments') or '{}'))) | |
| except ValueError: | |
| pass | |
| if len(out) >= k: | |
| return out | |
| return out | |
| def core_command(cmd): | |
| c = re.sub('\\s*(\\|\\||;|&&)\\s*(true|echo\\b[^|;&]*|:)\\s*', ' ', cmd or '') | |
| c = re.sub('\\s*\\d?>&\\d|\\s*2>\\s*/dev/null', ' ', c) | |
| return ' '.join(c.split()) | |
| CODE_START = re.compile('\\s*(?:from \\w[\\w.]* import|import \\w|def \\w|class \\w|#!/usr/bin/env python|if __name__)') | |
| def misplaced(call, body): | |
| if call is None or call[0] not in ('write', 'edit'): | |
| return False | |
| path = str(call[1].get('filePath') or '') | |
| text = str(call[1].get('content') if call[0] == 'write' else call[1].get('newString') or '') | |
| if not re.search('\\.(md|txt|rst)$', path, re.I) or not CODE_START.match(text): | |
| return False | |
| users = ' '.join((fmt.content_text(m.get('content')) for m in body.get('messages', []) if m.get('role') == 'user')) | |
| return re.split('[\\\\/]', path)[-1].lower() not in users.lower() | |
| def workdir(body): | |
| msgs = body.get('messages', []) | |
| return fmt.env_from_system('\n'.join((fmt.content_text(m.get('content')) for m in msgs if m.get('role') == 'system'))).get('Working directory', '') | |
| def rebase(call, body): | |
| wd = workdir(body) | |
| if call is None or not wd: | |
| return call | |
| pm = ntpath if re.match('[A-Za-z]:[\\\\/]', wd) else posixpath | |
| root = pm.normpath(wd) | |
| name, args = call | |
| new = dict(args) | |
| for k in ('filePath', 'path'): | |
| p = args.get(k) | |
| if not isinstance(p, str) or not pm.isabs(p): | |
| continue | |
| q = pm.normpath(p) | |
| if pm.normcase(q) == pm.normcase(root) or pm.normcase(q).startswith(pm.normcase(root) + pm.sep): | |
| continue | |
| anc = root | |
| for _ in range(2): | |
| if pm.dirname(anc) == anc: | |
| break | |
| anc = pm.dirname(anc) | |
| if pm.normcase(q) == pm.normcase(anc): | |
| new[k] = wd | |
| break | |
| if pm.normcase(q).startswith(pm.normcase(anc).rstrip(pm.sep) + pm.sep): | |
| if not os.path.exists(p): | |
| new[k] = pm.join(root, q[len(anc.rstrip(pm.sep)) + 1:]) | |
| break | |
| return call if new == args else (name, new) | |
| def missing_file(call, body): | |
| if call is None or call[0] not in ('read', 'edit'): | |
| return False | |
| path = str(call[1].get('filePath') or '') | |
| wd = workdir(body) | |
| if not path or not wd or (not os.path.isdir(wd)): | |
| return False | |
| return not os.path.exists(path if os.path.isabs(path) else os.path.join(wd, path)) | |
| def looping(call, prev, recent=()): | |
| if call is None: | |
| return False | |
| name, args = call | |
| if name == 'edit' and args.get('oldString') == args.get('newString'): | |
| return True | |
| if prev is not None and (name, args) == tuple(prev): | |
| return True | |
| if name == 'bash': | |
| core = core_command(args.get('command')) | |
| return sum((n == 'bash' and core_command(a.get('command')) == core for n, a in recent)) >= 2 | |
| return False | |
| TASK_HINT = re.compile('(?i)\\b(?:cr[ée]{1,2}r?|cr[ée]{1,2}s|[ée]cri[srt]?|[ée]crire|fai[st]|fais-moi|ajout\\w*|modifi\\w*|corrig\\w*|renomm\\w*|lance\\w*|teste[rz]?|code[rz]?|programm\\w*|g[ée]n[èe]re\\w*|supprim\\w*|enl[èe]v\\w*|chang\\w*|remplac\\w*|mets|met|install\\w*|r[ée]par\\w*|impl[ée]ment\\w*|transform\\w*|traduis|convertis|create|write|make|build|add|edit|fix|rename|run|implement|refactor|delete|remove|site|page|script|programme|appli\\w*|app|jeu|outil|todo|calculatrice|convertisseur|compteur|classe|fonction|fichier|projet|api|serveur|bot|formulaire|bouton|menu|readme|tests|html|css|python|javascript|js|flask)\\b|\\w\\.(?:py|js|ts|html|css|json|md|txt|sh|java|go|rs|c|cpp)\\b') | |
| DANGER = re.compile('(?i)bombes?\\b(?! de bain)|explosi|\\barmes?\\b|keylogger|espion|pirat|hack|phishing|fausse page de connexion|vol(?:er)? (?:des|les) (?:mots de passe|identifiants)|porn|sex|prompt syst|system prompt|ignore (?:tes|les) consignes') | |
| NO_TOOL_HINT = re.compile("(?i)^\\W*(?:explique|c'est quoi|qu'est[- ]ce|quel(?:le)?s?\\b|pourquoi|comment\\b|quelle est la diff|diff[ée]rence|r[ée]sume|d[ée]finis|que (?:veut|signifie)|est-ce que|peux-tu m'expliquer|tu peux m'expliquer)|\\b(?:donne|affiche|montre|r[ée]ponds|envoie)(?:-moi)? (?:uniquement |seulement |juste |directement )?(?:avec )?(?:le |du |que le )?code\\b|\\b(?:uniquement|seulement|juste) le code\\b|\\bsans (?:outils?|fichiers?|cr[ée]er de fichier)|\\bdans (?:ta|la) r[ée]ponse\\b|\\br[ée]ponds en\\b|\\ben \\d+ phrases?\\b|\\ben (?:une|deux|trois) phrases?\\b") | |
| FILE_NAME = re.compile('\\w\\.(?:py|js|ts|html|css|json|md|txt|sh|java|go|rs|c|cpp)\\b') | |
| def wants_tools(text): | |
| text = (text or '').strip() | |
| if not text or DANGER.search(text): | |
| return False | |
| if NO_TOOL_HINT.search(text) and (not FILE_NAME.search(text)): | |
| return False | |
| words = len(text.split()) | |
| return bool(TASK_HINT.search(text)) and (words >= 3 or bool(re.search('\\w\\.\\w{1,4}\\b', text))) | |
| def turn_calls(body): | |
| out = [] | |
| for m in body.get('messages', []): | |
| if m.get('role') == 'user': | |
| out = [] | |
| for c in m.get('tool_calls') or [] if m.get('role') == 'assistant' else []: | |
| f = c.get('function', {}) | |
| try: | |
| out.append((f.get('name'), json.loads(f.get('arguments') or '{}'))) | |
| except ValueError: | |
| out.append((f.get('name'), {})) | |
| return out | |
| CITED = re.compile('(?<![\\w/\\\\.-])((?:[\\w-]+[/\\\\])*[\\w-]+\\.(?:py|js|ts|html|css|json|md|txt|sh|csv))\\b') | |
| TESTED = re.compile("(?i)\\bj'ai (?:bien )?(?:test[ée]|v[ée]rifi[ée]|lanc[ée]|ex[ée]cut[ée])|\\btests? (?:passent|r[ée]ussi|pass(?:ed|ing)?\\b)|\\btests (?:sont|ont) (?:ok|verts|r[ée]ussi|pass[ée])|\\btest[ée]e?s? et (?:[çc]a|il|elle|tout) (?:marche|fonctionne)|\\bi (?:have )?(?:tested|ran|run|verified)\\b|\\ball tests pass") | |
| def dishonest(text, body): | |
| calls = turn_calls(body) | |
| if not calls or not (text or '').strip(): | |
| return '' | |
| if TESTED.search(text) and (not any((n == 'bash' for n, _ in calls))): | |
| return 'dit avoir testé sans commande' | |
| seen = [] | |
| for m in body.get('messages', []): | |
| if m.get('role') != 'system': | |
| seen.append(fmt.content_text(m.get('content'))) | |
| for c in m.get('tool_calls') or []: | |
| seen.append(str(c.get('function', {}).get('arguments') or '')) | |
| seen = '\n'.join(seen).lower().replace('\\\\', '/').replace('\\', '/') | |
| wd = workdir(body) | |
| local = bool(wd) and os.path.isdir(wd) | |
| for name in CITED.findall(text): | |
| if re.split('[/\\\\]', name)[-1].lower() in seen: | |
| continue | |
| if local and os.path.exists(os.path.join(wd, name)): | |
| continue | |
| return f'cite {name}, absent du projet' | |
| return '' | |
| def last_user(body): | |
| msgs = body.get('messages', []) | |
| return fmt.content_text(msgs[-1].get('content')) if msgs and msgs[-1].get('role') == 'user' else '' | |
| def finish_call(text, tools): | |
| if '<|call|>' not in text: | |
| return (text.strip(), None) | |
| before, _, call = text.partition('<|call|>') | |
| while call.startswith('<|call|>'): | |
| call = call[len('<|call|>'):] | |
| name, args = fmt.parse_call(call, tools) | |
| names = {t['function']['name'] for t in tools or []} | |
| if names and name not in names: | |
| vals = ' '.join((str(v) for v in args.values())) | |
| return (' '.join((x for x in (before.strip(), name, vals) if x)).strip(), None) | |
| return (before.strip(), (name, args)) | |
| def is_title_request(body): | |
| msgs = body.get('messages', []) | |
| sys_ = fmt.content_text(msgs[0].get('content')) if msgs and msgs[0].get('role') == 'system' else '' | |
| return not body.get('tools') and 'title' in sys_.lower() and ('generate' in sys_.lower()) | |
| def title_of(body): | |
| users = [fmt.content_text(m.get('content')) for m in body.get('messages', []) if m.get('role') == 'user'] | |
| text = users[-1] if users else '' | |
| text = re.sub('^\\s*Generate a title for this conversation:?\\s*', '', text, flags=re.I).strip().strip('"') | |
| words = re.sub('[@`*#>]', '', text).split() | |
| t = ' '.join(words[:6]) or 'Nouvelle session' | |
| return t[:1].upper() + t[1:] | |
| class Engine: | |
| def __init__(self, a): | |
| import torch | |
| from tetu_code.generate import Decoder, load | |
| dev = a.device if a.device != 'cuda' or torch.cuda.is_available() else 'cpu' | |
| self.model, self.tok = load(a.model, device=dev) | |
| memo = None | |
| if getattr(a, 'memo', ''): | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lab')) | |
| from tc_cles_exact import load_memo | |
| memo = load_memo(a.memo, dev) | |
| self.dec = Decoder(self.model, self.tok, max_len=a.max_len, graph=dev == 'cuda', compile=a.compile and dev == 'cuda', memo=memo) | |
| self.lock = threading.Lock() | |
| self.a = a | |
| self.stop_ids = {self.tok.token_to_id(t) for t in ('<|end|>', '<|/call|>', '<|eos|>')} | |
| self.call_id = self.tok.token_to_id('<|call|>') | |
| self.thoughts = collections.OrderedDict() | |
| d = os.environ.get('TETU_REP_LOG') or ('rep-log' if os.path.isdir('rep-log') else '') | |
| self.rep_log = open(os.path.join(d, f'serve-{a.port}.jsonl'), 'a', encoding='utf-8') if d else None | |
| def prompt_ids(self, body): | |
| msgs = body.get('messages', []) | |
| system = '\n'.join((fmt.content_text(m.get('content')) for m in msgs if m.get('role') == 'system')) | |
| env = fmt.env_from_system(system) | |
| sys_text = fmt.system_text(env) | |
| extra = project_instructions(system) | |
| if extra: | |
| sys_text += '\n\n' + extra | |
| tools = fmt.keep_tools(body.get('tools')) | |
| msgs = [self.with_thought(m) for m in msgs if m.get('role') != 'system'] | |
| text = fmt.text_of(fmt.render(msgs, tools, system=sys_text)) | |
| ids = self.tok.encode(text + '<|assistant|>', add_special_tokens=False).ids | |
| budget = self.a.max_len - self.a.max_new - 16 | |
| if len(ids) > budget: | |
| head = ids[:2048] | |
| ids = head + ids[-(budget - len(head)):] | |
| return (ids, tools) | |
| def thought_key(m): | |
| calls = m.get('tool_calls') or [] | |
| if calls and calls[0].get('id'): | |
| return calls[0]['id'] | |
| return 'txt:' + hashlib.md5(fmt.content_text(m.get('content')).strip().encode()).hexdigest() | |
| def with_thought(self, m): | |
| if m.get('role') != 'assistant' or m.get('reasoning') or m.get('reasoning_content'): | |
| return m | |
| t = self.thoughts.get(self.thought_key(m)) | |
| return dict(m, reasoning=t) if t else m | |
| def remember(self, key, think): | |
| if think: | |
| self.thoughts[key] = think | |
| while len(self.thoughts) > 5000: | |
| self.thoughts.popitem(last=False) | |
| def generate(self, body, on_text=None, on_think=None, info=None): | |
| tools_m, back = fmt.dialect(body.get('tools')) | |
| body = dict(body, tools=tools_m, messages=fmt.to_model(body.get('messages', []), back)) | |
| calls = sum((len(m.get('tool_calls') or []) for m in body.get('messages', []) if m.get('role') == 'assistant')) | |
| if self.a.max_calls and calls >= self.a.max_calls: | |
| text = "Je m'arrête ici : trop d'étapes pour cette demande. Dis-moi comment continuer." | |
| if on_text: | |
| on_text(text) | |
| return (text, None, 0, '') | |
| ids, tools = self.prompt_ids(body) | |
| if info is not None: | |
| info['prompt'] = len(ids) | |
| force = body.get('tool_choice') == 'required' or isinstance(body.get('tool_choice'), dict) | |
| mode = getattr(self.a, 'force', 'smart') | |
| if force and mode == 'smart': | |
| force = wants_tools(last_user(body)) | |
| elif mode == 'never': | |
| force = False | |
| temp = float(body.get('temperature', self.a.temp) if self.a.respect_temp else self.a.temp) | |
| max_new = min(int(body.get('max_tokens') or self.a.max_new), self.a.max_new) | |
| prev, recent, total = (last_call(body), recent_calls(body), 0) | |
| hold = bool(getattr(self.a, 'garde', 1)) and bool(turn_calls(body)) | |
| with self.lock: | |
| for k in range(1 + self.a.retries): | |
| t_k = temp if k == 0 else max(temp, (0.7, 1.0)[min(k, 2) - 1]) | |
| text, call, n, rep, think = self._sample(ids, tools, max_new, t_k, k, on_text if k == 0 and (not hold) else None, on_think if k == 0 else None, force=force) | |
| fixed = rebase(call, body) | |
| if fixed is not call and self.a.verbose: | |
| print(f' path moved back into the project: {call[1]} → {fixed[1]}'[:300], flush=True) | |
| call = fixed | |
| total += n | |
| if info is not None: | |
| info['last'] = n | |
| lie = dishonest(text, body) if hold and call is None and (k < self.a.retries) else '' | |
| if not lie and (not rep) and (not looping(call, prev, recent)) and (not (k < self.a.retries and (misplaced(call, body) or missing_file(call, body)))): | |
| break | |
| if self.a.verbose: | |
| why = f'anti-mensonge : {lie}' if lie else 'anti-boucle : ' + ('répétition en boucle' if rep else f'{call[0]} inutile') | |
| print(f' {why}, resampling (temperature {t_k:.1f})', flush=True) | |
| if hold and on_text and text: | |
| on_text(text) | |
| return (text, fmt.to_client(call, back), total, think) | |
| def _sample(self, ids, tools, max_new, temp, seed, on_text, on_think=None, force=False): | |
| out, shown, shown_th, in_call, n, rep = ([], '', '', False, 0, False) | |
| if force: | |
| ids, out, in_call = (list(ids) + [self.call_id], [self.call_id], True) | |
| for t in self.dec.generate(ids, max_new=max_new, temp=temp, top_p=self.a.top_p, stop=self.stop_ids, seed=seed): | |
| n += 1 | |
| if t in self.stop_ids: | |
| break | |
| out.append(t) | |
| if t == self.call_id: | |
| in_call = True | |
| if n % 32 == 0: | |
| keep = degenerate(out) | |
| if keep: | |
| if self.rep_log: | |
| self.rep_log.write(json.dumps({'ids': list(ids), 'gen': list(out), 'keep': keep, 'temp': temp}) + '\n') | |
| self.rep_log.flush() | |
| del out[keep:] | |
| rep = True | |
| break | |
| if (on_text or on_think) and (not in_call): | |
| s = self.tok.decode(out, skip_special_tokens=False) | |
| if not s.endswith('�'): | |
| th, vis = split_think(s) | |
| if on_think and len(th) > len(shown_th): | |
| on_think(th[len(shown_th):]) | |
| shown_th = th | |
| if on_text and len(vis) > len(shown): | |
| on_text(vis[len(shown):]) | |
| shown = vis | |
| think, text = split_think(self.tok.decode(out, skip_special_tokens=False)) | |
| text, call = finish_call(text, tools) | |
| return (text, call, n, rep, think.strip()) | |
| def make_handler(eng, a): | |
| class H(BaseHTTPRequestHandler): | |
| protocol_version = 'HTTP/1.1' | |
| def log_message(self, *args): | |
| pass | |
| def _json(self, code, obj): | |
| b = json.dumps(obj).encode() | |
| self.send_response(code) | |
| self.send_header('Content-Type', 'application/json') | |
| self.send_header('Content-Length', str(len(b))) | |
| self.end_headers() | |
| self.wfile.write(b) | |
| def do_GET(self): | |
| if self.path.rstrip('/').endswith('/models'): | |
| self._json(200, {'object': 'list', 'data': [{'id': a.name, 'object': 'model', 'owned_by': 'tetu', 'context_length': a.max_len}]}) | |
| else: | |
| self._json(404, {'error': 'not found'}) | |
| def do_POST(self): | |
| n = int(self.headers.get('Content-Length', 0)) | |
| body = json.loads(self.rfile.read(n) or b'{}') | |
| if not self.path.rstrip('/').endswith('/chat/completions'): | |
| self._json(404, {'error': 'not found'}) | |
| return | |
| cid, created = (f'chatcmpl-{uuid.uuid4().hex[:12]}', int(time.time())) | |
| base = {'id': cid, 'created': created, 'model': a.name} | |
| stream = bool(body.get('stream')) | |
| t0, info = (time.time(), {}) | |
| if is_title_request(body): | |
| text, call, ntok, think = (title_of(body), None, 0, '') | |
| chunks = None | |
| elif stream: | |
| self.send_response(200) | |
| self.send_header('Content-Type', 'text/event-stream') | |
| self.send_header('Cache-Control', 'no-cache') | |
| self.end_headers() | |
| def send(obj): | |
| self.wfile.write(f'data: {json.dumps(obj)}\n\n'.encode()) | |
| self.wfile.flush() | |
| send(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {'role': 'assistant', 'content': ''}, 'finish_reason': None}]}) | |
| text, call, ntok, think = eng.generate(body, on_text=lambda s: send(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {'content': s}, 'finish_reason': None}]}), on_think=lambda s: send(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {'reasoning_content': s}, 'finish_reason': None}]}), info=info) | |
| chunks = send | |
| else: | |
| text, call, ntok, think = eng.generate(body, info=info) | |
| chunks = None | |
| prompt, last = (info.get('prompt', 0), info.get('last', ntok)) | |
| usage = {'prompt_tokens': prompt, 'completion_tokens': last, 'total_tokens': prompt + last, 'context_window': a.max_len} | |
| tool_calls = None | |
| if call: | |
| tool_calls = [{'index': 0, 'id': f'call_{uuid.uuid4().hex[:10]}', 'type': 'function', 'function': {'name': call[0], 'arguments': json.dumps(call[1], ensure_ascii=False)}}] | |
| eng.remember(tool_calls[0]['id'] if tool_calls else 'txt:' + hashlib.md5(text.strip().encode()).hexdigest(), think) | |
| fin = 'tool_calls' if call else 'stop' | |
| if a.verbose: | |
| print(f"[{time.strftime('%H:%M:%S')}] {ntok} tokens in {time.time() - t0:.1f} s · {('call ' + call[0] if call else 'answer')}", flush=True) | |
| if chunks is not None: | |
| if tool_calls: | |
| chunks(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {'tool_calls': tool_calls}, 'finish_reason': None}]}) | |
| chunks(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': {}, 'finish_reason': fin}], 'usage': usage}) | |
| self.wfile.write(b'data: [DONE]\n\n') | |
| self.wfile.flush() | |
| self.close_connection = True | |
| return | |
| msg = {'role': 'assistant', 'content': text if not call or text else None} | |
| if think: | |
| msg['reasoning_content'] = think | |
| if tool_calls: | |
| msg['tool_calls'] = [{k: v for k, v in tc.items() if k != 'index'} for tc in tool_calls] | |
| if stream: | |
| self.send_response(200) | |
| self.send_header('Content-Type', 'text/event-stream') | |
| self.end_headers() | |
| for c in ({'role': 'assistant', 'content': text}, {}): | |
| self.wfile.write(f"data: {json.dumps(base | {'object': 'chat.completion.chunk', 'choices': [{'index': 0, 'delta': c, 'finish_reason': None if c else 'stop'}]})}\n\n".encode()) | |
| self.wfile.write(b'data: [DONE]\n\n') | |
| self.close_connection = True | |
| return | |
| self._json(200, base | {'object': 'chat.completion', 'usage': usage, 'choices': [{'index': 0, 'message': msg, 'finish_reason': fin}]}) | |
| return H | |
| def main(): | |
| p = argparse.ArgumentParser() | |
| p.add_argument('--model', required=True) | |
| p.add_argument('--name', default='tetu-code') | |
| p.add_argument('--host', default='127.0.0.1') | |
| p.add_argument('--port', type=int, default=8070) | |
| p.add_argument('--device', default='cuda') | |
| p.add_argument('--max_len', type=int, default=65536) | |
| p.add_argument('--max_new', type=int, default=4096) | |
| p.add_argument('--temp', type=float, default=0.2) | |
| p.add_argument('--top_p', type=float, default=0.95) | |
| p.add_argument('--respect_temp', action='store_true', help='use the temperature sent by the client') | |
| p.add_argument('--max_calls', type=int, default=40, help='max tool calls per conversation (0 = no limit)') | |
| p.add_argument('--compile', action='store_true', help='torch.compile the decoding step (+40 %% speed, 2-8 min at startup)') | |
| p.add_argument('--garde', type=int, default=1, help='1: check the final answer (cited file missing, "tested" without a command) and redraw if needed') | |
| p.add_argument('--retries', type=int, default=2, help='resamples when a call loops (0 = never)') | |
| p.add_argument('--force', choices=('smart', 'client', 'never'), default='smart', help='tool_choice "required": smart = only for requests that look like a task') | |
| p.add_argument('--verbose', action='store_true') | |
| p.add_argument('--memo', default='', help='optional add-on, not included in this release') | |
| a = p.parse_args() | |
| eng = Engine(a) | |
| ThreadingHTTPServer.daemon_threads = True | |
| srv = ThreadingHTTPServer((a.host, a.port), make_handler(eng, a)) | |
| print(f'Têtu Code ready: http://{a.host}:{a.port}/v1 (model "{a.name}")', flush=True) | |
| srv.serve_forever() | |
| if __name__ == '__main__': | |
| main() | |