Download tetu_code/fmt.py from tetu-AI/Tetu-Code-2: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/tetu-AI/Tetu-Code-2/resolve/main/tetu_code/fmt.py
- Command line
-
hf download hf://tetu-AI/Tetu-Code-2/tetu_code/fmt.py
-
curl -L -o fmt.py https://huggingface.co/tetu-AI/Tetu-Code-2/resolve/main/tetu_code/fmt.py
11.6 kB
| # Têtu Code 2 — inference code (tetu-AI). License: Apache 2.0, see README.md. | |
| import json | |
| import re | |
| SYSTEM = "Tu es Têtu Code, un agent de programmation qui travaille dans le terminal de l'utilisateur. Sers-toi des outils pour lire, chercher, modifier et tester le code, un appel à la fois. Lis un fichier avant de le modifier ; après une modification, vérifie (tests, exécution) quand c'est possible. Réponds dans la langue de l'utilisateur, brièvement et précisément." | |
| ENV_KEYS = ('Working directory', 'Is directory a git repo', 'Platform', "Today's date") | |
| TOOL_DESC = {'bash': 'run a shell command', 'read': 'read a file or list a directory', 'edit': 'replace an exact passage of a file (oldString must match exactly once, or set replaceAll)', 'write': 'write a whole file', 'glob': 'find files by name pattern', 'grep': 'search file contents with a regex', 'todowrite': 'update the task list', 'webfetch': 'fetch a web page'} | |
| DROP = {'task', 'skill', 'question', 'todowrite', 'webfetch', 'subagent', 'websearch', 'execute'} | |
| TOOL_ALIASES = {'shell': 'bash'} | |
| ARG_ALIASES = {'read': {'path': 'filePath'}, 'write': {'path': 'filePath'}, 'edit': {'path': 'filePath'}} | |
| OPENCODE_TOOLS = [('bash', [('command', 'string', True), ('timeout', 'integer', False), ('workdir', 'string', False)]), ('edit', [('filePath', 'string', True), ('oldString', 'string', True), ('newString', 'string', True), ('replaceAll', 'boolean', False)]), ('glob', [('pattern', 'string', True), ('path', 'string', False)]), ('grep', [('pattern', 'string', True), ('path', 'string', False), ('include', 'string', False)]), ('read', [('filePath', 'string', True), ('offset', 'integer', False), ('limit', 'integer', False)]), ('todowrite', [('todos', 'array', True)]), ('webfetch', [('url', 'string', True), ('format', 'string', False), ('timeout', 'number', False)]), ('write', [('content', 'string', True), ('filePath', 'string', True)])] | |
| MAX_RESULT = 12000 | |
| def opencode_tools(): | |
| out = [] | |
| for name, params in OPENCODE_TOOLS: | |
| props = {p: {'type': t} for p, t, _ in params} | |
| out.append({'type': 'function', 'function': {'name': name, 'description': TOOL_DESC[name], 'parameters': {'type': 'object', 'properties': props, 'required': [p for p, _, r in params if r]}}}) | |
| return out | |
| def keep_tools(tools): | |
| return [t for t in tools or [] if t.get('function', {}).get('name') not in DROP] | |
| def dialect(tools): | |
| out, back = ([], {}) | |
| for t in tools or []: | |
| f = t.get('function', {}) | |
| cname = f.get('name') | |
| name = TOOL_ALIASES.get(cname, cname) | |
| ps = f.get('parameters') or {} | |
| props = ps.get('properties') or {} | |
| ren = {p: m for p, m in ARG_ALIASES.get(name, {}).items() if p in props and m not in props} | |
| if name != cname or ren: | |
| ps = dict(ps, properties={ren.get(p, p): s for p, s in props.items()}, required=[ren.get(p, p) for p in ps.get('required', [])]) | |
| t = dict(t, function=dict(f, name=name, parameters=ps)) | |
| back[name] = (cname, {m: p for p, m in ren.items()}) | |
| out.append(t) | |
| order = [n for n, _ in OPENCODE_TOOLS] | |
| out.sort(key=lambda t: order.index(t['function']['name']) if t['function']['name'] in order else len(order)) | |
| return (out, back) | |
| def to_model(messages, back): | |
| fwd = {cname: (name, {c: m for m, c in args.items()}) for name, (cname, args) in back.items()} | |
| out = [] | |
| for m in messages: | |
| if m.get('role') == 'assistant' and m.get('tool_calls'): | |
| calls = [] | |
| for c in m['tool_calls']: | |
| f = c.get('function', {}) | |
| name, ren = fwd.get(f.get('name'), (f.get('name'), {})) | |
| f = dict(f, name=name) | |
| if ren: | |
| try: | |
| a = json.loads(f.get('arguments') or '{}') | |
| except ValueError: | |
| a = None | |
| if isinstance(a, dict): | |
| f['arguments'] = json.dumps({ren.get(k, k): v for k, v in a.items()}, ensure_ascii=False) | |
| calls.append(dict(c, function=f)) | |
| m = dict(m, tool_calls=calls) | |
| out.append(m) | |
| return out | |
| def to_client(call, back): | |
| if call is None: | |
| return None | |
| name, args = call | |
| cname, ren = back.get(name, (name, {})) | |
| return (cname, {ren.get(k, k): v for k, v in args.items()}) | |
| def tools_text(tools): | |
| lines = [] | |
| for t in keep_tools(tools): | |
| f = t['function'] | |
| ps = f.get('parameters', {}) or {} | |
| req = set(ps.get('required', [])) | |
| args = ', '.join((p + ('' if p in req else '?') for p in ps.get('properties') or {})) | |
| desc = TOOL_DESC.get(f['name']) or re.split('(?<=[.!?])\\s|\\n', (f.get('description') or '').strip())[0][:120] | |
| lines.append(f"{f['name']}({args}) — {desc}") | |
| return '\n'.join(lines) | |
| def env_from_system(text): | |
| m = re.search('<env>(.*?)</env>', text or '', re.S) | |
| env = {} | |
| for line in (m.group(1) if m else '').splitlines(): | |
| k, _, v = line.strip().partition(':') | |
| if k in ENV_KEYS: | |
| env[k] = v.strip() | |
| return env | |
| def system_text(env=None): | |
| env = env or {} | |
| lines = [f'{k}: {env[k]}' for k in ENV_KEYS if env.get(k)] | |
| return SYSTEM + ('\n<env>\n' + '\n'.join(lines) + '\n</env>' if lines else '') | |
| def arg_text(v): | |
| if isinstance(v, str): | |
| return v | |
| if isinstance(v, bool): | |
| return 'true' if v else 'false' | |
| if isinstance(v, (int, float)): | |
| return str(int(v)) if float(v).is_integer() else str(v) | |
| return json.dumps(v, ensure_ascii=False) | |
| def call_text(name, args): | |
| return f'<|call|>{name}\n' + ''.join((f'<|arg|>{k}\n{arg_text(v)}\n' for k, v in args.items())) + '<|/call|>' | |
| def parse_call(text, tools=None): | |
| name, _, rest = text.partition('<|arg|>') | |
| name, args = (name.strip(), {}) | |
| for part in ('<|arg|>' + rest).split('<|arg|>')[1:]: | |
| k, _, v = part.partition('\n') | |
| args[k.strip()] = v[:-1] if v.endswith('\n') else v | |
| types = {} | |
| for t in tools or []: | |
| if t['function']['name'] == name: | |
| types = {p: s.get('type') for p, s in (t['function'].get('parameters', {}).get('properties') or {}).items()} | |
| for k, v in list(args.items()): | |
| ty = types.get(k) | |
| try: | |
| if ty in ('number', 'integer'): | |
| args[k] = int(v) if re.fullmatch('\\s*-?\\d+\\s*', v) else float(v) | |
| elif ty == 'boolean': | |
| args[k] = v.strip().lower() == 'true' | |
| elif ty in ('array', 'object'): | |
| args[k] = json.loads(v) | |
| except ValueError: | |
| pass | |
| return (name, args) | |
| def clip(text, limit=MAX_RESULT): | |
| if len(text) <= limit: | |
| return text | |
| head, tail = (text[:limit * 2 // 3], text[-limit // 3:]) | |
| head, tail = (head[:head.rfind('\n') + 1] or head, tail[tail.find('\n') + 1:] or tail) | |
| cut = text.count('\n', len(head), len(text) - len(tail)) | |
| return f'{head}… ({cut} lines cut) …\n{tail}' | |
| def compact_read(text): | |
| m = re.match('\\s*<path>(.*?)</path>\\s*<type>(.*?)</type>\\s*<content>\\n?(.*?)\\n?</content>\\s*$', text, re.S) | |
| if not m: | |
| return compact_read2(text) | |
| path, kind, body = m.groups() | |
| note = '' | |
| n = re.search('\\n*\\((End of file - total (\\d+) lines|[^()\\n]*(?:offset|lines)[^()\\n]*)\\)\\s*$', body) | |
| if n: | |
| body, note = (body[:n.start()], n.group(1)) | |
| lines = body.split('\n') | |
| nums = [re.match('(\\d+): ?', l) for l in lines] | |
| if kind.strip() == 'file' and lines and all(nums): | |
| first, last = (int(nums[0].group(1)), int(nums[-1].group(1))) | |
| body = '\n'.join((l[k.end():] for l, k in zip(lines, nums))) | |
| total = n.group(2) if n and n.group(2) else (re.search('of (\\d+)', note) or [None, None])[1] | |
| if n and n.group(2) and (first == 1): | |
| head = f'{path} (end of file, {total} lines)' | |
| else: | |
| head = f'{path} (lines {first}-{last}' + (f' of {total}' if total else '') + ')' | |
| return head + '\n' + clip(body) | |
| return f'{path} ({kind.strip()})\n' + clip(body) + (f'\n({note})' if note else '') | |
| def compact_read2(text): | |
| m = re.match('Read (file|directory) (.+?), (?:lines|entries) (\\d+)-(\\d+)(?: of (\\d+))?\\r?\\n?(.*)\\Z', text, re.S) | |
| if not m: | |
| return text | |
| kind, path, first, last, total, body = m.groups() | |
| lines = body.replace('\r\n', '\n').rstrip('\r').split('\n') | |
| if kind == 'directory': | |
| entries = [l[:-1] + '/' if l.endswith('\\') else l for l in lines if l] | |
| return f'{path} (directory)\n' + clip('\n'.join(entries)) + f'\n({len(entries)} entries)' | |
| nums = [re.match('(\\d+): ?', l) for l in lines] | |
| if not all(nums): | |
| return text | |
| body = '\n'.join((l[k.end():] for l, k in zip(lines, nums))) | |
| first, last = (int(first), int(last)) | |
| if first == 1 and (total is None and last - first + 1 < 2000 or (total and last >= int(total))): | |
| return f'{path} (end of file, {last} lines)\n' + clip(body) | |
| return f'{path} (lines {first}-{last}' + (f' of {total}' if total else '') + ')\n' + clip(body) | |
| def read_result(path, content, first=1, total=None): | |
| n = content.count('\n') + 1 if content else 0 | |
| if first == 1 and (total is None or n >= total): | |
| return f'{path} (end of file, {n} lines)\n' + clip(content) | |
| return f'{path} (lines {first}-{first + n - 1}' + (f' of {total}' if total else '') + ')\n' + clip(content) | |
| def compact_result(name, text): | |
| text = text if isinstance(text, str) else json.dumps(text, ensure_ascii=False) | |
| if text.startswith('{"error"'): | |
| try: | |
| text = 'Error: ' + json.loads(text)['error']['message'] | |
| except (ValueError, KeyError, TypeError): | |
| pass | |
| return compact_read(text) if name == 'read' else clip(text) | |
| def content_text(c): | |
| if isinstance(c, list): | |
| return ''.join((x.get('text', '') for x in c if isinstance(x, dict))) | |
| return c or '' | |
| def think_text(m): | |
| t = (m.get('reasoning') or m.get('reasoning_content') or '').strip() | |
| return f'<|think|>{t}<|/think|>' if t else '' | |
| def render(messages, tools=None, env=None, system=None): | |
| out = [(f'<|system|>{(system if system is not None else system_text(env))}', False)] | |
| if tools: | |
| out.append((f'<|tools|>{tools_text(tools)}', False)) | |
| results = {m.get('tool_call_id'): m for m in messages if m['role'] == 'tool'} | |
| for m in messages: | |
| role = m['role'] | |
| if role == 'system': | |
| continue | |
| if role == 'user': | |
| out.append((f"<|user|>{content_text(m.get('content'))}", False)) | |
| elif role == 'assistant': | |
| text, calls = (content_text(m.get('content')).strip(), m.get('tool_calls') or []) | |
| th = think_text(m) | |
| learn = m.get('learn', True) | |
| if not calls: | |
| out += [('<|assistant|>', False), (f'{th}{text}<|end|>', learn)] | |
| continue | |
| for j, c in enumerate(calls): | |
| f = c['function'] | |
| args = f.get('arguments') or '{}' | |
| args = json.loads(args) if isinstance(args, str) else args | |
| out += [('<|assistant|>', False), ((th + text + ('\n' if text else '') if j == 0 else '') + call_text(f['name'], args), learn)] | |
| r = results.get(c.get('id')) | |
| out.append((f"<|tool|>{compact_result(f['name'], content_text(r.get('content')) if r else '(no result)')}", False)) | |
| return out | |
| def text_of(segments): | |
| return ''.join((t for t, _ in segments)) | |