File size: 11,554 Bytes
2309cb1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 | # Têtu Code 2 — inference code (tetu-AI). License: Apache 2.0, see README.md.
import json
import re
SYSTEM = "Tu es Têtu Code, un agent de programmation qui travaille dans le terminal de l'utilisateur. Sers-toi des outils pour lire, chercher, modifier et tester le code, un appel à la fois. Lis un fichier avant de le modifier ; après une modification, vérifie (tests, exécution) quand c'est possible. Réponds dans la langue de l'utilisateur, brièvement et précisément."
ENV_KEYS = ('Working directory', 'Is directory a git repo', 'Platform', "Today's date")
TOOL_DESC = {'bash': 'run a shell command', 'read': 'read a file or list a directory', 'edit': 'replace an exact passage of a file (oldString must match exactly once, or set replaceAll)', 'write': 'write a whole file', 'glob': 'find files by name pattern', 'grep': 'search file contents with a regex', 'todowrite': 'update the task list', 'webfetch': 'fetch a web page'}
DROP = {'task', 'skill', 'question', 'todowrite', 'webfetch', 'subagent', 'websearch', 'execute'}
TOOL_ALIASES = {'shell': 'bash'}
ARG_ALIASES = {'read': {'path': 'filePath'}, 'write': {'path': 'filePath'}, 'edit': {'path': 'filePath'}}
OPENCODE_TOOLS = [('bash', [('command', 'string', True), ('timeout', 'integer', False), ('workdir', 'string', False)]), ('edit', [('filePath', 'string', True), ('oldString', 'string', True), ('newString', 'string', True), ('replaceAll', 'boolean', False)]), ('glob', [('pattern', 'string', True), ('path', 'string', False)]), ('grep', [('pattern', 'string', True), ('path', 'string', False), ('include', 'string', False)]), ('read', [('filePath', 'string', True), ('offset', 'integer', False), ('limit', 'integer', False)]), ('todowrite', [('todos', 'array', True)]), ('webfetch', [('url', 'string', True), ('format', 'string', False), ('timeout', 'number', False)]), ('write', [('content', 'string', True), ('filePath', 'string', True)])]
MAX_RESULT = 12000
def opencode_tools():
out = []
for name, params in OPENCODE_TOOLS:
props = {p: {'type': t} for p, t, _ in params}
out.append({'type': 'function', 'function': {'name': name, 'description': TOOL_DESC[name], 'parameters': {'type': 'object', 'properties': props, 'required': [p for p, _, r in params if r]}}})
return out
def keep_tools(tools):
return [t for t in tools or [] if t.get('function', {}).get('name') not in DROP]
def dialect(tools):
out, back = ([], {})
for t in tools or []:
f = t.get('function', {})
cname = f.get('name')
name = TOOL_ALIASES.get(cname, cname)
ps = f.get('parameters') or {}
props = ps.get('properties') or {}
ren = {p: m for p, m in ARG_ALIASES.get(name, {}).items() if p in props and m not in props}
if name != cname or ren:
ps = dict(ps, properties={ren.get(p, p): s for p, s in props.items()}, required=[ren.get(p, p) for p in ps.get('required', [])])
t = dict(t, function=dict(f, name=name, parameters=ps))
back[name] = (cname, {m: p for p, m in ren.items()})
out.append(t)
order = [n for n, _ in OPENCODE_TOOLS]
out.sort(key=lambda t: order.index(t['function']['name']) if t['function']['name'] in order else len(order))
return (out, back)
def to_model(messages, back):
fwd = {cname: (name, {c: m for m, c in args.items()}) for name, (cname, args) in back.items()}
out = []
for m in messages:
if m.get('role') == 'assistant' and m.get('tool_calls'):
calls = []
for c in m['tool_calls']:
f = c.get('function', {})
name, ren = fwd.get(f.get('name'), (f.get('name'), {}))
f = dict(f, name=name)
if ren:
try:
a = json.loads(f.get('arguments') or '{}')
except ValueError:
a = None
if isinstance(a, dict):
f['arguments'] = json.dumps({ren.get(k, k): v for k, v in a.items()}, ensure_ascii=False)
calls.append(dict(c, function=f))
m = dict(m, tool_calls=calls)
out.append(m)
return out
def to_client(call, back):
if call is None:
return None
name, args = call
cname, ren = back.get(name, (name, {}))
return (cname, {ren.get(k, k): v for k, v in args.items()})
def tools_text(tools):
lines = []
for t in keep_tools(tools):
f = t['function']
ps = f.get('parameters', {}) or {}
req = set(ps.get('required', []))
args = ', '.join((p + ('' if p in req else '?') for p in ps.get('properties') or {}))
desc = TOOL_DESC.get(f['name']) or re.split('(?<=[.!?])\\s|\\n', (f.get('description') or '').strip())[0][:120]
lines.append(f"{f['name']}({args}) — {desc}")
return '\n'.join(lines)
def env_from_system(text):
m = re.search('<env>(.*?)</env>', text or '', re.S)
env = {}
for line in (m.group(1) if m else '').splitlines():
k, _, v = line.strip().partition(':')
if k in ENV_KEYS:
env[k] = v.strip()
return env
def system_text(env=None):
env = env or {}
lines = [f'{k}: {env[k]}' for k in ENV_KEYS if env.get(k)]
return SYSTEM + ('\n<env>\n' + '\n'.join(lines) + '\n</env>' if lines else '')
def arg_text(v):
if isinstance(v, str):
return v
if isinstance(v, bool):
return 'true' if v else 'false'
if isinstance(v, (int, float)):
return str(int(v)) if float(v).is_integer() else str(v)
return json.dumps(v, ensure_ascii=False)
def call_text(name, args):
return f'<|call|>{name}\n' + ''.join((f'<|arg|>{k}\n{arg_text(v)}\n' for k, v in args.items())) + '<|/call|>'
def parse_call(text, tools=None):
name, _, rest = text.partition('<|arg|>')
name, args = (name.strip(), {})
for part in ('<|arg|>' + rest).split('<|arg|>')[1:]:
k, _, v = part.partition('\n')
args[k.strip()] = v[:-1] if v.endswith('\n') else v
types = {}
for t in tools or []:
if t['function']['name'] == name:
types = {p: s.get('type') for p, s in (t['function'].get('parameters', {}).get('properties') or {}).items()}
for k, v in list(args.items()):
ty = types.get(k)
try:
if ty in ('number', 'integer'):
args[k] = int(v) if re.fullmatch('\\s*-?\\d+\\s*', v) else float(v)
elif ty == 'boolean':
args[k] = v.strip().lower() == 'true'
elif ty in ('array', 'object'):
args[k] = json.loads(v)
except ValueError:
pass
return (name, args)
def clip(text, limit=MAX_RESULT):
if len(text) <= limit:
return text
head, tail = (text[:limit * 2 // 3], text[-limit // 3:])
head, tail = (head[:head.rfind('\n') + 1] or head, tail[tail.find('\n') + 1:] or tail)
cut = text.count('\n', len(head), len(text) - len(tail))
return f'{head}… ({cut} lines cut) …\n{tail}'
def compact_read(text):
m = re.match('\\s*<path>(.*?)</path>\\s*<type>(.*?)</type>\\s*<content>\\n?(.*?)\\n?</content>\\s*$', text, re.S)
if not m:
return compact_read2(text)
path, kind, body = m.groups()
note = ''
n = re.search('\\n*\\((End of file - total (\\d+) lines|[^()\\n]*(?:offset|lines)[^()\\n]*)\\)\\s*$', body)
if n:
body, note = (body[:n.start()], n.group(1))
lines = body.split('\n')
nums = [re.match('(\\d+): ?', l) for l in lines]
if kind.strip() == 'file' and lines and all(nums):
first, last = (int(nums[0].group(1)), int(nums[-1].group(1)))
body = '\n'.join((l[k.end():] for l, k in zip(lines, nums)))
total = n.group(2) if n and n.group(2) else (re.search('of (\\d+)', note) or [None, None])[1]
if n and n.group(2) and (first == 1):
head = f'{path} (end of file, {total} lines)'
else:
head = f'{path} (lines {first}-{last}' + (f' of {total}' if total else '') + ')'
return head + '\n' + clip(body)
return f'{path} ({kind.strip()})\n' + clip(body) + (f'\n({note})' if note else '')
def compact_read2(text):
m = re.match('Read (file|directory) (.+?), (?:lines|entries) (\\d+)-(\\d+)(?: of (\\d+))?\\r?\\n?(.*)\\Z', text, re.S)
if not m:
return text
kind, path, first, last, total, body = m.groups()
lines = body.replace('\r\n', '\n').rstrip('\r').split('\n')
if kind == 'directory':
entries = [l[:-1] + '/' if l.endswith('\\') else l for l in lines if l]
return f'{path} (directory)\n' + clip('\n'.join(entries)) + f'\n({len(entries)} entries)'
nums = [re.match('(\\d+): ?', l) for l in lines]
if not all(nums):
return text
body = '\n'.join((l[k.end():] for l, k in zip(lines, nums)))
first, last = (int(first), int(last))
if first == 1 and (total is None and last - first + 1 < 2000 or (total and last >= int(total))):
return f'{path} (end of file, {last} lines)\n' + clip(body)
return f'{path} (lines {first}-{last}' + (f' of {total}' if total else '') + ')\n' + clip(body)
def read_result(path, content, first=1, total=None):
n = content.count('\n') + 1 if content else 0
if first == 1 and (total is None or n >= total):
return f'{path} (end of file, {n} lines)\n' + clip(content)
return f'{path} (lines {first}-{first + n - 1}' + (f' of {total}' if total else '') + ')\n' + clip(content)
def compact_result(name, text):
text = text if isinstance(text, str) else json.dumps(text, ensure_ascii=False)
if text.startswith('{"error"'):
try:
text = 'Error: ' + json.loads(text)['error']['message']
except (ValueError, KeyError, TypeError):
pass
return compact_read(text) if name == 'read' else clip(text)
def content_text(c):
if isinstance(c, list):
return ''.join((x.get('text', '') for x in c if isinstance(x, dict)))
return c or ''
def think_text(m):
t = (m.get('reasoning') or m.get('reasoning_content') or '').strip()
return f'<|think|>{t}<|/think|>' if t else ''
def render(messages, tools=None, env=None, system=None):
out = [(f'<|system|>{(system if system is not None else system_text(env))}', False)]
if tools:
out.append((f'<|tools|>{tools_text(tools)}', False))
results = {m.get('tool_call_id'): m for m in messages if m['role'] == 'tool'}
for m in messages:
role = m['role']
if role == 'system':
continue
if role == 'user':
out.append((f"<|user|>{content_text(m.get('content'))}", False))
elif role == 'assistant':
text, calls = (content_text(m.get('content')).strip(), m.get('tool_calls') or [])
th = think_text(m)
learn = m.get('learn', True)
if not calls:
out += [('<|assistant|>', False), (f'{th}{text}<|end|>', learn)]
continue
for j, c in enumerate(calls):
f = c['function']
args = f.get('arguments') or '{}'
args = json.loads(args) if isinstance(args, str) else args
out += [('<|assistant|>', False), ((th + text + ('\n' if text else '') if j == 0 else '') + call_text(f['name'], args), learn)]
r = results.get(c.get('id'))
out.append((f"<|tool|>{compact_result(f['name'], content_text(r.get('content')) if r else '(no result)')}", False))
return out
def text_of(segments):
return ''.join((t for t, _ in segments))
|