flock-run-viewer / token_classes.py
cmpatino's picture
cmpatino HF Staff
Incremental SQLite ingest (legacy adapter) and read views
6d39c3f
Raw History Blame Contribute Delete
2.48 kB
"""Token classes for the work / comm / overhead split (method C of token_split.py).
Kept from the old viewer for the incremental split (full version, see README): `classifier` maps a
tool name to its class and `compact` reduces a `message` event to the counts the split reads.
Classes: comm = flock's comms tools (post, read, search) + world tools whose manifest entry has
class comm/communication + `extra_comm` (default verdicts, mentions); overhead = done, text-only
replies, system prompt, round prompts, compaction; work = every other tool (world, folder, interviews).
"""
import json
FLOCK_COMMS = ('post', 'read', 'search') # flock.models.COMMS_TOOLS without the `comms:` prefix
DEFAULT_EXTRA_COMM = ('verdicts', 'mentions')
CLASSES = ['work', 'comm', 'overhead']
def classifier(manifest_tools=(), extra_comm=DEFAULT_EXTRA_COMM):
comm, over, work = set(FLOCK_COMMS) | set(extra_comm), {'done', None}, set()
for t in manifest_tools or ():
c = str(t.get('class') or '').lower()
if c in ('comm', 'comms', 'communication'): comm.add(t.get('name'))
elif c == 'work': work.add(t.get('name'))
elif c == 'overhead': over.add(t.get('name'))
comm -= work
return lambda n: 'overhead' if n in over else 'comm' if n in comm else 'work'
def model_group(model):
m = (model or '').lower()
return 'DeepSeek' if 'deepseek' in m else 'GLM' if 'glm' in m else 'Qwen' if 'qwen' in m else (model or '?').split('/')[-1]
def compact(m):
"""A message reduced to what split() reads: role, model, usage, toolName and, per content item, its type,
tool name and character count (the strings themselves are dropped: a 300-agent run's streams hold
gigabytes of them). split() takes compacted and full messages alike."""
content = m.get('content') or []
if isinstance(content, str): content = [{'type': 'text', 'text': content}]
items = []
for c in content:
if not isinstance(c, dict): continue
t = c.get('type')
if t == 'toolCall':
items.append({'type': t, 'name': c.get('name'), 'chars': len(c.get('name') or '') + len(json.dumps(c.get('arguments', {})))})
elif t == 'thinking':
items.append({'type': t, 'chars': len(c.get('thinking', '') or c.get('text', ''))})
else:
items.append({'type': t, 'chars': len(c.get('text', ''))})
return {k: m[k] for k in ('role', 'model', 'usage', 'toolName') if k in m} | {'content': items}