"""Token classes for the work / comm / overhead split (method C of token_split.py). Kept from the old viewer for the incremental split (full version, see README): `classifier` maps a tool name to its class and `compact` reduces a `message` event to the counts the split reads. Classes: comm = flock's comms tools (post, read, search) + world tools whose manifest entry has class comm/communication + `extra_comm` (default verdicts, mentions); overhead = done, text-only replies, system prompt, round prompts, compaction; work = every other tool (world, folder, interviews). """ import json FLOCK_COMMS = ('post', 'read', 'search') # flock.models.COMMS_TOOLS without the `comms:` prefix DEFAULT_EXTRA_COMM = ('verdicts', 'mentions') CLASSES = ['work', 'comm', 'overhead'] def classifier(manifest_tools=(), extra_comm=DEFAULT_EXTRA_COMM): comm, over, work = set(FLOCK_COMMS) | set(extra_comm), {'done', None}, set() for t in manifest_tools or (): c = str(t.get('class') or '').lower() if c in ('comm', 'comms', 'communication'): comm.add(t.get('name')) elif c == 'work': work.add(t.get('name')) elif c == 'overhead': over.add(t.get('name')) comm -= work return lambda n: 'overhead' if n in over else 'comm' if n in comm else 'work' def model_group(model): m = (model or '').lower() return 'DeepSeek' if 'deepseek' in m else 'GLM' if 'glm' in m else 'Qwen' if 'qwen' in m else (model or '?').split('/')[-1] def compact(m): """A message reduced to what split() reads: role, model, usage, toolName and, per content item, its type, tool name and character count (the strings themselves are dropped: a 300-agent run's streams hold gigabytes of them). split() takes compacted and full messages alike.""" content = m.get('content') or [] if isinstance(content, str): content = [{'type': 'text', 'text': content}] items = [] for c in content: if not isinstance(c, dict): continue t = c.get('type') if t == 'toolCall': items.append({'type': t, 'name': c.get('name'), 'chars': len(c.get('name') or '') + len(json.dumps(c.get('arguments', {})))}) elif t == 'thinking': items.append({'type': t, 'chars': len(c.get('thinking', '') or c.get('text', ''))}) else: items.append({'type': t, 'chars': len(c.get('text', ''))}) return {k: m[k] for k in ('role', 'model', 'usage', 'toolName') if k in m} | {'content': items}