Spaces:
Running
Running
Download token_classes.py from cmpatino/flock-run-viewer: direct link, hf CLI and curl.
- Browser
- Download file 2.48 kB
-
https://huggingface.co/spaces/cmpatino/flock-run-viewer/resolve/main/token_classes.py
- Command line
-
hf download hf://spaces/cmpatino/flock-run-viewer/token_classes.py
-
curl -L -o token_classes.py https://huggingface.co/spaces/cmpatino/flock-run-viewer/resolve/main/token_classes.py
2.48 kB
| """Token classes for the work / comm / overhead split (method C of token_split.py). | |
| Kept from the old viewer for the incremental split (full version, see README): `classifier` maps a | |
| tool name to its class and `compact` reduces a `message` event to the counts the split reads. | |
| Classes: comm = flock's comms tools (post, read, search) + world tools whose manifest entry has | |
| class comm/communication + `extra_comm` (default verdicts, mentions); overhead = done, text-only | |
| replies, system prompt, round prompts, compaction; work = every other tool (world, folder, interviews). | |
| """ | |
| import json | |
| FLOCK_COMMS = ('post', 'read', 'search') # flock.models.COMMS_TOOLS without the `comms:` prefix | |
| DEFAULT_EXTRA_COMM = ('verdicts', 'mentions') | |
| CLASSES = ['work', 'comm', 'overhead'] | |
| def classifier(manifest_tools=(), extra_comm=DEFAULT_EXTRA_COMM): | |
| comm, over, work = set(FLOCK_COMMS) | set(extra_comm), {'done', None}, set() | |
| for t in manifest_tools or (): | |
| c = str(t.get('class') or '').lower() | |
| if c in ('comm', 'comms', 'communication'): comm.add(t.get('name')) | |
| elif c == 'work': work.add(t.get('name')) | |
| elif c == 'overhead': over.add(t.get('name')) | |
| comm -= work | |
| return lambda n: 'overhead' if n in over else 'comm' if n in comm else 'work' | |
| def model_group(model): | |
| m = (model or '').lower() | |
| return 'DeepSeek' if 'deepseek' in m else 'GLM' if 'glm' in m else 'Qwen' if 'qwen' in m else (model or '?').split('/')[-1] | |
| def compact(m): | |
| """A message reduced to what split() reads: role, model, usage, toolName and, per content item, its type, | |
| tool name and character count (the strings themselves are dropped: a 300-agent run's streams hold | |
| gigabytes of them). split() takes compacted and full messages alike.""" | |
| content = m.get('content') or [] | |
| if isinstance(content, str): content = [{'type': 'text', 'text': content}] | |
| items = [] | |
| for c in content: | |
| if not isinstance(c, dict): continue | |
| t = c.get('type') | |
| if t == 'toolCall': | |
| items.append({'type': t, 'name': c.get('name'), 'chars': len(c.get('name') or '') + len(json.dumps(c.get('arguments', {})))}) | |
| elif t == 'thinking': | |
| items.append({'type': t, 'chars': len(c.get('thinking', '') or c.get('text', ''))}) | |
| else: | |
| items.append({'type': t, 'chars': len(c.get('text', ''))}) | |
| return {k: m[k] for k in ('role', 'model', 'usage', 'toolName') if k in m} | {'content': items} | |