Melikshah's picture
download
raw
794 Bytes
import torch
def _special(tok):
return set(tok.all_special_tokens) | set(tok.get_added_vocab())
def opener_ids(tok):
special = _special(tok)
for t in ("<|tool_call>", "<tool_call>"):
if t in special:
return [tok.convert_tokens_to_ids(t)]
if "<|python_tag|>" in special:
return sorted({tok.convert_tokens_to_ids("<|python_tag|>")} | {tok.encode(s, add_special_tokens=False)[0] for s in ('{"', "{")})
raise ValueError("no known tool-call opener in this tokenizer")
def action_logodds(logits, opener):
logits = logits.float()
ids = [opener] if isinstance(opener, int) else list(opener)
other = logits.clone()
other[..., ids] = float("-inf")
return torch.logsumexp(logits[..., ids], dim=-1) - torch.logsumexp(other, dim=-1)

Xet Storage Details

Size:
794 Bytes
·
Xet hash:
ac43d6fe853eb4a94b4862521adb69c9c0bb75440b4d97d879b6c3880cfa4e03

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.