Download internal/runtime/executor/token_helpers.go from shimen/API: direct link, hf CLI and curl.
- Browser
- Download file 7.11 kB
-
https://huggingface.co/spaces/shimen/API/resolve/main/internal/runtime/executor/token_helpers.go
- Command line
-
hf download hf://spaces/shimen/API/internal/runtime/executor/token_helpers.go
-
curl -L -o token_helpers.go https://huggingface.co/spaces/shimen/API/resolve/main/internal/runtime/executor/token_helpers.go
7.11 kB
| package executor | |
| import ( | |
| "fmt" | |
| "strings" | |
| "github.com/tidwall/gjson" | |
| "github.com/tiktoken-go/tokenizer" | |
| ) | |
| // tokenizerForModel returns a tokenizer codec suitable for an OpenAI-style model id. | |
| func tokenizerForModel(model string) (tokenizer.Codec, error) { | |
| sanitized := strings.ToLower(strings.TrimSpace(model)) | |
| switch { | |
| case sanitized == "": | |
| return tokenizer.Get(tokenizer.Cl100kBase) | |
| case strings.HasPrefix(sanitized, "gpt-5"): | |
| return tokenizer.ForModel(tokenizer.GPT5) | |
| case strings.HasPrefix(sanitized, "gpt-5.1"): | |
| return tokenizer.ForModel(tokenizer.GPT5) | |
| case strings.HasPrefix(sanitized, "gpt-4.1"): | |
| return tokenizer.ForModel(tokenizer.GPT41) | |
| case strings.HasPrefix(sanitized, "gpt-4o"): | |
| return tokenizer.ForModel(tokenizer.GPT4o) | |
| case strings.HasPrefix(sanitized, "gpt-4"): | |
| return tokenizer.ForModel(tokenizer.GPT4) | |
| case strings.HasPrefix(sanitized, "gpt-3.5"), strings.HasPrefix(sanitized, "gpt-3"): | |
| return tokenizer.ForModel(tokenizer.GPT35Turbo) | |
| case strings.HasPrefix(sanitized, "o1"): | |
| return tokenizer.ForModel(tokenizer.O1) | |
| case strings.HasPrefix(sanitized, "o3"): | |
| return tokenizer.ForModel(tokenizer.O3) | |
| case strings.HasPrefix(sanitized, "o4"): | |
| return tokenizer.ForModel(tokenizer.O4Mini) | |
| default: | |
| return tokenizer.Get(tokenizer.O200kBase) | |
| } | |
| } | |
| // countOpenAIChatTokens approximates prompt tokens for OpenAI chat completions payloads. | |
| func countOpenAIChatTokens(enc tokenizer.Codec, payload []byte) (int64, error) { | |
| if enc == nil { | |
| return 0, fmt.Errorf("encoder is nil") | |
| } | |
| if len(payload) == 0 { | |
| return 0, nil | |
| } | |
| root := gjson.ParseBytes(payload) | |
| segments := make([]string, 0, 32) | |
| collectOpenAIMessages(root.Get("messages"), &segments) | |
| collectOpenAITools(root.Get("tools"), &segments) | |
| collectOpenAIFunctions(root.Get("functions"), &segments) | |
| collectOpenAIToolChoice(root.Get("tool_choice"), &segments) | |
| collectOpenAIResponseFormat(root.Get("response_format"), &segments) | |
| addIfNotEmpty(&segments, root.Get("input").String()) | |
| addIfNotEmpty(&segments, root.Get("prompt").String()) | |
| joined := strings.TrimSpace(strings.Join(segments, "\n")) | |
| if joined == "" { | |
| return 0, nil | |
| } | |
| count, err := enc.Count(joined) | |
| if err != nil { | |
| return 0, err | |
| } | |
| return int64(count), nil | |
| } | |
| // buildOpenAIUsageJSON returns a minimal usage structure understood by downstream translators. | |
| func buildOpenAIUsageJSON(count int64) []byte { | |
| return []byte(fmt.Sprintf(`{"usage":{"prompt_tokens":%d,"completion_tokens":0,"total_tokens":%d}}`, count, count)) | |
| } | |
| func collectOpenAIMessages(messages gjson.Result, segments *[]string) { | |
| if !messages.Exists() || !messages.IsArray() { | |
| return | |
| } | |
| messages.ForEach(func(_, message gjson.Result) bool { | |
| addIfNotEmpty(segments, message.Get("role").String()) | |
| addIfNotEmpty(segments, message.Get("name").String()) | |
| collectOpenAIContent(message.Get("content"), segments) | |
| collectOpenAIToolCalls(message.Get("tool_calls"), segments) | |
| collectOpenAIFunctionCall(message.Get("function_call"), segments) | |
| return true | |
| }) | |
| } | |
| func collectOpenAIContent(content gjson.Result, segments *[]string) { | |
| if !content.Exists() { | |
| return | |
| } | |
| if content.Type == gjson.String { | |
| addIfNotEmpty(segments, content.String()) | |
| return | |
| } | |
| if content.IsArray() { | |
| content.ForEach(func(_, part gjson.Result) bool { | |
| partType := part.Get("type").String() | |
| switch partType { | |
| case "text", "input_text", "output_text": | |
| addIfNotEmpty(segments, part.Get("text").String()) | |
| case "image_url": | |
| addIfNotEmpty(segments, part.Get("image_url.url").String()) | |
| case "input_audio", "output_audio", "audio": | |
| addIfNotEmpty(segments, part.Get("id").String()) | |
| case "tool_result": | |
| addIfNotEmpty(segments, part.Get("name").String()) | |
| collectOpenAIContent(part.Get("content"), segments) | |
| default: | |
| if part.IsArray() { | |
| collectOpenAIContent(part, segments) | |
| return true | |
| } | |
| if part.Type == gjson.JSON { | |
| addIfNotEmpty(segments, part.Raw) | |
| return true | |
| } | |
| addIfNotEmpty(segments, part.String()) | |
| } | |
| return true | |
| }) | |
| return | |
| } | |
| if content.Type == gjson.JSON { | |
| addIfNotEmpty(segments, content.Raw) | |
| } | |
| } | |
| func collectOpenAIToolCalls(calls gjson.Result, segments *[]string) { | |
| if !calls.Exists() || !calls.IsArray() { | |
| return | |
| } | |
| calls.ForEach(func(_, call gjson.Result) bool { | |
| addIfNotEmpty(segments, call.Get("id").String()) | |
| addIfNotEmpty(segments, call.Get("type").String()) | |
| function := call.Get("function") | |
| if function.Exists() { | |
| addIfNotEmpty(segments, function.Get("name").String()) | |
| addIfNotEmpty(segments, function.Get("description").String()) | |
| addIfNotEmpty(segments, function.Get("arguments").String()) | |
| if params := function.Get("parameters"); params.Exists() { | |
| addIfNotEmpty(segments, params.Raw) | |
| } | |
| } | |
| return true | |
| }) | |
| } | |
| func collectOpenAIFunctionCall(call gjson.Result, segments *[]string) { | |
| if !call.Exists() { | |
| return | |
| } | |
| addIfNotEmpty(segments, call.Get("name").String()) | |
| addIfNotEmpty(segments, call.Get("arguments").String()) | |
| } | |
| func collectOpenAITools(tools gjson.Result, segments *[]string) { | |
| if !tools.Exists() { | |
| return | |
| } | |
| if tools.IsArray() { | |
| tools.ForEach(func(_, tool gjson.Result) bool { | |
| appendToolPayload(tool, segments) | |
| return true | |
| }) | |
| return | |
| } | |
| appendToolPayload(tools, segments) | |
| } | |
| func collectOpenAIFunctions(functions gjson.Result, segments *[]string) { | |
| if !functions.Exists() || !functions.IsArray() { | |
| return | |
| } | |
| functions.ForEach(func(_, function gjson.Result) bool { | |
| addIfNotEmpty(segments, function.Get("name").String()) | |
| addIfNotEmpty(segments, function.Get("description").String()) | |
| if params := function.Get("parameters"); params.Exists() { | |
| addIfNotEmpty(segments, params.Raw) | |
| } | |
| return true | |
| }) | |
| } | |
| func collectOpenAIToolChoice(choice gjson.Result, segments *[]string) { | |
| if !choice.Exists() { | |
| return | |
| } | |
| if choice.Type == gjson.String { | |
| addIfNotEmpty(segments, choice.String()) | |
| return | |
| } | |
| addIfNotEmpty(segments, choice.Raw) | |
| } | |
| func collectOpenAIResponseFormat(format gjson.Result, segments *[]string) { | |
| if !format.Exists() { | |
| return | |
| } | |
| addIfNotEmpty(segments, format.Get("type").String()) | |
| addIfNotEmpty(segments, format.Get("name").String()) | |
| if schema := format.Get("json_schema"); schema.Exists() { | |
| addIfNotEmpty(segments, schema.Raw) | |
| } | |
| if schema := format.Get("schema"); schema.Exists() { | |
| addIfNotEmpty(segments, schema.Raw) | |
| } | |
| } | |
| func appendToolPayload(tool gjson.Result, segments *[]string) { | |
| if !tool.Exists() { | |
| return | |
| } | |
| addIfNotEmpty(segments, tool.Get("type").String()) | |
| addIfNotEmpty(segments, tool.Get("name").String()) | |
| addIfNotEmpty(segments, tool.Get("description").String()) | |
| if function := tool.Get("function"); function.Exists() { | |
| addIfNotEmpty(segments, function.Get("name").String()) | |
| addIfNotEmpty(segments, function.Get("description").String()) | |
| if params := function.Get("parameters"); params.Exists() { | |
| addIfNotEmpty(segments, params.Raw) | |
| } | |
| } | |
| } | |
| func addIfNotEmpty(segments *[]string, value string) { | |
| if segments == nil { | |
| return | |
| } | |
| if trimmed := strings.TrimSpace(value); trimmed != "" { | |
| *segments = append(*segments, trimmed) | |
| } | |
| } | |