Spaces:
Running
Running
| import type { TokenizerFormat } from "./types"; | |
| const TIKTOKEN_LINE = /^[A-Za-z0-9+/]+={0,2} [0-9]+$/u; | |
| export function refineTokenizerFormat( | |
| declaredFormat: TokenizerFormat, | |
| filename: string, | |
| bytes: Uint8Array, | |
| ): TokenizerFormat { | |
| if ( | |
| declaredFormat !== "sentencepiece" || | |
| !filename.endsWith("tokenizer.model") | |
| ) { | |
| return declaredFormat; | |
| } | |
| const sample = bytes.subarray(0, Math.min(bytes.byteLength, 4096)); | |
| if (sample.some((byte) => byte === 0 || byte > 127)) return declaredFormat; | |
| const firstLine = new TextDecoder() | |
| .decode(sample) | |
| .split(/\r?\n/u) | |
| .find((line) => line.length > 0); | |
| return firstLine && TIKTOKEN_LINE.test(firstLine) | |
| ? "tiktoken" | |
| : declaredFormat; | |
| } | |