Spaces:
Paused
Paused
Download src/tokenizer.ts from Rachit-Tw/PRIX: direct link, hf CLI and curl.
- Browser
- Download file 367 Bytes
-
https://huggingface.co/spaces/Rachit-Tw/PRIX/resolve/refs%2Fpr%2F2/src/tokenizer.ts
- Command line
-
hf download hf://spaces/Rachit-Tw/PRIX@refs/pr/2/src/tokenizer.ts
-
curl -L -o tokenizer.ts https://huggingface.co/spaces/Rachit-Tw/PRIX/resolve/refs%2Fpr%2F2/src/tokenizer.ts
367 Bytes
| // eslint-disable-next-line camelcase | |
| import {get_encoding} from '@dqbd/tiktoken' | |
| const tokenizer = get_encoding('cl100k_base') | |
| export function encode(input: string): Uint32Array { | |
| return tokenizer.encode(input) | |
| } | |
| export function getTokenCount(input: string): number { | |
| input = input.replace(/<\|endoftext\|>/g, '') | |
| return encode(input).length | |
| } | |