Spaces:
Paused
Paused
File size: 367 Bytes
9284ad7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 | // eslint-disable-next-line camelcase
import {get_encoding} from '@dqbd/tiktoken'
const tokenizer = get_encoding('cl100k_base')
export function encode(input: string): Uint32Array {
return tokenizer.encode(input)
}
export function getTokenCount(input: string): number {
input = input.replace(/<\|endoftext\|>/g, '')
return encode(input).length
}
|