File size: 659 Bytes
75a94e7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 | export function encodeFor(text,encoding,mode='bpe'){
if(mode==='dual_bpe'){
const prefix='Write a short plan, then simple Python code.\nRequest: ',suffix='\nPlan:\n';
if(text.startsWith(prefix)&&text.endsWith(suffix)){
const request=text.slice(prefix.length,-suffix.length);
return [...encoding.encode(prefix+request+'\nVerbatim request:\n',[],[]),...encodeFor(request,encoding,'space_bpe'),...encoding.encode(suffix,[],[])];
}
return encoding.encode(text,[],[]);
}
return mode==='space_bpe'
? text.split(/([ \t\r\n\v\f]+)/).filter(Boolean).flatMap(part=>encoding.encode(part,[],[]))
: encoding.encode(text,[],[]);
}
|