tokenize.mjs

// Single-message BERT tokenization with the same 256-token limit as Python.
export function encodeMessage(tokenizer, text) {
  const encoded = tokenizer.encode(text, { add_special_tokens: true, return_token_type_ids: true });
  const truncated = encoded.ids.length > 256;
  const ids = truncated ? [...encoded.ids.slice(0, 255), tokenizer.token_to_id('[SEP]')] : encoded.ids;
  return { input_ids: ids, attention_mask: ids.map(() => 1), token_type_ids: ids.map(() => 0), truncated };
}