import type { TokenizerFormat } from "./types";

const TIKTOKEN_LINE = /^[A-Za-z0-9+/]+={0,2} [0-9]+$/u;

export function refineTokenizerFormat(
  declaredFormat: TokenizerFormat,
  filename: string,
  bytes: Uint8Array,
): TokenizerFormat {
  if (
    declaredFormat !== "sentencepiece" ||
    !filename.endsWith("tokenizer.model")
  ) {
    return declaredFormat;
  }

  const sample = bytes.subarray(0, Math.min(bytes.byteLength, 4096));
  if (sample.some((byte) => byte === 0 || byte > 127)) return declaredFormat;

  const firstLine = new TextDecoder()
    .decode(sample)
    .split(/\r?\n/u)
    .find((line) => line.length > 0);
  return firstLine && TIKTOKEN_LINE.test(firstLine)
    ? "tiktoken"
    : declaredFormat;
}