import { describe, expect, it } from "vitest";

import { refineTokenizerFormat } from "./format-detection";

const encoder = new TextEncoder();

describe("refineTokenizerFormat", () => {
  it("recognizes tiktoken rank files named tokenizer.model", () => {
    expect(
      refineTokenizerFormat(
        "sentencepiece",
        "tokenizer.model",
        encoder.encode("IQ== 0\nIg== 1\n"),
      ),
    ).toBe("tiktoken");
  });

  it("keeps binary tokenizer.model files as sentencepiece", () => {
    expect(
      refineTokenizerFormat(
        "sentencepiece",
        "tokenizer.model",
        new Uint8Array([10, 7, 0, 255]),
      ),
    ).toBe("sentencepiece");
  });
});