Spaces:

Orion-zhen
/

tokenize-it

Running

tokenize-it / src /tokenizer /format-detection.test.ts

:boom: use web stack

21ab33c unverified about 2 months ago

683 Bytes

import { describe, expect, it } from "vitest";

import { refineTokenizerFormat } from "./format-detection";

const encoder = new TextEncoder();

describe("refineTokenizerFormat", () => {
  it("recognizes tiktoken rank files named tokenizer.model", () => {
    expect(
      refineTokenizerFormat(
        "sentencepiece",
        "tokenizer.model",
        encoder.encode("IQ== 0\nIg== 1\n"),
      ),
    ).toBe("tiktoken");
  });

  it("keeps binary tokenizer.model files as sentencepiece", () => {
    expect(
      refineTokenizerFormat(
        "sentencepiece",
        "tokenizer.model",
        new Uint8Array([10, 7, 0, 255]),
      ),
    ).toBe("sentencepiece");
  });
});

	import { describe, expect, it } from "vitest";

	import { refineTokenizerFormat } from "./format-detection";

	const encoder = new TextEncoder();

	describe("refineTokenizerFormat", () => {
	it("recognizes tiktoken rank files named tokenizer.model", () => {
	expect(
	refineTokenizerFormat(
	"sentencepiece",
	"tokenizer.model",
	encoder.encode("IQ== 0\nIg== 1\n"),
	),
	).toBe("tiktoken");
	});

	it("keeps binary tokenizer.model files as sentencepiece", () => {
	expect(
	refineTokenizerFormat(
	"sentencepiece",
	"tokenizer.model",
	new Uint8Array([10, 7, 0, 255]),
	),
	).toBe("sentencepiece");
	});
	});