Spaces:
Sleeping
Sleeping
File size: 4,964 Bytes
21ab33c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 | import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
fetchTokenizerAsset,
TOKENIZER_CACHE,
tokenizerCacheSize,
} from "./cache";
import type { RemoteFile } from "./types";
const HELLO_SHA256 =
"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824";
const HELLO_GIT_SHA1 = "b6fc4c620b67d95f953a5c1c1230aaab5db5a1b0";
const CHANGED_SHA256 =
"d67e2e944994496c8d8ec76eed0cf9f09679448d584b532bebf941852a37f5ed";
function file(url: string, algorithm: "git-sha1" | "sha256", value: string): RemoteFile {
return {
filename: "tokenizer.json",
url,
hash: { algorithm, value },
};
}
describe("tokenizer asset cache", () => {
const stores = new Map<string, Map<string, Response>>();
beforeEach(() => {
stores.clear();
vi.stubGlobal("self", { location: { origin: "https://tokenize-it.test" } });
vi.stubGlobal("caches", {
async open(name: string) {
let store = stores.get(name);
if (!store) {
store = new Map();
stores.set(name, store);
}
return {
async match(key: RequestInfo | URL) {
const url =
typeof key === "string"
? key
: key instanceof URL
? key.href
: key.url;
return store.get(url)?.clone();
},
async put(key: string, response: Response) {
store.set(key, response.clone());
},
async keys() {
return Array.from(store.keys(), (key) => new Request(key));
},
};
},
async delete(name: string) {
return stores.delete(name);
},
async keys() {
return Array.from(stores.keys());
},
});
});
afterEach(() => {
vi.unstubAllGlobals();
});
it("reuses identical content across different repository URLs", async () => {
const fetcher = vi.fn<typeof fetch>().mockResolvedValue(
new Response("hello", {
headers: { "Content-Type": "application/json" },
}),
);
vi.stubGlobal("fetch", fetcher);
const first = await fetchTokenizerAsset(
file("https://huggingface.co/owner/model-a/resolve/commit-a/tokenizer.json", "sha256", HELLO_SHA256),
vi.fn(),
);
const second = await fetchTokenizerAsset(
file("https://huggingface.co/owner/model-b/resolve/commit-b/tokenizer.json", "sha256", HELLO_SHA256),
vi.fn(),
);
expect(new TextDecoder().decode(first)).toBe("hello");
expect(new TextDecoder().decode(second)).toBe("hello");
expect(fetcher).toHaveBeenCalledTimes(1);
expect(stores.get(TOKENIZER_CACHE)?.size).toBe(1);
});
it("downloads a new record when the file hash changes", async () => {
const fetcher = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response("hello"))
.mockResolvedValueOnce(new Response("changed"));
vi.stubGlobal("fetch", fetcher);
await fetchTokenizerAsset(
file("https://huggingface.co/owner/model/resolve/commit-a/tokenizer.json", "sha256", HELLO_SHA256),
vi.fn(),
);
await fetchTokenizerAsset(
file("https://huggingface.co/owner/model/resolve/commit-b/tokenizer.json", "sha256", CHANGED_SHA256),
vi.fn(),
);
expect(fetcher).toHaveBeenCalledTimes(2);
expect(stores.get(TOKENIZER_CACHE)?.size).toBe(2);
});
it("reports the total size of cached tokenizer assets", async () => {
const fetcher = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response("hello"))
.mockResolvedValueOnce(new Response("changed"));
vi.stubGlobal("fetch", fetcher);
await fetchTokenizerAsset(
file("https://huggingface.co/owner/model/resolve/commit-a/tokenizer.json", "sha256", HELLO_SHA256),
vi.fn(),
);
await fetchTokenizerAsset(
file("https://huggingface.co/owner/model/resolve/commit-b/tokenizer.json", "sha256", CHANGED_SHA256),
vi.fn(),
);
await expect(tokenizerCacheSize()).resolves.toBe(12);
});
it("validates regular files using the Git blob hash", async () => {
vi.stubGlobal(
"fetch",
vi.fn<typeof fetch>().mockResolvedValue(new Response("hello")),
);
await expect(
fetchTokenizerAsset(
file("https://huggingface.co/owner/model/resolve/commit/tokenizer.json", "git-sha1", HELLO_GIT_SHA1),
vi.fn(),
),
).resolves.toEqual(new TextEncoder().encode("hello"));
});
it("rejects downloaded content that does not match the Hub hash", async () => {
vi.stubGlobal(
"fetch",
vi.fn<typeof fetch>().mockResolvedValue(new Response("changed")),
);
await expect(
fetchTokenizerAsset(
file("https://huggingface.co/owner/model/resolve/commit/tokenizer.json", "sha256", HELLO_SHA256),
vi.fn(),
),
).rejects.toThrow("文件与预期哈希不一致");
expect(stores.get(TOKENIZER_CACHE)?.size).toBe(0);
});
});
|