Spaces:
Runtime error
Runtime error
Update index.html
Browse filesRemoved bpe fallback
- index.html +20 -56
index.html
CHANGED
|
@@ -279,66 +279,30 @@
|
|
| 279 |
async function loadTokenizerQwenLocal() {
|
| 280 |
resetProgress("Tokenizer");
|
| 281 |
setStatus("Tokenizer: starting…");
|
| 282 |
-
// Always resolve from current URL and force the /qwen/ segment.
|
| 283 |
-
let base = new URL("assets/models/qwen/", document.baseURI).href;
|
| 284 |
-
if (!/\/qwen\/$/.test(base)) {
|
| 285 |
-
base = new URL("assets/models/qwen/", base).href;
|
| 286 |
-
}
|
| 287 |
-
|
| 288 |
-
// Build exact file URLs
|
| 289 |
-
const tjson = new URL("tokenizer.json", base).href;
|
| 290 |
-
const tcfg = new URL("tokenizer_config.json", base).href;
|
| 291 |
-
const smap = new URL("special_tokens_map.json", base).href;
|
| 292 |
-
|
| 293 |
-
// Detect tokenizer.json first
|
| 294 |
-
const hasTokenizerJSON = await fetch(tjson, { method: "HEAD" }).then(r => r.ok).catch(() => false);
|
| 295 |
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
fetch(tjson).then(r => { if (!r.ok) throw new Error("missing tokenizer.json"); return r.json(); }),
|
| 300 |
-
fetch(tcfg ).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json"); return r.json(); }),
|
| 301 |
-
fetch(smap ).then(r => r.ok ? r.json() : null),
|
| 302 |
-
]);
|
| 303 |
-
|
| 304 |
-
files.set("tokenizer.json", new Blob([JSON.stringify(tokJSON)], { type: "application/json" }));
|
| 305 |
-
files.set("tokenizer_config.json", new Blob([JSON.stringify(tokCfgJSON)], { type: "application/json" }));
|
| 306 |
-
if (smapJSON) files.set("special_tokens_map.json", new Blob([JSON.stringify(smapJSON)], { type: "application/json" }));
|
| 307 |
-
|
| 308 |
-
tokenizer = await AutoTokenizer.from_pretrained("qwen", {
|
| 309 |
-
files,
|
| 310 |
-
local_files_only: true,
|
| 311 |
-
progress_callback: onProgress,
|
| 312 |
-
});
|
| 313 |
-
return;
|
| 314 |
-
} else {
|
| 315 |
-
// ---- fallback: legacy BPE (vocab.json + merges.txt) ----
|
| 316 |
-
const vocabURL = new URL("vocab.json", base).href;
|
| 317 |
-
const mergesURL = new URL("merges.txt", base).href;
|
| 318 |
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
]);
|
| 324 |
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
delete cfg.merges;
|
| 331 |
-
if (cfg.tokenizer_file) delete cfg.tokenizer_file;
|
| 332 |
-
files.set("tokenizer_config.json", new Blob([JSON.stringify(cfg)], { type: "application/json" }));
|
| 333 |
-
}
|
| 334 |
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
}
|
| 341 |
-
}
|
|
|
|
| 342 |
|
| 343 |
// Core model state (module scope)
|
| 344 |
let tokenizer = null, model = null;
|
|
|
|
| 279 |
async function loadTokenizerQwenLocal() {
|
| 280 |
resetProgress("Tokenizer");
|
| 281 |
setStatus("Tokenizer: starting…");
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 282 |
|
| 283 |
+
// Resolve the /qwen/ folder off the current page
|
| 284 |
+
let base = new URL("assets/models/qwen/", document.baseURI).href;
|
| 285 |
+
if (!/\/qwen\/$/.test(base)) base = new URL("assets/models/qwen/", base).href;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
|
| 287 |
+
// Read the tokenizer JSON, config and specials
|
| 288 |
+
const tokJSON = await fetch(new URL("tokenizer.json", base).href).then(r => { if (!r.ok) throw new Error("missing tokenizer.json"); return r.json(); });
|
| 289 |
+
const tokCfg = await fetch(new URL("tokenizer_config.json", base).href).then(r => r.ok ? r.json() : null);
|
| 290 |
+
const smapJSON = await fetch(new URL("special_tokens_map.json", base).href).then(r => r.ok ? r.json() : null);
|
|
|
|
| 291 |
|
| 292 |
+
// Build a fresh files map
|
| 293 |
+
const files = new Map();
|
| 294 |
+
files.set("tokenizer.json", new Blob([JSON.stringify(tokJSON)], { type: "application/json" }));
|
| 295 |
+
if (tokCfg) files.set("tokenizer_config.json", new Blob([JSON.stringify(tokCfg)], { type: "application/json" }));
|
| 296 |
+
if (smapJSON) files.set("special_tokens_map.json", new Blob([JSON.stringify(smapJSON)], { type: "application/json" }));
|
|
|
|
|
|
|
|
|
|
|
|
|
| 297 |
|
| 298 |
+
// Load the tokenizer strictly from local files
|
| 299 |
+
tokenizer = await AutoTokenizer.from_pretrained("qwen", {
|
| 300 |
+
files,
|
| 301 |
+
local_files_only: true,
|
| 302 |
+
progress_callback: onProgress,
|
| 303 |
+
});
|
| 304 |
+
}
|
| 305 |
+
|
| 306 |
|
| 307 |
// Core model state (module scope)
|
| 308 |
let tokenizer = null, model = null;
|