PeterPinetree commited on
Commit
87ce18d
·
verified ·
1 Parent(s): cb2b9a9

Update index.html

Browse files

Removed bpe fallback

Files changed (1) hide show
  1. index.html +20 -56
index.html CHANGED
@@ -279,66 +279,30 @@
279
  async function loadTokenizerQwenLocal() {
280
  resetProgress("Tokenizer");
281
  setStatus("Tokenizer: starting…");
282
- // Always resolve from current URL and force the /qwen/ segment.
283
- let base = new URL("assets/models/qwen/", document.baseURI).href;
284
- if (!/\/qwen\/$/.test(base)) {
285
- base = new URL("assets/models/qwen/", base).href;
286
- }
287
-
288
- // Build exact file URLs
289
- const tjson = new URL("tokenizer.json", base).href;
290
- const tcfg = new URL("tokenizer_config.json", base).href;
291
- const smap = new URL("special_tokens_map.json", base).href;
292
-
293
- // Detect tokenizer.json first
294
- const hasTokenizerJSON = await fetch(tjson, { method: "HEAD" }).then(r => r.ok).catch(() => false);
295
 
296
- if (hasTokenizerJSON) {
297
- // load tokenizer.json, tokenizer_config.json and special tokens map
298
- const [tokJSON, tokCfgJSON, smapJSON] = await Promise.all([
299
- fetch(tjson).then(r => { if (!r.ok) throw new Error("missing tokenizer.json"); return r.json(); }),
300
- fetch(tcfg ).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json"); return r.json(); }),
301
- fetch(smap ).then(r => r.ok ? r.json() : null),
302
- ]);
303
-
304
- files.set("tokenizer.json", new Blob([JSON.stringify(tokJSON)], { type: "application/json" }));
305
- files.set("tokenizer_config.json", new Blob([JSON.stringify(tokCfgJSON)], { type: "application/json" }));
306
- if (smapJSON) files.set("special_tokens_map.json", new Blob([JSON.stringify(smapJSON)], { type: "application/json" }));
307
-
308
- tokenizer = await AutoTokenizer.from_pretrained("qwen", {
309
- files,
310
- local_files_only: true,
311
- progress_callback: onProgress,
312
- });
313
- return;
314
- } else {
315
- // ---- fallback: legacy BPE (vocab.json + merges.txt) ----
316
- const vocabURL = new URL("vocab.json", base).href;
317
- const mergesURL = new URL("merges.txt", base).href;
318
 
319
- const [vocabText, mergesText, cfgMaybe] = await Promise.all([
320
- fetch(vocabURL ).then(r => { if (!r.ok) throw new Error("missing vocab.json"); return r.text(); }),
321
- fetch(mergesURL).then(r => { if (!r.ok) throw new Error("missing merges.txt"); return r.text(); }),
322
- fetch(new URL("tokenizer_config.json", base).href).then(r => r.ok ? r.json() : null),
323
- ]);
324
 
325
- const files = new Map();
326
- files.set("vocab.json", new Blob([vocabText], { type: "application/json" }));
327
- files.set("merges.txt", new Blob([mergesText], { type: "text/plain" }));
328
- if (cfgMaybe) {
329
- const cfg = { ...cfgMaybe };
330
- delete cfg.merges;
331
- if (cfg.tokenizer_file) delete cfg.tokenizer_file;
332
- files.set("tokenizer_config.json", new Blob([JSON.stringify(cfg)], { type: "application/json" }));
333
- }
334
 
335
- tokenizer = await AutoTokenizer.from_pretrained("", {
336
- files,
337
- local_files_only: true,
338
- progress_callback: onProgress,
339
- });
340
- }
341
- } // end of loadTokenizerQwenLocal
 
342
 
343
  // Core model state (module scope)
344
  let tokenizer = null, model = null;
 
279
  async function loadTokenizerQwenLocal() {
280
  resetProgress("Tokenizer");
281
  setStatus("Tokenizer: starting…");
 
 
 
 
 
 
 
 
 
 
 
 
 
282
 
283
+ // Resolve the /qwen/ folder off the current page
284
+ let base = new URL("assets/models/qwen/", document.baseURI).href;
285
+ if (!/\/qwen\/$/.test(base)) base = new URL("assets/models/qwen/", base).href;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
286
 
287
+ // Read the tokenizer JSON, config and specials
288
+ const tokJSON = await fetch(new URL("tokenizer.json", base).href).then(r => { if (!r.ok) throw new Error("missing tokenizer.json"); return r.json(); });
289
+ const tokCfg = await fetch(new URL("tokenizer_config.json", base).href).then(r => r.ok ? r.json() : null);
290
+ const smapJSON = await fetch(new URL("special_tokens_map.json", base).href).then(r => r.ok ? r.json() : null);
 
291
 
292
+ // Build a fresh files map
293
+ const files = new Map();
294
+ files.set("tokenizer.json", new Blob([JSON.stringify(tokJSON)], { type: "application/json" }));
295
+ if (tokCfg) files.set("tokenizer_config.json", new Blob([JSON.stringify(tokCfg)], { type: "application/json" }));
296
+ if (smapJSON) files.set("special_tokens_map.json", new Blob([JSON.stringify(smapJSON)], { type: "application/json" }));
 
 
 
 
297
 
298
+ // Load the tokenizer strictly from local files
299
+ tokenizer = await AutoTokenizer.from_pretrained("qwen", {
300
+ files,
301
+ local_files_only: true,
302
+ progress_callback: onProgress,
303
+ });
304
+ }
305
+
306
 
307
  // Core model state (module scope)
308
  let tokenizer = null, model = null;