| import {MultiIndic} from './multiindic.mjs'; |
| const assetBase=new URL('../',import.meta.url).href; |
| const names={hi:'Hindi',bn:'Bengali',te:'Telugu',kn:'Kannada',ta:'Tamil'}; |
| const CACHE='aksharalp-multiindic-assets-v1'; |
| const kindFor=path=>path.includes('/dict/')||path.endsWith('/dict.tsv.gz')?'dictionary':path.includes('/sentence/')?'sentence':'model'; |
| let engine,next=null,running=false,current=null,controller=null,context=null; |
| const tick=ms=>new Promise(resolve=>setTimeout(resolve,ms)); |
| async function openCache(){try{return await caches.open(CACHE);}catch{return null;}} |
| const manifestPromise=(async()=>{ |
| const url=new URL('loading-manifest.json',assetBase).href,cache=await openCache(); |
| try{const r=await globalThis.fetch(url,{cache:'no-cache'});if(!r.ok)throw new Error(`Manifest HTTP ${r.status}`);const data=await r.json();try{await cache?.put(url,new Response(JSON.stringify(data)));}catch{}return data.files;} |
| catch{try{return (await (await cache?.match(url))?.json())?.files||{};}catch{return {};}} |
| })(); |
| function paths(language,sentenceOnly=false){ |
| if(sentenceOnly)return ['models/hi/sentence/nlm.q8.gz']; |
| return language==='hi'?['models/hi/manifest.json.gz','models/hi/xlit.q4.gz','models/hi/params.json.gz','models/hi/dict/lex.bin.gz','models/hi/dict/lm.bin.gz','models/hi/dict/params.json.gz']:[`models/${language}/xlit.q4.gz`,`models/${language}/params.json`,`models/${language}/dict.tsv.gz`]; |
| } |
| function emit(ctx,forcePhase){ |
| if(ctx.signal.aborted)return; |
| const values=[...ctx.files.values()],loaded=values.reduce((n,f)=>n+f.loaded,0),total=values.every(f=>f.total>0)?values.reduce((n,f)=>n+f.total,0):null; |
| const modelPending=values.some(f=>f.kind!=='dictionary'&&!f.done),dictPending=values.some(f=>f.kind==='dictionary'&&!f.done); |
| let phase=forcePhase||(ctx.network===0&&ctx.cached>0?'cache':modelPending?'download':dictPending?'dictionary':'preparing'); |
| const label=phase==='cache'?'Loading from cache…':phase==='preparing'?'Preparing…':phase==='dictionary'?`Loading ${names[ctx.language]} dictionary…`:`Downloading ${names[ctx.language]} ${ctx.sentenceOnly?'sentence add-on':'model'}…`; |
| postMessage({id:ctx.id,progress:{phase,label,loaded,total,source:ctx.network===0&&ctx.cached>0?'cache':'network'}}); |
| } |
| async function streamedFetch(url){ |
| const ctx=context,path=String(url).slice(assetBase.length),meta=ctx.manifest[path]||{}; |
| let state=ctx.files.get(path);if(!state){state={loaded:0,total:meta.bytes||0,kind:meta.kind||kindFor(path),done:false};ctx.files.set(path,state);} |
| const key=new URL(url);key.searchParams.set('aksharalp',meta.sha256?.slice(0,16)||'v1'); |
| const cache=await openCache(); |
| if(ctx.signal.aborted)throw new DOMException('Language changed','AbortError'); |
| let hit;try{hit=await cache?.match(key.href);}catch{} |
| if(hit){const b=await hit.arrayBuffer();if(!meta.bytes||b.byteLength===meta.bytes){state.loaded=b.byteLength;state.total=b.byteLength;state.done=true;ctx.cached++;emit(ctx);return new Response(b);}} |
| ctx.network++;emit(ctx); |
| const response=await globalThis.fetch(url,{signal:ctx.signal,credentials:'same-origin'}); |
| if(!response.ok)throw new Error(`Could not load ${path}: HTTP ${response.status}`); |
| const length=Number(response.headers.get('content-length'));if(length>0&&!meta.bytes)state.total=length; |
| const chunks=[]; |
| if(response.body?.getReader){ |
| const reader=response.body.getReader(); |
| try{while(true){const {done,value}=await reader.read();if(done)break;if(ctx.signal.aborted)throw new DOMException('Language changed','AbortError');chunks.push(value);state.loaded+=value.byteLength;emit(ctx);}} |
| finally{reader.releaseLock();} |
| }else{const b=new Uint8Array(await response.arrayBuffer());chunks.push(b);state.loaded=b.byteLength;} |
| if(ctx.signal.aborted)throw new DOMException('Language changed','AbortError'); |
| state.done=true;emit(ctx); |
| |
| const result=new Response(new Blob(chunks),{headers:{'Content-Type':response.headers.get('content-type')||'application/octet-stream'}}); |
| try{await cache?.put(key.href,result.clone());}catch{} |
| return result; |
| } |
| async function process(m){ |
| const manifest=await manifestPromise; |
| if(controller.signal.aborted)return; |
| const languageCached=engine?.cache.has(m.language),sentenceOnly=Boolean(languageCached&&m.language==='hi'&&m.sentence&&!engine.cache.get('hi').hi.nlm); |
| const needLoad=!languageCached||sentenceOnly; |
| const ctx={id:m.id,language:m.language,sentenceOnly,manifest,signal:controller.signal,files:new Map(),network:0,cached:0};context=ctx; |
| const expected=needLoad?paths(m.language,sentenceOnly):[]; |
| if(m.language==='hi'&&m.sentence&&!sentenceOnly&&!engine?.cache.get('hi')?.hi.nlm)expected.push(...paths('hi',true)); |
| for(const path of expected){const f=manifest[path]||{};ctx.files.set(path,{loaded:0,total:f.bytes||0,kind:f.kind||kindFor(path),done:false});} |
| if(!needLoad){if(m.loading){emit(ctx,'cache');await tick(140);}}else emit(ctx); |
| if(!engine)engine=await MultiIndic.load(assetBase,{language:m.language,fetch:streamedFetch}); |
| else if(engine.language!==m.language)await engine.setLanguage(m.language); |
| if(ctx.signal.aborted)return; |
| await engine.setSentence(m.language==='hi'&&m.sentence); |
| if(ctx.signal.aborted||next)return; |
| if(needLoad||m.loading){emit(ctx,'preparing'); |
| |
| await tick(25); |
| } |
| if(ctx.signal.aborted||next)return; |
| const result=engine.transliterate(m.text,{k:8,overrides:m.overrides||{}}); |
| postMessage({id:m.id,language:m.language,sentence:m.language==='hi'&&Boolean(m.sentence),result,fromCache:!needLoad||ctx.network===0}); |
| } |
| async function pump(){ |
| if(running)return;running=true; |
| while(next){ |
| const m=next;next=null;current=m;controller=new AbortController(); |
| try{await process(m);}catch(e){if(!controller.signal.aborted&&!next){controller.abort();postMessage({id:m.id,error:String(e.message||e)});}} |
| } |
| current=null;controller=null;running=false; |
| } |
| self.onmessage=({data:m})=>{ |
| next=m; |
| if(current&&(current.language!==m.language||Boolean(current.sentence)!==Boolean(m.sentence)))controller?.abort(); |
| pump(); |
| }; |
|
|
