| |
| import {BhashaCore, XlitModel, parseWeights, TOKEN_RE} from './aksharalp.mjs'; |
| const norm=s=>s.replace(/[\u200c\u200d]/g,'').normalize('NFC'); |
| const protectedTokens=/(https?:\/\/\S+|www\.\S+|[\w.+-]+@[\w.-]+\.[A-Za-z]+|[@#][\p{L}\p{N}_]+)|[A-Za-z]+/gu; |
| async function decompress(b) { |
| if (typeof process !== 'undefined' && process.versions?.node) {const {gunzipSync}=await import('node:zlib');const x=gunzipSync(new Uint8Array(b));return x.buffer.slice(x.byteOffset,x.byteOffset+x.byteLength);} |
| if (!globalThis.DecompressionStream) throw new Error('This browser needs DecompressionStream (Chrome/Edge 80+, Firefox 113+, Safari 16.4+).'); |
| return new Response(new Blob([b]).stream().pipeThrough(new DecompressionStream('gzip'))).arrayBuffer(); |
| } |
| function rerank(cands,lex,p) { |
| const rr=p.reranker, unis=cands.map(([s])=>lex.get(s)), mu=Math.max(...unis.filter(x=>x!==undefined),-Infinity), msc=Math.max(...cands.map(c=>c[1])); |
| return cands.map(([text,sc],i)=>{const u=unis[i],inside=+(u!==undefined),f=[sc,sc/(text.length+1),inside,inside?u:0,+(i===0),i/cands.length,sc-msc,inside?u-mu:0]; |
| const score=f.reduce((a,x,j)=>a+(x-rr.mu[j])/rr.sd[j]*rr.w[j],0);return {text,score};}).sort((a,b)=>b.score-a.score); |
| } |
| export class MultiIndic { |
| static async load(base='./', {language='hi',sentence=false,fetch:fetchImpl=globalThis.fetch.bind(globalThis),onProgress}={}) { |
| |
| const e=new MultiIndic();e.base=base.replace(/\/$/,'');e.fetch=fetchImpl;e.onProgress=onProgress;e.cache=new Map();e.pending=new Map();e.sentenceMode=false;await e.setLanguage(language);if(sentence)await e.setSentence(true);return e; |
| } |
| async bytes(path) {const r=await this.fetch(`${this.base}/${path}`);if(!r.ok)throw new Error(`Could not load ${path}: HTTP ${r.status}`);let b=await r.arrayBuffer();this.onProgress?.(path,b.byteLength);return path.endsWith('.gz')?decompress(b):b;} |
| async compressedFetch(url) {const path=url.slice(this.base.length+1),b=await this.bytes(path+'.gz');return new Response(b);} |
| async setLanguage(language) { |
| if(!['hi','bn','te','kn','ta'].includes(language))throw new Error('Unsupported language'); |
| if(!this.cache.has(language)) { |
| if(!this.pending.has(language))this.pending.set(language,(async()=>{ |
| if(language==='hi'){const hi=await BhashaCore.load(`${this.base}/models/hi`,{fetch:u=>this.compressedFetch(u),cache:false,dict:true});if(!hi.hasDict)throw new Error(hi.dictError||'Hindi dictionary failed to load');return {hi};} |
| const prefix=`models/${language}/`,[mb,pb,lb]=await Promise.all([this.bytes(prefix+'xlit.q4.gz'),this.bytes(prefix+'params.json'),this.bytes(prefix+'dict.tsv.gz')]); |
| const p=JSON.parse(new TextDecoder().decode(pb)),lex=new Map(); |
| for(const line of new TextDecoder().decode(lb).trim().split('\n')){const j=line.lastIndexOf('\t');lex.set(line.slice(0,j),Math.log(Number(line.slice(j+1))/p.dictionary_total));} |
| return {model:new XlitModel(parseWeights(mb)),p,lex,memo:new Map()}; |
| })()); |
| try {this.cache.set(language,await this.pending.get(language));} |
| finally {this.pending.delete(language);} |
| } |
| this.language=language;this.active=this.cache.get(language);this.sentenceMode=false;if(this.active.hi)this.active.hi.sentenceMode=false;return this; |
| } |
| async setSentence(enabled=true) { |
| if(this.language!=='hi'&&enabled)throw new Error('Sentence mode is Hindi-only'); |
| if(enabled&&!this.active.hi.nlm) {const [{attachSentence},bytes]=await Promise.all([import('./sent.mjs'),this.bytes('models/hi/sentence/nlm.q8.gz')]);attachSentence(this.active.hi,bytes);} |
| this.sentenceMode=enabled;if(this.active.hi)this.active.hi.sentenceMode=enabled;return this; |
| } |
| word(word,k=8) { |
| if(!/^[A-Za-z]+$/.test(word))return [{text:word,score:0}]; |
| const a=this.active;if(a.hi)return a.hi.word(word,k); |
| word=word.toLowerCase();if(!a.memo.has(word)){ |
| const ids=[...word.slice(0,31)].map(c=>c.charCodeAt(0)-97+4).concat([2]),enc=a.model.encode(ids),seen=new Set(),cands=[]; |
| for(const [s,sc] of a.model.beam(enc,ids.length,{beam:8,nbest:8})){const t=norm(s);if(t&&!seen.has(t)){seen.add(t);cands.push([t,sc]);}} |
| if(a.memo.size>1000)a.memo.clear();a.memo.set(word,rerank(cands,a.lex,a.p)); |
| } |
| return a.memo.get(word).slice(0,k); |
| } |
| transliterate(text,{k=8,overrides={}}={}) { |
| const t0=performance.now(); |
| if(this.active.hi){ |
| |
| const h=this.active.hi,spans=[...text.matchAll(TOKEN_RE)].map(m=>[m.index,m.index+m[0].length,m[0]]); |
| const slots=spans.map(([a,b,t])=>overrides[a]!==undefined?['fix',overrides[a]]:/^[A-Za-z]+$/.test(t)?['x',t]:['fix',t]); |
| const r=h.decodeSlots(slots,this.sentenceMode);if(r.out.at(-1)==='.')r.out[r.out.length-1]='\u0964'; |
| let output='',pos=0;const tokens=[]; |
| spans.forEach(([a,b,t],i)=>{output+=text.slice(pos,a)+r.out[i];pos=b;tokens.push({text:t,start:a,end:b,output:r.out[i],candidates:slots[i][0]==='x'?h.word(t,k):[]});}); |
| return {output:output+text.slice(pos),tokens,ms:performance.now()-t0,sentence:null,guard:r.guard||{ok:true,words:0,outside:0}}; |
| } |
| const tokens=[];const output=text.replace(protectedTokens,(word,protectedWord,offset)=>{ |
| if(protectedWord)return word;const candidates=this.word(word,k),out=overrides[offset]??candidates[0]?.text??word; |
| tokens.push({text:word,start:offset,end:offset+word.length,output:out,candidates});return out;}); |
| return {output,tokens,ms:performance.now()-t0,sentence:null}; |
| } |
| clearCache(){if(this.active.memo)this.active.memo.clear();if(this.active.hi)this.active.hi.cache.clear();} |
| } |
| export async function loadFromDir(dir,opt={}) {const fs=await import('node:fs/promises');return MultiIndic.load(dir,{...opt,fetch:async u=>{try{return new Response(await fs.readFile(u));}catch{return new Response(null,{status:404});}}});} |
| export default MultiIndic; |
|
|
