| |
| |
| import {PlaybackClock} from './clock.js'; |
| const WORKLET=` |
| class Mic16k extends AudioWorkletProcessor { |
| constructor(){super();this.step=sampleRate/16000;this.pos=-1;this.prev=0;this.out=new Float32Array(320);this.n=0;this.t0=0;} |
| process(inputs){const x=inputs[0]?.[0];if(!x?.length)return true;const f0=currentFrame; |
| if(this.pos<f0-1)this.pos=f0; |
| for(;;){const p=Math.floor(this.pos),i=p-f0;if(i+1>=x.length)break; |
| const a=i<0?this.prev:x[i],f=this.pos-p;if(!this.n)this.t0=this.pos/sampleRate; |
| this.out[this.n++]=a+(x[i+1]-a)*f; |
| if(this.n===320){this.port.postMessage({t:this.t0,x:this.out.slice()});this.n=0;}this.pos+=this.step;} |
| this.prev=x[x.length-1];return true;}} |
| registerProcessor('robo-mic',Mic16k);`; |
|
|
| export class RoboClient { |
| constructor(on={}){this.on=on;this.ctx=null;this.ws=null;this.stream=null;this.sources=[];this.next=0;this.queued={};this.epoch=0;this.connectionEpoch=0;this.startPromise=null;this.metrics={mic_blocks:0,tagged_blocks:0,chunks:0,reply_chunks:0};} |
| async start(withMic=true){ |
| if(this.startPromise){await this.startPromise;if(withMic&&!this.stream)return this.start(true);return;} |
| const promise=this._start(withMic,this.connectionEpoch);this.startPromise=promise; |
| try{return await promise;}finally{if(this.startPromise===promise)this.startPromise=null;} |
| } |
| async _start(withMic,connectionEpoch){ |
| try{ |
| if(!this.ctx){this.ctx=new AudioContext({latencyHint:'interactive'});this.clock=new PlaybackClock(this.ctx); |
| this.output=this.ctx.createGain();this.analyser=this.ctx.createAnalyser();this.analyser.fftSize=256; |
| this.output.connect(this.analyser);this.analyser.connect(this.ctx.destination);await this.ctx.resume();} |
| |
| |
| if(this.ctx.state&&this.ctx.state!=='running')await this.ctx.resume(); |
| if(connectionEpoch!==this.connectionEpoch)throw new Error('Conversation start cancelled.'); |
| if(withMic&&!this.stream)await this.microphone(connectionEpoch); |
| if(this.ws?.readyState===1)return; |
| await new Promise((resolve,reject)=>{const u=new URL('/ws',location.href);u.protocol=location.protocol==='https:'?'wss:':'ws:'; |
| const ws=this.ws=new WebSocket(u);ws.binaryType='arraybuffer'; |
| let opened=false; |
| ws.onopen=()=>{if(connectionEpoch!==this.connectionEpoch){ws.close();reject(new Error('Conversation start cancelled.'));return;}opened=true;this.on.connection?.(true);resolve();}; |
| ws.onerror=()=>reject(new Error('Robo is busy or disconnected. End another session or try again shortly.')); |
| ws.onclose=()=>{if(!opened)reject(new Error('Conversation start cancelled or server busy.'));if(this.ws===ws){this.ws=null;this.stop();}}; |
| ws.onmessage=e=>{if(this.ws!==ws)return;if(e.data instanceof ArrayBuffer)this.play(e.data);else{const m=JSON.parse(e.data);if(m.type==='barge')this.stopAudio();this.on.message?.(m);}};}); |
| }catch(error){if(connectionEpoch===this.connectionEpoch)this.stop();throw error;} |
| } |
| async microphone(connectionEpoch){ |
| const stream=await navigator.mediaDevices.getUserMedia({audio:{channelCount:1,echoCancellation:true,noiseSuppression:true,autoGainControl:true}}); |
| if(connectionEpoch!==this.connectionEpoch){stream.getTracks().forEach(t=>t.stop());throw new Error('Microphone start cancelled.');} |
| this.stream=stream; |
| const latency=this.stream.getAudioTracks()[0].getSettings().latency; |
| this.clock.inputLatency=Number.isFinite(latency)?latency:null; |
| const url=URL.createObjectURL(new Blob([WORKLET],{type:'application/javascript'})); |
| try{await this.ctx.audioWorklet.addModule(url);}finally{URL.revokeObjectURL(url);} |
| if(connectionEpoch!==this.connectionEpoch){stream.getTracks().forEach(t=>t.stop());throw new Error('Microphone start cancelled.');} |
| this.micSource=this.ctx.createMediaStreamSource(this.stream);this.worklet=new AudioWorkletNode(this.ctx,'robo-mic'); |
| const silent=this.ctx.createGain();silent.gain.value=0;this.micSource.connect(this.worklet);this.worklet.connect(silent);silent.connect(this.ctx.destination); |
| this.captureStarted=performance.now();this.lastCapture=0;this.captureBlocks=0; |
| this.stream.getAudioTracks()[0].onended=()=>{if(connectionEpoch===this.connectionEpoch){this.on.input?.({state:'ended'});this.stop();}}; |
| this.worklet.port.onmessage=e=>{if(connectionEpoch!==this.connectionEpoch)return; |
| const x=e.data.x;this.captureBlocks++;this.lastCapture=performance.now(); |
| if(!this.lastInputNotice||this.lastCapture-this.lastInputNotice>250){ |
| this.lastInputNotice=this.lastCapture;let square=0;for(const v of x)square+=v*v; |
| this.on.input?.({state:this.ctx.state,rms:Math.sqrt(square/x.length),captured:this.captureBlocks,sent:this.metrics.mic_blocks}); |
| }this.mic(e.data.t,x);};this.on.microphone?.(true); |
| } |
| mic(t,x){ |
| if(this.ws?.readyState!==1)return; |
| |
| |
| for(const part of this.clock.pieces(t,x.length)){ |
| const buffer=new ArrayBuffer(8+2*(part.end-part.first)),dv=new DataView(buffer),pcm=new Int16Array(buffer,8); |
| dv.setInt32(0,part.id,true);dv.setInt32(4,part.at,true); |
| for(let i=part.first;i<part.end;i++)pcm[i-part.first]=Math.max(-32768,Math.min(32767,Math.round(x[i]*32767))); |
| this.ws.send(buffer);this.metrics.mic_blocks++;if(part.at>=0)this.metrics.tagged_blocks++; |
| } |
| if(this.metrics.mic_blocks%25===0)this.on.sync?.(this.clock.report()); |
| } |
| play(raw){ |
| const id=new DataView(raw).getInt32(0,true),pcm=new Int16Array(raw,4),b=this.ctx.createBuffer(1,pcm.length,24000),ch=b.getChannelData(0); |
| if(id&&this.sources.some(s=>s.roboId===0)){ |
| for(const s of this.sources.filter(s=>s.roboId===0)){s.onended=null;try{s.stop();}catch{}} |
| this.sources=this.sources.filter(s=>s.roboId!==0);if(!this.sources.length)this.next=0; |
| } |
| for(let i=0;i<pcm.length;i++)ch[i]=pcm[i]/32768; |
| const source=this.ctx.createBufferSource();source.roboId=id;source.buffer=b;source.connect(this.output); |
| |
| const lead=Math.max(128/this.ctx.sampleRate,(this.ctx.baseLatency||0)*1.25); |
| const start=Math.max(this.next,this.ctx.currentTime+lead),end=start+b.duration; |
| const first=this.queued[id]||0;this.queued[id]=first+pcm.length; |
| if(id)this.clock.add(id,start,pcm.length,first); |
| const epoch=this.epoch;this.next=end;source.start(start);this.sources.push(source);this.metrics.chunks++;if(id)this.metrics.reply_chunks++; |
| this.on.playing?.(true); |
| source.onended=()=>{this.sources=this.sources.filter(s=>s!==source); |
| if(epoch!==this.epoch)return; |
| if(!id){if(!this.sources.length)this.on.playing?.(false);return;} |
| const complete=()=>{if(epoch!==this.epoch||this.next>end+1e-6)return; |
| if(this.clock.outputHead()+1e-4<end){setTimeout(complete,10);return;} |
| this.send('played:'+id+':'+this.queued[id]);this.on.playing?.(false);};complete();}; |
| } |
| send(text){if(this.ws?.readyState===1)this.ws.send(text);} |
| stopAudio(){this.epoch++;for(const s of this.sources){s.onended=null;try{s.stop();}catch{}} |
| this.sources=[];this.next=0;this.clock?.stop();this.on.playing?.(false);} |
| stop(){this.connectionEpoch++;this.startPromise=null;this.stopAudio();const ws=this.ws;this.ws=null;ws?.close();this.stream?.getTracks().forEach(t=>t.stop());this.stream=null; |
| this.micSource?.disconnect();this.worklet?.disconnect();this.ctx?.close().catch(()=>{});this.ctx=null;this.queued={};this.on.microphone?.(false);this.on.connection?.(false);} |
| level(){if(!this.analyser||this.ctx?.state!=='running')return 0;const x=new Float32Array(256);this.analyser.getFloatTimeDomainData(x);return Math.sqrt(x.reduce((a,b)=>a+b*b,0)/x.length);} |
| } |
|
|
