// TYGER wire and capture worklet adapted from nineninesix-ai/tyger-s2s (Apache-2.0). // One AudioContext owns capture AND playback; tags represent DAC playback, not arrival. import {PlaybackClock} from './clock.js'; const WORKLET=` class Mic16k extends AudioWorkletProcessor { constructor(){super();this.step=sampleRate/16000;this.pos=-1;this.prev=0;this.out=new Float32Array(320);this.n=0;this.t0=0;} process(inputs){const x=inputs[0]?.[0];if(!x?.length)return true;const f0=currentFrame; if(this.pos=x.length)break; const a=i<0?this.prev:x[i],f=this.pos-p;if(!this.n)this.t0=this.pos/sampleRate; this.out[this.n++]=a+(x[i+1]-a)*f; if(this.n===320){this.port.postMessage({t:this.t0,x:this.out.slice()});this.n=0;}this.pos+=this.step;} this.prev=x[x.length-1];return true;}} registerProcessor('robo-mic',Mic16k);`; export class RoboClient { constructor(on={}){this.on=on;this.ctx=null;this.ws=null;this.stream=null;this.sources=[];this.next=0;this.queued={};this.epoch=0;this.connectionEpoch=0;this.startPromise=null;this.metrics={mic_blocks:0,tagged_blocks:0,chunks:0,reply_chunks:0};} async start(withMic=true){ if(this.startPromise){await this.startPromise;if(withMic&&!this.stream)return this.start(true);return;} const promise=this._start(withMic,this.connectionEpoch);this.startPromise=promise; try{return await promise;}finally{if(this.startPromise===promise)this.startPromise=null;} } async _start(withMic,connectionEpoch){ try{ if(!this.ctx){this.ctx=new AudioContext({latencyHint:'interactive'});this.clock=new PlaybackClock(this.ctx); this.output=this.ctx.createGain();this.analyser=this.ctx.createAnalyser();this.analyser.fftSize=256; this.output.connect(this.analyser);this.analyser.connect(this.ctx.destination);await this.ctx.resume();} // A typed session may already own a context suspended by the browser. The // microphone button is a fresh user gesture: resume capture as well as TTS. if(this.ctx.state&&this.ctx.state!=='running')await this.ctx.resume(); if(connectionEpoch!==this.connectionEpoch)throw new Error('Conversation start cancelled.'); if(withMic&&!this.stream)await this.microphone(connectionEpoch); if(this.ws?.readyState===1)return; await new Promise((resolve,reject)=>{const u=new URL('/ws',location.href);u.protocol=location.protocol==='https:'?'wss:':'ws:'; const ws=this.ws=new WebSocket(u);ws.binaryType='arraybuffer'; let opened=false; ws.onopen=()=>{if(connectionEpoch!==this.connectionEpoch){ws.close();reject(new Error('Conversation start cancelled.'));return;}opened=true;this.on.connection?.(true);resolve();}; ws.onerror=()=>reject(new Error('Robo is busy or disconnected. End another session or try again shortly.')); ws.onclose=()=>{if(!opened)reject(new Error('Conversation start cancelled or server busy.'));if(this.ws===ws){this.ws=null;this.stop();}}; ws.onmessage=e=>{if(this.ws!==ws)return;if(e.data instanceof ArrayBuffer)this.play(e.data);else{const m=JSON.parse(e.data);if(m.type==='barge')this.stopAudio();this.on.message?.(m);}};}); }catch(error){if(connectionEpoch===this.connectionEpoch)this.stop();throw error;} } async microphone(connectionEpoch){ const stream=await navigator.mediaDevices.getUserMedia({audio:{channelCount:1,echoCancellation:true,noiseSuppression:true,autoGainControl:true}}); if(connectionEpoch!==this.connectionEpoch){stream.getTracks().forEach(t=>t.stop());throw new Error('Microphone start cancelled.');} this.stream=stream; const latency=this.stream.getAudioTracks()[0].getSettings().latency; this.clock.inputLatency=Number.isFinite(latency)?latency:null; const url=URL.createObjectURL(new Blob([WORKLET],{type:'application/javascript'})); try{await this.ctx.audioWorklet.addModule(url);}finally{URL.revokeObjectURL(url);} if(connectionEpoch!==this.connectionEpoch){stream.getTracks().forEach(t=>t.stop());throw new Error('Microphone start cancelled.');} this.micSource=this.ctx.createMediaStreamSource(this.stream);this.worklet=new AudioWorkletNode(this.ctx,'robo-mic'); const silent=this.ctx.createGain();silent.gain.value=0;this.micSource.connect(this.worklet);this.worklet.connect(silent);silent.connect(this.ctx.destination); this.captureStarted=performance.now();this.lastCapture=0;this.captureBlocks=0; this.stream.getAudioTracks()[0].onended=()=>{if(connectionEpoch===this.connectionEpoch){this.on.input?.({state:'ended'});this.stop();}}; this.worklet.port.onmessage=e=>{if(connectionEpoch!==this.connectionEpoch)return; const x=e.data.x;this.captureBlocks++;this.lastCapture=performance.now(); if(!this.lastInputNotice||this.lastCapture-this.lastInputNotice>250){ this.lastInputNotice=this.lastCapture;let square=0;for(const v of x)square+=v*v; this.on.input?.({state:this.ctx.state,rms:Math.sqrt(square/x.length),captured:this.captureBlocks,sent:this.metrics.mic_blocks}); }this.mic(e.data.t,x);};this.on.microphone?.(true); } mic(t,x){ if(this.ws?.readyState!==1)return; // Split at scheduled playback edges instead of mislabeling the remainder // of a 20-ms block when playback begins, stops or has a buffered gap. for(const part of this.clock.pieces(t,x.length)){ const buffer=new ArrayBuffer(8+2*(part.end-part.first)),dv=new DataView(buffer),pcm=new Int16Array(buffer,8); dv.setInt32(0,part.id,true);dv.setInt32(4,part.at,true); for(let i=part.first;i=0)this.metrics.tagged_blocks++; } if(this.metrics.mic_blocks%25===0)this.on.sync?.(this.clock.report()); } play(raw){ const id=new DataView(raw).getInt32(0,true),pcm=new Int16Array(raw,4),b=this.ctx.createBuffer(1,pcm.length,24000),ch=b.getChannelData(0); if(id&&this.sources.some(s=>s.roboId===0)){ for(const s of this.sources.filter(s=>s.roboId===0)){s.onended=null;try{s.stop();}catch{}} this.sources=this.sources.filter(s=>s.roboId!==0);if(!this.sources.length)this.next=0; } for(let i=0;i{this.sources=this.sources.filter(s=>s!==source); if(epoch!==this.epoch)return; if(!id){if(!this.sources.length)this.on.playing?.(false);return;} const complete=()=>{if(epoch!==this.epoch||this.next>end+1e-6)return; if(this.clock.outputHead()+1e-4t.stop());this.stream=null; this.micSource?.disconnect();this.worklet?.disconnect();this.ctx?.close().catch(()=>{});this.ctx=null;this.queued={};this.on.microphone?.(false);this.on.connection?.(false);} level(){if(!this.analyser||this.ctx?.state!=='running')return 0;const x=new Float32Array(256);this.analyser.getFloatTimeDomainData(x);return Math.sqrt(x.reduce((a,b)=>a+b*b,0)/x.length);} }