| import {RoboClient} from './client.js'; |
| import {createRobbyRig,controlsFromFace} from './rig.js'; |
| import {profiles,pose,mix,ActingTimeline} from './emotions.js'; |
| const $=id=>document.getElementById(id); |
| let ready=false,connected=false,mic=false,speaking=false,partial=null,answer=null; |
| let desiredGate=true,lastServerAudio=0,lastCapture=0,micStarted=0,serverAudioSeconds=0; |
| let desiredUtteranceThreshold=.50,desiredFastThreshold=null; |
| let cameraStream=null,landmarker=null,lastVideo=-1,face={}; |
| const deskView=document.body.dataset.robotView==='desk3d'; |
| let rig; |
| if(deskView){const {createDeskRobotRig}=await import('./desk-robot3d.js');rig=createDeskRobotRig($('avatar'));} |
| else{const avatar=await fetch('/static/robby.svg').then(r=>r.text());$('avatar').innerHTML=avatar;rig=createRobbyRig($('avatar').querySelector('svg'));} |
| const timeline=new ActingTimeline(); |
| for(const p of profiles){const o=document.createElement('option');o.value=p.slug;o.textContent=p.label;$('expression').append(o);} |
| window.robotFace={rig,timeline,profiles,pose}; |
| if(deskView){const {attachDeskControls}=await import('./desk-controls.js');attachDeskControls(rig);} |
| $('acting').onchange=()=>timeline.enabled=$('acting').checked; |
| $('robot-keypoints').onchange=()=>rig.showKeypoints($('robot-keypoints').checked); |
| function phase(text,kind=''){ $('phase-text').textContent=text;$('phase').className='phase '+kind; } |
| function bubble(text,role){$('chat').querySelector('.welcome')?.remove();const e=document.createElement('div');e.className='bubble '+role;e.textContent=text;$('chat').append(e);$('chat').scrollTop=$('chat').scrollHeight;return e;} |
| const client=window.robo=new RoboClient({ |
| connection(on){connected=on;if(!on){mic=false;gate.checked=desiredGate;identityModel.value='v4-long';timeout.value=900;$('speaker-chips').replaceChildren();$('call').textContent='◉ Start talking';$('call').classList.remove('active');phase(ready?'Ready when you are':'Connecting…','ready');}}, |
| microphone(on){mic=on;micStarted=on?performance.now():0;lastCapture=lastServerAudio=0;serverAudioSeconds=0;$('mic-level').value=0;$('mic-status').textContent=on?'Microphone started. Waiting for audio frames…':'Microphone off. Click Start talking and allow microphone access.';$('call').textContent=on?'■ End conversation':'◉ Start talking';$('call').classList.toggle('active',on);phase(on?'Listening to you':'Ready when you are',on?'ready':'');}, |
| input(s){lastCapture=performance.now();$('mic-level').value=Math.min(1,(s.rms||0)*12);$('mic-status').textContent=s.state==='ended'?'The microphone was disconnected. Start talking again.':`${s.rms>.001?'Audio detected':'Microphone active · quiet input'} · ${s.sent} packets sent · ${serverAudioSeconds.toFixed(1)} s received by server.`;}, |
| playing(on){speaking=on;phase(on?'Robby is speaking':mic?'Listening to you':'Ready when you are',on?'speaking':'ready');}, |
| sync(r){$('sync-status').textContent=`Audio sync: ${r.method} · output ${Math.round(r.output_ms)} ms · input ${r.input_ms===null?'unknown':Math.round(r.input_ms)+' ms'}${r.uncertain?' · device estimate incomplete':''}. No fixed playback offset.`;}, |
| message(m){ |
| if(m.type==='ready'){phase(mic?(desiredGate?'Say Hey Robbie to join':'Listening to you'):'Ready when you are','ready');if(m.identity_mode)identityModel.value=m.identity_mode;if(m.identity_enabled!==desiredGate)client.send('identity:'+(desiredGate?'on':'off'));if(desiredFastThreshold!==null)client.send('identity:threshold:'+desiredFastThreshold);if(desiredUtteranceThreshold!==.50)client.send('identity:utterance_threshold:'+desiredUtteranceThreshold);gate.checked=desiredGate;$('asr-status').textContent=desiredGate?'Speaker filter ON. Say “Hey Robbie, ich bin …” to join. Recognition previews below are not sent to Luna.':'Speaker filter OFF. Speak normally; TYGER submits your turn after a pause.';} |
| if(m.type==='input_status'){lastServerAudio=performance.now();serverAudioSeconds=m.seconds;} |
| if(m.type==='asr_preview'&&m.text?.trim()&&!speaking&&!(m.source==='speaker_candidate'&&m.status!=='enrolled'&&$('speaker-chips').childElementCount)){$('asr-preview').textContent=m.text;$('asr-status').textContent=m.status==='checking_identity'?'Speech recognized · checking speaker identity…':m.status==='enrolled'?'Voice enrolled · finish your sentence, then pause.':$('speaker-chips').childElementCount?'Speech recognition preview · not yet submitted.':'Speech recognized (not submitted). Say “Hey Robbie, ich bin …”, or use Talk without speaker filter.';} |
| if(m.type==='partial'){if(!partial)partial=bubble(m.text,'user partial');else partial.textContent=m.text;$('asr-preview').textContent=m.text;$('asr-status').textContent='Speech recognized · waiting for the end of your turn.';} |
| if(m.type==='endpoint'){const text=(m.speaker_name?m.speaker_name+': ':'')+m.transcript;if(partial){partial.classList.remove('partial');partial.textContent=text;partial=null;}else if(m.transcript)bubble(text,'user');answer=null;$('asr-preview').textContent=m.transcript;$('asr-status').textContent='Turn submitted'+(m.speaker_name?' · '+m.speaker_name:'')+'.';phase('Thinking of a reply');} |
| if(m.type==='reply'){timeline.set(m.reply_id,m.performance);if(!answer)answer=bubble(m.text,'assistant');else answer.textContent=m.text;$('subtitle').textContent=m.text.length>140?m.text.slice(0,137)+'…':m.text;} |
| if(m.type==='state'){$('end-meter').value=m.p_end||0;$('barge-meter').value=m.yield||0;} |
| if(m.type==='barge'){timeline.clear();answer=null;phase('I’m listening','ready');$('subtitle').textContent='Go ahead. You can interrupt me naturally.';} |
| if(m.type==='turn_done'){answer=null;phase(mic?'Listening to you':'Ready when you are','ready');} |
| if(m.type==='error'){const text=m.error==='audio_too_fast'?'The voice server fell behind the microphone. End the conversation and reconnect; if it happens again, the shared GPU service needs checking.':m.error;phase('Something needs attention');$('subtitle').textContent=text;$('asr-status').textContent='Audio processing error: '+text;} |
| if(m.type==='ignored'){phase('Waiting for an enrolled speaker');$('subtitle').textContent=m.reason;$('asr-status').textContent=m.reason;partial=null;} |
| if(m.type==='identity'){ |
| if(typeof m.enabled==='boolean'){gate.checked=m.enabled;desiredGate=m.enabled;} |
| if(m.mode)identityModel.value=m.mode; |
| const p=m.decision||m.prediction||{}; |
| if(typeof m.threshold==='number')strictness.value=m.threshold; |
| if(typeof m.utterance_threshold==='number'){utteranceStrictness.value=m.utterance_threshold;$('utterance-threshold-value').value=m.utterance_threshold.toFixed(2);} |
| $('speaker-chips').replaceChildren(...(m.speakers||[]).map(s=>{const e=document.createElement('span');e.className='speaker-chip';e.textContent=s.name+(s.reference_ready?'':' · joining…');return e;})); |
| $('identity-status').textContent=`Speaker cue: ${p.name||p.kind||'waiting'}${p.similarity==null?'':' · cosine '+p.similarity.toFixed(2)}. Enrolled: ${(m.users||[]).join(', ')||'none — say Hey Robbie, I’m …'}. ${m.enabled?'Filter ON · '+(m.diarization_active?'Nemotron diarization + V4 · ':'embedding-only · ')+'submit after ~1 s without this speaker.':'Filter off: original TYGER endpointing.'} RAM only; not authentication.`; |
| } |
| if(m.type==='warning'){$('identity-status').textContent=m.text;$('asr-status').textContent=m.text;} |
| if(m.type==='busy'){$('subtitle').textContent=m.text;} |
| if(m.type==='service_status'){phase(m.stage==='preparing'?'Preparing a spoken reply':'Waiting for the shared voice model');$('subtitle').textContent=m.text;$('asr-status').textContent=m.text;} |
| if(m.type==='calibration'){ |
| if(m.available){client.clock.acousticLatency=m.delay_ms/1000;$('sync-status').textContent=`Measured extra echo delay: ${m.delay_ms.toFixed(1)} ms · correlation ${m.confidence.toFixed(2)}.`;} |
| else $('sync-status').textContent='Echo calibration uncertain: '+m.reason+'. Keeping device-timestamp mapping; no extra delay guessed.'; |
| } |
| } |
| }); |
| $('call').onclick=async()=>{try{if(mic)client.stop();else await client.start(true);}catch(e){$('subtitle').textContent=e.message;phase('Microphone unavailable — try typing');}}; |
| $('unfiltered-talk').onclick=async()=>{desiredGate=false;try{await client.start(true);client.send('identity:off');gate.checked=false;$('asr-status').textContent='Speaker filter OFF. Speak normally; background voices may also be transcribed.';phase('Listening to you','ready');}catch(e){$('mic-status').textContent='Microphone unavailable: '+e.message;}}; |
| setInterval(()=>{if(!mic)return;const now=performance.now();if(now-micStarted<2500)return; |
| if(!lastCapture||now-lastCapture>2500)$('mic-status').textContent=`No audio frames from the microphone (${client.ctx?.state||'no audio context'}). Check browser permission/input device, then End conversation and Start talking.`; |
| else if(!lastServerAudio||now-lastServerAudio>4000)$('mic-status').textContent='Microphone captures audio, but the server has not acknowledged recent packets. Reconnect the conversation.'; |
| },1000); |
| $('interrupt').onclick=()=>{timeline.clear();client.stopAudio();client.send('interrupt');}; |
| const calibrate=document.createElement('button');calibrate.className='text-button';calibrate.textContent='Calibrate acoustic echo';calibrate.onclick=async()=>{try{await client.start(true);client.clock.acousticLatency=0;client.send('calibrate');$('sync-status').textContent='Listening for a short, quiet calibration tone…';}catch(e){$('sync-status').textContent=e.message;}};$('sync-status').after(calibrate); |
| async function submit(text){if(!ready)return;try{await client.start(false);bubble(text,'user');answer=null;client.send('text:'+text);phase('Thinking of a reply');}catch(e){$('subtitle').textContent=e.message;}} |
| $('form').onsubmit=e=>{e.preventDefault();const text=$('message').value.trim();if(text){$('message').value='';submit(text);}}; |
| for(const b of document.querySelectorAll('[data-text]'))b.onclick=()=>submit(b.dataset.text); |
| $('clear').onclick=()=>{timeline.clear();client.stop();$('chat').replaceChildren();partial=answer=null;}; |
| const gateLabel=document.createElement('label'),gate=document.createElement('input');gate.type='checkbox';gate.id='identity-gate';gate.disabled=true;gate.checked=true;gateLabel.append(gate,' Recognize enrolled speakers · ignore background turns');$('speaker-controls').append(gateLabel); |
| const modelLabel=document.createElement('label');modelLabel.textContent='Speaker recognition ';const identityModel=document.createElement('select');identityModel.id='identity-model';for(const [value,label] of [['v4-long','V4 · 320 ms + 2-s reference confirmation'],['v4','V4 · 320-ms layer-mix only'],['v4-long3','V4 + optimized 3.04 s (training pending)'],['ecapa','ECAPA comparison · legacy endpoint gate']]){const o=document.createElement('option');o.value=value;o.textContent=label;o.disabled=value==='v4-long3';identityModel.append(o);}identityModel.value='v4-long';modelLabel.append(identityModel);gateLabel.after(modelLabel);identityModel.onchange=async()=>{const desired=identityModel.value;try{await client.start(true);client.send('identity:mode:'+desired);gate.checked=desired.startsWith('v4');gate.disabled=false;}catch(e){$('identity-status').textContent=e.message;}}; |
| gate.onchange=async()=>{const enabled=gate.checked;desiredGate=enabled;try{await client.start(true);client.send('identity:'+(enabled?'on':'off'));$('asr-status').textContent=enabled?'Speaker filter ON. Say “Hey Robbie, ich bin …” to join.':'Speaker filter OFF. Speak normally; background voices may also be transcribed.';}catch(e){gate.checked=false;desiredGate=false;$('identity-status').textContent=e.message;}}; |
| const timeoutLabel=document.createElement('label');timeoutLabel.textContent='Forget inactive users after ';const timeout=document.createElement('select');for(const [value,label] of [[60,'1 minute'],[300,'5 minutes'],[900,'15 minutes'],[0,'this conversation']]){const o=document.createElement('option');o.value=value;o.textContent=label;timeout.append(o);}timeout.value=900;timeout.onchange=()=>client.send('identity:timeout:'+timeout.value);timeoutLabel.append(timeout);modelLabel.after(timeoutLabel); |
| const strictLabel=document.createElement('label');strictLabel.textContent='Fast cue / enrollment stability ';const strictness=document.createElement('input');strictness.id='speaker-strictness';strictness.type='range';strictness.min=.1;strictness.max=.95;strictness.step=.01;strictness.value=.62;strictness.onchange=()=>{desiredFastThreshold=+strictness.value;client.send('identity:threshold:'+strictness.value);};strictLabel.append(strictness);timeoutLabel.after(strictLabel); |
| const utteranceLabel=document.createElement('label');utteranceLabel.textContent='Full-sentence voice match (higher rejects more) ';const utteranceStrictness=document.createElement('input');utteranceStrictness.id='utterance-strictness';utteranceStrictness.type='range';utteranceStrictness.min=.3;utteranceStrictness.max=.9;utteranceStrictness.step=.01;utteranceStrictness.value=.5;const utteranceValue=document.createElement('output');utteranceValue.id='utterance-threshold-value';utteranceValue.value='0.50';utteranceStrictness.oninput=()=>utteranceValue.value=(+utteranceStrictness.value).toFixed(2);utteranceStrictness.onchange=()=>{desiredUtteranceThreshold=+utteranceStrictness.value;client.send('identity:utterance_threshold:'+utteranceStrictness.value);};utteranceLabel.append(utteranceStrictness,utteranceValue);strictLabel.after(utteranceLabel); |
| const forget=document.createElement('button');forget.className='text-button';forget.textContent='Forget enrolled users';forget.onclick=()=>client.send('identity:clear');timeoutLabel.after(forget); |
| const studyLink=document.createElement('a');studyLink.href='/speaker-study/';studyLink.target='_blank';studyLink.rel='noopener';studyLink.textContent='Speaker predictor study ↗';$('identity-status').after(studyLink); |
| const overview=document.createElement('a');overview.href='/static/report.html';overview.target='_blank';overview.rel='noopener';overview.textContent='Implementation, tests and limitations ↗';studyLink.after(document.createElement('br'),overview); |
| const identityHelp=document.createElement('p');identityHelp.textContent='Default: Nemotron 3 streaming diarization + the completed V4 layer-mix predictor and the best completed two-second reference. V4-only is available as an ablation; a completed three-second reference can also be selected. Names/voice memory are session-only; the LLM cannot enroll an unrelated voice by inventing a name. ECAPA is a legacy endpoint comparison.';studyLink.after(identityHelp); |
| async function poll(){try{const s=await fetch('/api/state').then(r=>r.json());ready=s.ready;gate.disabled=!s.identity_available;$('call').disabled=!ready;const long=s.identity_models?.long,long3=s.identity_models?.long3;identityModel.querySelector('[value="v4-long3"]').disabled=!long3;if(long)identityModel.querySelector('[value="v4-long"]').textContent='V4 320 ms + '+long.label;if(long3)identityModel.querySelector('[value="v4-long3"]').textContent='V4 320 ms + '+long3.label;if(!connected){phase(s.ready?'Ready when you are':s.error||s.phase,s.ready?'ready':'');$('identity-status').textContent='Speaker extension: '+s.identity_model+'. Voiceprints stay in RAM for this conversation only.';}}catch{phase('Reconnecting…');}} |
| poll();setInterval(poll,10000); |
|
|
| $('camera').onclick=async()=>{ |
| if(cameraStream){cameraStream.getTracks().forEach(t=>t.stop());cameraStream=null;$('video').hidden=true;$('mesh').hidden=true;$('camera').textContent='Camera off';$('camera-status').textContent='Camera off; no video is sent to the server.';face={};return;} |
| try{ |
| $('camera-status').textContent='Loading local face tracking…'; |
| if(!landmarker){const {FilesetResolver,FaceLandmarker}=await import('https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.21/vision_bundle.mjs'); |
| const files=await FilesetResolver.forVisionTasks('https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.21/wasm'); |
| landmarker=await FaceLandmarker.createFromOptions(files,{baseOptions:{modelAssetPath:'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',delegate:'CPU'},runningMode:'VIDEO',numFaces:1,outputFaceBlendshapes:true,outputFacialTransformationMatrixes:true});} |
| cameraStream=await navigator.mediaDevices.getUserMedia({video:{width:640,height:480,facingMode:'user'},audio:false}); |
| $('video').srcObject=cameraStream;await $('video').play();$('video').hidden=false;$('camera').textContent='Camera on';$('camera-status').textContent='Tracking locally. Camera frames and landmarks never leave this browser.'; |
| }catch(e){$('camera-status').textContent='Camera tracking unavailable: '+e.message;} |
| }; |
| function observeFace(){ |
| const video=$('video');if(!cameraStream||!landmarker||video.readyState<2||video.currentTime===lastVideo)return; |
| lastVideo=video.currentTime; |
| const r=landmarker.detectForVideo(video,performance.now()),points=r.faceLandmarks?.[0]; |
| if(!points){$('camera-status').textContent='No face visible. Tracking is still local.';return;} |
| face=controlsFromFace(r); |
| const canvas=$('mesh');canvas.hidden=!$('landmarks').checked; |
| if(!canvas.hidden){canvas.width=320;canvas.height=240;const c=canvas.getContext('2d');c.clearRect(0,0,320,240);c.fillStyle='#b88a50';for(const p of points){c.beginPath();c.arc((1-p.x)*320,p.y*240,1,0,Math.PI*2);c.fill();}} |
| } |
| let lastFace=0,lastAnimation=0; |
| function animate(t){ |
| if(t-lastFace>80){try{observeFace();}catch{}lastFace=t;} |
| const cue=timeline.at(client.ctx?.state==='running'?client.clock?.position():null); |
| const preview=$('expression').value,acting=preview==='auto'&&cue; |
| const follows=cameraStream&&$('mirror').checked&&(deskView||!speaking),controls=follows?(deskView&&speaking?{x:face.x,y:face.y}:face):{}; |
| const idle=deskView?rig.idlePose():pose(); |
| const target=preview!=='auto'?pose(preview):acting?cue.pose:follows?(deskView?mix(idle,pose('amusement',.65),Math.min(.65,face.smile||0)):pose('amusement',Math.min(.5,face.smile||0))):idle; |
| const chosen=mix(pose(),target,Number($('face-intensity').value)); |
| const description=preview!=='auto'?'Silent preview: '+preview.replaceAll('_',' '):acting? |
| `${cue.emotion.replaceAll('_',' ')}${cue.secondary?' + '+cue.secondary.replaceAll('_',' '):''} · sentence ${cue.index+1} · Luna SCRIPT · estimated sentence timing`: |
| 'Luna’s sentence-level acting directions drive the face. Audio only moves the mouth.'; |
| if($('acting-status').textContent!==description)$('acting-status').textContent=description; |
| const automaticBlink=Math.pow(Math.max(0,Math.cos(t/900)),160)*.85; |
| rig.update({...controls,pose:chosen,blinkLeft:Math.max(controls.blinkLeft||0,automaticBlink),blinkRight:Math.max(controls.blinkRight||0,automaticBlink),audioLevel:Math.min(1,client.level()*7)},lastAnimation?t-lastAnimation:16.7,matchMedia('(prefers-reduced-motion: reduce)').matches?0:t);lastAnimation=t; |
| requestAnimationFrame(animate); |
| } |
| requestAnimationFrame(animate); |
|
|
