| import{corticalSurface}from'./cortical-surface.js?v=brain-tribe-3'; |
| import{$,api,state,escape,toast,action,card,probabilities,fileImage,download,speechButton,speak,activity}from'./common.js'; |
|
|
| function sampleCard(word){const c=document.createElement('canvas');c.width=c.height=320;const ctx=c.getContext('2d');ctx.fillStyle='#ecf1df';ctx.fillRect(0,0,320,320);ctx.fillStyle='#547b5d';ctx.font='80px serif';ctx.textAlign='center';const shapes={dinosaur:'🦕',basketball:'🏀',cafeteria:'☕',moon:'☾'};ctx.fillText(shapes[word]||'✦',160,172);ctx.font='20px sans-serif';ctx.fillText(word,160,235);return c.toDataURL('image/png')} |
|
|
| export async function dream(root){root.innerHTML=card('A shelf of little experiences',`<label class="upload">＋ Add an image memory<br><small>Its caption and image embedding are produced locally.</small><input type="file" id="memory-upload" accept="image/*" multiple></label><div class="toolbar"><span class="note">Or start with a supplied memory card:</span><div class="group"><button class="secondary" data-memory="dinosaur">Dinosaur</button><button class="secondary" data-memory="basketball">Basketball</button><button class="secondary" data-memory="cafeteria">Cafeteria</button></div></div><div class="memory-grid" id="memories"></div><p class="note">Check the memories to emphasize. Leave them unchecked to retrieve up to three by CLIP similarity to the theme. Supplied cards use their supplied captions; uploaded photos are captioned by BLIP.</p>`,'IMAGE + MEMORY')+card('Let the brain dream',`<div class="field-row"><div class="field"><label for="dream-theme">DREAM THEME</label><input id="dream-theme" type="text" value="a magical floating world"></div><div class="field" style="max-width:90px"><label for="dream-seed">SEED</label><input id="dream-seed" type="number" min="0" max="2147483647" value="21" style="width:100%;padding:9px;border:1px solid #e1e5d8;border-radius:5px"></div></div><button class="primary" id="generate-dream">☾ Enter night mode & dream</button><p class="note">Memory retrieval → language-model scene plan → local SD-Turbo generation. Change the theme or selected memories to make a second version.</p><div id="dream-output"></div>`,'ACTUAL IMAGE GENERATION');let alive=true,list=[],dreams=[]; |
| async function refresh(){list=await api('memories');if(!alive)return;$('#memories').innerHTML=list.length?list.map(m=>`<label class="memory-card"><input type="checkbox" value="${m.id}" aria-label="Select ${escape(m.caption)}"><img src="${m.image}" alt="${escape(m.caption)}"><p>${escape(m.caption)}</p></label>`).join(''):'<div class="empty">A blank memory shelf. Add an experience to begin.</div>'} |
| $('#memory-upload').onchange=async e=>{const files=[...e.target.files];for(const file of files){try{toast('Encoding a new image memory…');const image=await fileImage(file);await api('memories',{image});activity(['occipital','hippocampus']);await refresh()}catch(err){toast(err.message,true);break}}};document.querySelectorAll('[data-memory]').forEach(b=>b.onclick=()=>action(b,async()=>{await api('memories',{image:sampleCard(b.dataset.memory),caption:b.dataset.memory});await refresh();activity(['hippocampus'])},'…')); |
| $('#generate-dream').onclick=()=>action($('#generate-dream'),async()=>{const ids=[...document.querySelectorAll('#memories input:checked')].map(x=>x.value);$('#dream-output').innerHTML='<p class="message">Retrieving memories, planning a scene, and generating an image locally…</p>';try{const r=await api('dream',{theme:$('#dream-theme').value,ids,seed:Number($('#dream-seed').value)});if(!alive)return;dreams.push(r);$('#dream-output').innerHTML=dreams.slice(-2).map((d,i)=>`<div style="margin-top:20px"><div class="card-title"><h2>Dream ${dreams.length-dreams.slice(-2).length+i+1}</h2><span class="badge">Seed ${d.seed}</span></div><img class="result-image" src="${d.image}" alt="${escape(d.prompt)}"><p class="caption"><b>Scene plan:</b> ${escape(d.prompt)}</p><p class="caption"><b>Retrieved memories:</b> ${d.memories.map(m=>escape(m.caption)).join(' · ')}</p><button class="secondary" data-save-dream="${i}">Download this dream ↗</button></div>`).join('');document.querySelectorAll('[data-save-dream]').forEach(b=>b.onclick=()=>download(dreams.slice(-2)[Number(b.dataset.saveDream)].image,'brain-lab-dream.png'));activity(['association','frontal','pineal'])}catch(e){if(alive)$('#dream-output').innerHTML=`<p class="message">${escape(e.message)}</p>`;throw e}},'Dreaming locally…');await refresh();return()=>alive=false} |
|
|
| export async function telephone(root){root.innerHTML=card('Start with an idea',`<div class="field"><label for="telephone-text">WHAT SHOULD THE FIRST PICTURE SHOW?</label><textarea id="telephone-text">A cat wearing a crown and riding a skateboard.</textarea></div><div class="button-row"><button class="secondary" id="telephone-mic">◉ Speak</button><select id="telephone-rounds" aria-label="Number of Telephone rounds" style="width:125px"><option value="1">1 round</option><option value="2">2 rounds</option><option value="3" selected>3 rounds</option></select><button class="primary" id="telephone-run">Pass the idea along ↗</button></div><p class="note">The recognizer is BLIP. It receives only the generated image. The original sentence, prompt, and previous captions are never included in its input.</p>`,'NO PROMPT LEAKAGE')+card('Follow the message',`<div class="inference-row" style="margin-top:0;padding-top:0;border:0"><span class="step-label"><span>1</span>Speech / text</span><span class="arrow">→</span><span class="step-label"><span>2</span>Language</span><span class="arrow">→</span><span class="step-label"><span>3</span>Image</span><span class="arrow">→</span><span class="step-label"><span>4</span>Vision caption</span></div><div id="telephone-output"><div class="empty">Will the crown survive? Will the skateboard?<br>Make a prediction before you begin.</div></div>`);let alive=true;const stop=speechButton($('#telephone-mic'),$('#telephone-text')); |
| $('#telephone-run').onclick=()=>action($('#telephone-run'),async()=>{$('#telephone-output').innerHTML='<p class="message">The models are passing your idea along. Each round creates and independently captions a new image…</p>';try{const r=await api('telephone',{text:$('#telephone-text').value,rounds:Number($('#telephone-rounds').value)});if(!alive)return;$('#telephone-output').innerHTML=`<p class="message"><b>Original:</b> ${escape(r.original)}</p><div class="timeline">${r.rounds.map((round,i)=>`<article class="round"><h4>ROUND ${(i+1).toString().padStart(2,'0')}</h4><img src="${round.image}" alt="${escape(round.recognized)}"><p><b>Language model planned:</b> ${escape(round.instructions)}</p><blockquote>${escape(round.recognized)}</blockquote><button class="secondary" data-round="${i}">Save picture</button></article>`).join('')}</div><p class="message"><b>Final:</b> ${escape(r.final)}</p><div class="button-row"><button class="secondary" id="read-telephone">Read the final message</button></div><p class="note">These are actual outputs, including mistakes. Compare where details first disappeared: transcription, scene planning, generation, or visual recognition.</p>`;document.querySelectorAll('[data-round]').forEach(b=>b.onclick=()=>download(r.rounds[Number(b.dataset.round)].image,'telephone-round-'+(Number(b.dataset.round)+1)+'.png'));$('#read-telephone').onclick=()=>speak(r.final);activity(['association','motor','occipital','broca'])}catch(e){if(alive)$('#telephone-output').innerHTML=`<p class="message">${escape(e.message)}</p>`;throw e}},'Passing it along…');return()=>{alive=false;stop()}} |
|
|
| export async function cinema(root){root.innerHTML=card('The same scene. A different soundtrack.',`<div class="cinema-split"><div><div class="stimulus-label">VISUAL STIMULUS</div><canvas id="film" class="cinema-screen" width="600" height="360"></canvas><video id="uploaded-film" class="cinema-screen" controls hidden></video></div><div><div class="stimulus-label">SOFTWARE’S VISUAL INTERPRETATION</div><div id="cinema-candidates" class="empty">Play the built-in animation or add a local video. Capture a frame to inspect its image features.</div><div id="cinema-activity"></div></div></div><div class="toolbar"><div class="group"><button id="film-play" class="primary">▶ Play animation</button><button id="film-sound" class="secondary">Sound off</button><button id="film-analyze" class="secondary">Analyze frame</button></div><label class="secondary">Choose video<input id="film-file" type="file" accept="video/*" hidden></label></div><p class="note">The activity indicators above show which software paths executed. They are not human fMRI predictions. The built-in clip and soundtrack are generated teaching stimuli.</p>`,'COMPARE INPUTS')+card('TRIBE’s predicted cortical response',`<div class="button-row"><button class="primary" id="tribe-demo-visual">Try visual-only demo</button><button class="secondary" id="tribe-demo-audio">Compare with soundtrack</button></div><p class="note" id="tribe-runtime-status">Checking the local TRIBE runtime…</p><div id="tribe-output"><p class="note">Choose a demo to see predicted cortical responses.</p></div><details><summary style="font-size:11px;cursor:pointer;margin:15px 0">Run TRIBE on your own clip</summary><div class="field-row"><div class="field"><label for="tribe-mode">INPUT TO THE MODEL</label><select id="tribe-mode"><option value="visual">Video only</option><option value="audiovisual">Video + soundtrack</option><option value="audio">Audio only</option></select></div><div class="field"><label for="tribe-media">YOUR CLIP (SHORT CLIPS WORK BEST)</label><input id="tribe-media" type="file" accept="video/*,audio/*"></div></div><button class="primary" id="tribe-run">Predict cortical response</button><div id="tribe-job"></div></details><p class="note">The official checkpoint and original video/audio encoders run locally. Language/transcript features are absent. Predictions concern an average subject’s cortex, not your measured brain.</p><details><summary style="font-size:10px;cursor:pointer;margin-top:15px">Import an existing prediction archive</summary><label class="upload">＋ Import a TRIBE prediction archive (.npz)<br><small>Produced by the official-model runner.</small><input type="file" id="tribe-file" accept=".npz"></label></details>`,`TRIBE v2`);let alive=true,playing=false,sound=false,audio=null,osc=null,gain=null,anim,frame=0,objectURL=null,imported=[],rotation=0,surfaceAnim,surfaces=[],pollTimer,predictionTimer;root.prepend(root.children[1]);const c=$('#film'),ctx=c.getContext('2d'),video=$('#uploaded-film'); |
| function render(){if(playing)frame+=.02;ctx.fillStyle='#1d3c32';ctx.fillRect(0,0,600,360);ctx.strokeStyle='#3c5e4d';ctx.lineWidth=1;for(let x=0;x<600;x+=40){ctx.beginPath();ctx.moveTo(x,0);ctx.lineTo(x,360);ctx.stroke()}for(let y=0;y<360;y+=40){ctx.beginPath();ctx.moveTo(0,y);ctx.lineTo(600,y);ctx.stroke()}const x=300+120*Math.sin(frame),y=174+45*Math.cos(frame*1.6);ctx.fillStyle='#e0b559';ctx.beginPath();ctx.arc(x,y,36,0,Math.PI*2);ctx.fill();ctx.fillStyle='#89b49a';ctx.beginPath();ctx.moveTo(170,70);ctx.lineTo(205,125);ctx.lineTo(140,125);ctx.fill();ctx.save();ctx.translate(418,255);ctx.rotate(frame*.4);ctx.fillStyle='#b39ec5';ctx.fillRect(-25,-25,50,50);ctx.restore();ctx.fillStyle='#b3c6b0';ctx.font='10px monospace';ctx.fillText('A SMALL STUDY OF SHAPE & MOTION',26,329);if(osc)osc.frequency.value=220+80*Math.sin(frame);if(alive)anim=requestAnimationFrame(render)}render(); |
| async function audioToggle(){sound=!sound;if(!video.hidden){video.muted=!sound;video.volume=.4}else if(sound){audio??=new AudioContext();await audio.resume();osc=audio.createOscillator();gain=audio.createGain();gain.gain.value=.025;osc.type='sine';osc.connect(gain);gain.connect(audio.destination);osc.start()}else{osc?.stop();osc=null}$('#film-sound').textContent=sound?'Sound on':'Sound off'} |
| $('#film-play').onclick=()=>{if(!video.hidden){if(video.paused)video.play();else video.pause();return}playing=!playing;$('#film-play').textContent=playing?'Ⅱ Pause':'▶ Play animation'};$('#film-sound').onclick=audioToggle; |
| $('#film-analyze').onclick=()=>action($('#film-analyze'),async()=>{let image;if(video.hidden)image=c.toDataURL('image/png');else{if(!video.videoWidth)throw Error('Wait for the video to load.');const snap=document.createElement('canvas');snap.width=512;snap.height=video.videoHeight*512/video.videoWidth;snap.getContext('2d').drawImage(video,0,0,snap.width,snap.height);image=snap.toDataURL('image/png')}const r=await api('cinema/frame',{image,audio_on:sound&&!video.paused||sound&&video.hidden&&playing});if(!alive)return;$('#cinema-candidates').innerHTML=probabilities(r.candidates);$('#cinema-activity').innerHTML=`<p class="message">Vision path: on<br>Audio path: ${r.activity.temporal?'on':'off'}<br>Meaning integration: ${r.activity.association?'on':'off'}</p>`;activity(Object.keys(r.activity).filter(k=>r.activity[k]))},'Analyzing…');$('#film-file').onchange=e=>{const file=e.target.files[0];if(!file)return;playing=false;osc?.stop();osc=null;sound=false;$('#film-sound').textContent='Sound off';if(objectURL)URL.revokeObjectURL(objectURL);objectURL=URL.createObjectURL(file);video.src=objectURL;video.hidden=false;video.muted=true;c.hidden=true;$('#film-play').textContent='▶ Play video'}; |
| function drawSurface(){if(!alive||!surfaces.length)return;rotation+=.006;const frame=Number($('#tribe-frame')?.value)||0;const range=[Math.min(...imported.map(m=>m.range[0])),Math.max(...imported.map(m=>m.range[1]))];surfaces.forEach(s=>s.draw(frame,rotation,range));surfaceAnim=requestAnimationFrame(drawSurface)} |
| function showPredictions(r){clearInterval(predictionTimer);imported.push(r);imported=imported.slice(-2);cancelAnimationFrame(surfaceAnim);surfaces.forEach(s=>s.destroy());surfaces=[];const frames=Math.min(...imported.map(m=>m.frames.length)),paired=imported.length===2&&imported[0].metadata.stimulus_sha256&&imported[0].metadata.stimulus_sha256===imported[1].metadata.stimulus_sha256&&JSON.stringify(imported[0].metadata.time_seconds)===JSON.stringify(imported[1].metadata.time_seconds);$('#tribe-output').innerHTML=`<div class="cinema-split" style="margin-top:18px">${imported.map((m,i)=>`<div><div class="stimulus-label">${escape(m.metadata.modality||m.metadata.stimulus_name||'Imported prediction')}</div><canvas data-surface="${i}" width="640" height="440" class="brain-surface"></canvas></div>`).join('')}</div><div class="frame-row"><button class="secondary" id="tribe-play">▶ Play</button><label for="tribe-frame">Prediction time</label><input type="range" id="tribe-frame" min="0" max="${frames-1}" value="0"><output id="tribe-time">1 / ${frames}</output></div><div class="response-legend"><span>Lower</span><i></i><span>Higher predicted response</span></div><p class="note">Anatomical fsaverage5 cortical surface · ${r.vertices.length.toLocaleString()} vertices · actual TRIBE outputs. Both panels share a fixed color scale; blue means lower and orange means higher predicted response. The upstream model compensates for the hemodynamic lag. ${r.metadata.transcript_features===false?'Transcript/language features are absent.':''}</p>${imported.length===2?`<p class="note">${paired?'Matched clip and time points. The synchronized slider compares the same visual input with the audio branch enabled in one run.':'These archives describe different or unverified stimuli. View them qualitatively; differences cannot be attributed solely to sound.'}</p>`:''}`;document.querySelectorAll('[data-surface]').forEach(el=>surfaces.push(corticalSurface(el,imported[Number(el.dataset.surface)])));$('#tribe-frame').oninput=e=>$('#tribe-time').textContent=`${Number(e.target.value)+1} / ${frames}`;$('#tribe-play').onclick=()=>{if(predictionTimer){clearInterval(predictionTimer);predictionTimer=null;$('#tribe-play').textContent='▶ Play';return}$('#tribe-play').textContent='Ⅱ Pause';predictionTimer=setInterval(()=>{const slider=$('#tribe-frame');slider.value=(Number(slider.value)+1)%frames;slider.dispatchEvent(new Event('input'))},(r.metadata.TR_seconds||1)*1000)};drawSurface();activity(['occipital','temporal'])} |
| async function loadResult(id){const r=await api('tribe/results/'+id);if(alive&&c.isConnected)showPredictions(r)} |
| async function followJob(job){if(!alive||!c.isConnected)return;if(job.state==='complete'){await loadResult(job.result_id||job.id);$('#tribe-job').innerHTML='<p class="message">Genuine model predictions are ready.</p>';return}if(job.state==='failed')throw Error(job.error||'TRIBE did not complete.');$('#tribe-job').innerHTML=`<p class="message">${escape(job.phase||'Preparing the model')}<br>Video encoding can take several minutes on this Mac. Other experiments remain available.</p>`;pollTimer=setTimeout(async()=>{try{await followJob(await api('tribe/jobs/'+job.id))}catch(e){if(alive)$('#tribe-job').innerHTML=`<p class="message">${escape(e.message)}</p>`}},2000)} |
| async function demo(mode){clearTimeout(pollTimer);video.src='/api/tribe/demo-media';video.hidden=false;video.muted=true;c.hidden=true;$('#film-play').textContent='▶ Play video';const job=await api('tribe/demo/'+mode,{});await followJob(job)} |
| $('#tribe-demo-visual').onclick=()=>action($('#tribe-demo-visual'),()=>demo('visual'),'Loading…');$('#tribe-demo-audio').onclick=()=>action($('#tribe-demo-audio'),()=>demo('audiovisual'),'Loading…'); |
| $('#tribe-run').onclick=()=>action($('#tribe-run'),async()=>{const file=$('#tribe-media').files[0];if(!file)throw Error('Choose a video or audio clip first.');clearTimeout(pollTimer);const form=new FormData();form.append('media',file);const job=await api('tribe/jobs?mode='+$('#tribe-mode').value,form);await followJob(job)},'Starting…'); |
| $('#tribe-file').onchange=async e=>{const file=e.target.files[0];if(!file)return;try{const form=new FormData();form.append('archive',file);const r=await api('cinema/import',form);if(alive&&c.isConnected)showPredictions(r)}catch(err){toast(err.message,true)}}; |
| const runtime=await api('tribe/status');if(alive&&c.isConnected)$('#tribe-runtime-status').textContent=runtime.installed&&runtime.checkpoint?'Official model installed. Demos play immediately after their first computation.':'The local runtime or checkpoint needs setup.';if(alive&&c.isConnected&&runtime.demos.some(d=>d.mode==='visual'&&d.ready))await demo('visual'); |
| return()=>{alive=false;cancelAnimationFrame(anim);cancelAnimationFrame(surfaceAnim);clearTimeout(pollTimer);clearInterval(predictionTimer);surfaces.forEach(s=>s.destroy());osc?.stop();audio?.close();video.pause();if(objectURL)URL.revokeObjectURL(objectURL)}} |
|
|
