| import {api,card,escape,toast} from './common.js'; |
| import {liveFMRI} from './mindeye-live.js?v=brain-hub-37'; |
|
|
| |
| |
| const images=new Map(); |
| function loadImage(key,url){ |
| if(!images.has(key))images.set(key,new Promise((resolve,reject)=>{ |
| const image=new Image();image.decoding='async';image.onload=()=>resolve(image); |
| image.onerror=()=>{images.delete(key);reject(Error('The published research images could not load. Try again.'))};image.src=url; |
| })); |
| return images.get(key); |
| } |
|
|
| export async function mindsEye(root){ |
| root.innerHTML=card('What did this person see?',` |
| <div class="mindeye-intro"><span class="mindeye-replay" id="mindeye-source-badge">PUBLISHED RESEARCH REPLAY</span><p id="mindeye-intro-text">A person looked at a photograph inside an MRI scanner. MindEye2 reconstructed an image from their recorded brain responses. Can you match it to the photograph?</p></div> |
| <div class="mindeye-toolbar"><div class="segmented" role="tablist" aria-label="Mind’s Eye activity"><button id="mindeye-play-tab" role="tab" aria-selected="true">Guess the picture</button><button id="mindeye-explore-tab" role="tab" aria-selected="false">Explore examples</button><button id="mindeye-live-tab" role="tab" aria-selected="false">New fMRI decode</button></div><span class="mindeye-score" id="mindeye-score">Your matches: 0 / 0</span></div> |
| <div class="mindeye-training" id="mindeye-training-controls"><span>Training with this person</span><div class="button-row"><button class="secondary selected" data-mindeye-training="one-hour">1 hour</button><button class="secondary" data-mindeye-training="forty-hours">40 hours</button></div></div> |
| <p class="note" id="mindeye-training-note">The model was already pretrained on seven other participants. These buttons choose saved research results.</p> |
| <p class="mindeye-loading" id="mindeye-loading" role="status">Loading the published examples…</p> |
| <section id="mindeye-play" role="tabpanel" aria-label="Guess the picture"> |
| <div class="mindeye-game"><div class="mindeye-reconstruction"><span class="eyebrow">THE AI’S RECONSTRUCTION</span><canvas id="mindeye-reconstruction" width="450" height="450" role="img" aria-label="MindEye2’s published reconstruction from a recorded fMRI example"></canvas><span class="mindeye-caption" id="mindeye-training-caption">1 hour of participant training</span></div><div><div class="mindeye-choice-heading"><h3>Which photograph was it?</h3><select id="mindeye-difficulty" aria-label="Number of photographs"><option value="4">4 photographs</option><option value="8">8 photographs</option></select></div><div class="mindeye-choices" id="mindeye-choices"></div></div></div> |
| <div class="mindeye-feedback" id="mindeye-feedback" aria-live="polite" hidden></div> |
| <div class="button-row mindeye-actions"><button class="primary" id="mindeye-next" disabled>Next picture →</button><button class="secondary" id="mindeye-show-original" hidden>Compare with the original</button><button class="secondary" id="mindeye-reset">Reset score</button></div> |
| <p class="note">Your matches count your guesses in this game. They are not a measurement of MindEye2’s accuracy.</p> |
| </section> |
| <section id="mindeye-explore" role="tabpanel" aria-label="Explore examples" hidden> |
| <div class="mindeye-browse"><label for="mindeye-example">Example <input id="mindeye-example" type="number" min="1" max="1000" value="3"> / 1,000</label><div class="button-row"><button class="secondary" id="mindeye-example-back" aria-label="Previous example">←</button><button class="secondary" id="mindeye-example-go">Show example</button><button class="secondary" id="mindeye-example-next" aria-label="Next example">→</button></div></div> |
| <div class="mindeye-compare"><div><span class="eyebrow">WHAT THE PERSON SAW</span><canvas id="mindeye-original" width="450" height="450" role="img" aria-label="Original photograph viewed by the participant"></canvas></div><div><span class="eyebrow">WHAT MINDEYE2 RECONSTRUCTED</span><canvas id="mindeye-compared" width="450" height="450" role="img" aria-label="Published MindEye2 reconstruction of the same example"></canvas></div></div> |
| <div class="mindeye-observe"><b>Look for the clues.</b> Which objects, colors, and positions survived? Which details changed?</div> |
| <details class="simple-details" id="mindeye-wipe-details"><summary>Slide between the two images</summary><canvas id="mindeye-wipe" width="600" height="600" role="img" aria-label="Original and reconstruction comparison"></canvas><label class="mindeye-wipe-label">Original <input id="mindeye-wipe-position" type="range" min="0" max="100" value="50" aria-label="Original versus reconstruction boundary"> Reconstruction</label></details> |
| </section> |
| <section id="mindeye-live" role="tabpanel" aria-label="New fMRI decode" hidden></section> |
| <details class="simple-details" id="mindeye-practice-details"><summary>Try the training-data experiment</summary><p class="note">The same giraffe photograph, with different amounts of training data from one participant. Move through the authors’ actual saved results.</p><div class="mindeye-compare mindeye-practice"><div><span class="eyebrow">THE ORIGINAL</span><canvas id="mindeye-giraffe-original" width="360" height="360" role="img" aria-label="Giraffe photograph from the authors’ training-data figure"></canvas></div><div><span class="eyebrow">THE RECONSTRUCTION</span><canvas id="mindeye-giraffe-recon" width="360" height="360" role="img" aria-label="Published reconstruction at the selected training duration"></canvas></div></div><label class="mindeye-practice-label">Training data: <b id="mindeye-practice-value">1 hour</b><input id="mindeye-practice" type="range" min="0" max="5" step="1" value="2" aria-label="Amount of participant training data"></label><div class="mindeye-practice-ticks"><span>10 min</span><span>30 min</span><span>1 h</span><span>2 h</span><span>10 h</span><span>40 h</span></div><p class="note">An individual example can still contain mistakes. The time labels describe training data, not how long you need to wait.</p></details> |
| <details class="simple-details" id="mindeye-method"><summary>How does a brain pattern become a picture?</summary><div class="mindeye-pipeline"><div><span>01</span><b>Record the response</b><p>fMRI records patterns while a volunteer views photographs.</p></div><i>→</i><div><span>02</span><b>Find visual features</b><p>A learned mapping predicts an image representation in CLIP space.</p></div><i>→</i><div><span>03</span><b>Build an image</b><p>A diffusion model turns the predicted features into a reconstruction.</p></div></div><p class="note">This exhibit uses 1,000 published pairs for NSD participant 1. Each reconstruction is an enhanced output released by the MindEye2 authors. The AI ran in their study; the game and comparison controls run here. Raw brain recordings are not included in this replay.</p><div class="source-links"><a href="https://medarc-ai.github.io/mindeye2/" target="_blank" rel="noreferrer">MindEye2 project ↗</a><a href="https://arxiv.org/abs/2403.11207" target="_blank" rel="noreferrer">ICML 2024 paper ↗</a><a href="https://github.com/MedARC-AI/MindEyeV2" target="_blank" rel="noreferrer">Authors’ code & figures ↗</a><a href="/static/mindeye/provenance.json" target="_blank" rel="noreferrer">Sources & verification ↗</a></div></details> |
| `,'MINDEYE2 · NSD PARTICIPANT 1'); |
| const $=selector=>root.querySelector(selector); |
| let alive=true,info,game,training='one-hour',tab='play',example=2,comparison=null,renderId=0,requestId=0; |
| const paints=new WeakMap(); |
| const mounted=()=>alive&&root.isConnected; |
| const urls={}; |
| const live=liveFMRI($('#mindeye-live'),async(canvas,index)=>{const data=await api('mindeye/examples/'+index);await draw(canvas,data.original)}); |
| function loading(message=''){if(mounted()){$('#mindeye-loading').textContent=message;$('#mindeye-loading').hidden=!message}} |
| async function draw(canvas,view,seen=renderId){ |
| const painting=Symbol();paints.set(canvas,painting); |
| const pending=canvas.getContext('2d');pending.fillStyle='#edf1e7';pending.fillRect(0,0,canvas.width,canvas.height); |
| const image=await loadImage(view.asset,urls[view.asset]); |
| if(!mounted()||!canvas.isConnected||paints.get(canvas)!==painting)return; |
| const ctx=canvas.getContext('2d');ctx.imageSmoothingEnabled=true;ctx.imageSmoothingQuality='high'; |
| ctx.clearRect(0,0,canvas.width,canvas.height);ctx.drawImage(image,...view.rect,0,0,canvas.width,canvas.height); |
| } |
| function score(){const s=game?.score||{matches:0,attempts:0};$('#mindeye-score').textContent=`Your matches: ${s.matches} / ${s.attempts}`} |
| function modeButtons(){root.querySelectorAll('[data-mindeye-training]').forEach(b=>b.classList.toggle('selected',b.dataset.mindeyeTraining===training));$('#mindeye-training-caption').textContent=(training==='one-hour'?'1 hour':'40 hours')+' of participant training'} |
| function setTab(which){ |
| requestId++;renderId++; |
| tab=which;$('#mindeye-play').hidden=which!=='play';$('#mindeye-explore').hidden=which!=='explore';$('#mindeye-live').hidden=which!=='live'; |
| for(const selector of ['#mindeye-score','#mindeye-training-controls','#mindeye-training-note','#mindeye-practice-details','#mindeye-method'])$(selector).hidden=which==='live'; |
| $('#mindeye-source-badge').textContent=which==='live'?'LOCAL fMRI DECODER':'PUBLISHED RESEARCH REPLAY'; |
| $('#mindeye-intro-text').textContent=which==='live'?'Choose a recorded example or submit a compatible fMRI array. MindEye2 makes a new coarse image from the recorded brain responses.':'A person looked at a photograph inside an MRI scanner. MindEye2 reconstructed an image from their recorded brain responses. Can you match it to the photograph?'; |
| $('#mindeye-live-tab').setAttribute('aria-selected',String(which==='live'));if(which==='live'){loading();live.show()} |
| $('#mindeye-play-tab').setAttribute('aria-selected',String(which==='play'));$('#mindeye-explore-tab').setAttribute('aria-selected',String(which==='explore')); |
| if(which==='explore')showExample(example); |
| } |
| async function renderRound(){ |
| if(!mounted()||!game?.round)return; |
| const round=game.round,seen=++renderId;training=round.training;modeButtons();score(); |
| $('#mindeye-choices').innerHTML=round.choices.map((c,i)=>`<button class="mindeye-photo ${round.answered?(c.id===round.correct_choice?'correct':c.id===round.selected_choice?'missed':''):''}" data-mindeye-choice="${c.id}" ${round.answered?'disabled':''} aria-label="${round.answered?'Photograph':'Choose photograph'} ${i+1}${round.answered&&c.id===round.correct_choice?' — original seen image':''}${round.answered&&c.id===round.selected_choice?' — your choice':''}"><canvas width="240" height="240" aria-hidden="true"></canvas><span>Photograph ${i+1}${round.answered&&c.id===round.correct_choice?' · Original':''}</span></button>`).join(''); |
| const feedback=$('#mindeye-feedback');feedback.hidden=!round.answered;feedback.classList.toggle('matched',Boolean(round.correct)); |
| if(round.answered)feedback.textContent=round.correct?'You matched it. This is the photograph the participant saw.':'The outlined photograph is the one the participant saw. Compare the details to find the clues.'; |
| $('#mindeye-show-original').hidden=!round.answered; |
| root.querySelectorAll('[data-mindeye-choice]').forEach(button=>button.onclick=async()=>{ |
| root.querySelectorAll('[data-mindeye-choice]').forEach(b=>b.disabled=true);const req=++requestId; |
| try{const data=await api('mindeye/answer',{round_id:round.id,choice_id:button.dataset.mindeyeChoice});if(!mounted()||req!==requestId)return;game=data;await renderRound()} |
| catch(e){if(mounted()){toast(e.message,true);await renderRound()}} |
| }); |
| loading('Loading published images…'); |
| try{ |
| await Promise.all([draw($('#mindeye-reconstruction'),round.reconstruction,seen),...round.choices.map((c,i)=>draw(root.querySelectorAll('[data-mindeye-choice] canvas')[i],c.image,seen))]); |
| if(mounted()&&seen===renderId){loading();$('#mindeye-next').disabled=false} |
| }catch(e){if(mounted()){loading(e.message);$('#mindeye-next').disabled=false}} |
| } |
| async function newRound(){ |
| const req=++requestId;$('#mindeye-next').disabled=true;loading('Choosing another published example…'); |
| try{const data=await api('mindeye/round',{training,choices:Number($('#mindeye-difficulty').value)});if(!mounted()||req!==requestId)return;game=data;await renderRound()} |
| catch(e){if(mounted()){loading(e.message);$('#mindeye-next').disabled=false}} |
| } |
| async function showExample(index){ |
| index=Math.max(0,Math.min(999,Math.trunc(index)));const req=++requestId;loading('Loading the comparison…'); |
| try{ |
| const data=await api('mindeye/examples/'+index);if(!mounted()||req!==requestId)return; |
| comparison=data;example=index;$('#mindeye-example').value=index+1;const seen=++renderId; |
| await Promise.all([draw($('#mindeye-original'),data.original,seen),draw($('#mindeye-compared'),data.reconstructions[training],seen)]); |
| if(mounted()&&seen===renderId){loading();if($('#mindeye-wipe-details').open)await drawWipe()} |
| }catch(e){if(mounted())loading(e.message)} |
| } |
| async function drawWipe(){ |
| if(!comparison||!mounted())return; |
| const seen=renderId,data=comparison; |
| const [original,reconstruction]=await Promise.all([loadImage('one-hour',urls['one-hour']),loadImage(training,urls[training])]); |
| if(!mounted()||seen!==renderId)return; |
| const canvas=$('#mindeye-wipe'),ctx=canvas.getContext('2d'),edge=Number($('#mindeye-wipe-position').value)*canvas.width/100; |
| ctx.drawImage(reconstruction,...data.reconstructions[training].rect,0,0,canvas.width,canvas.height); |
| ctx.save();ctx.beginPath();ctx.rect(0,0,edge,canvas.height);ctx.clip();ctx.drawImage(original,...data.original.rect,0,0,canvas.width,canvas.height);ctx.restore(); |
| ctx.strokeStyle='#fffefa';ctx.lineWidth=5;ctx.beginPath();ctx.moveTo(edge,0);ctx.lineTo(edge,canvas.height);ctx.stroke(); |
| } |
| async function drawPractice(){ |
| if(!info||!mounted())return; |
| const data=info.training_example,selected=data.settings[Number($('#mindeye-practice').value)]; |
| $('#mindeye-practice-value').textContent=selected.label; |
| try{await Promise.all([draw($('#mindeye-giraffe-original'),{asset:'training-figure',rect:data.original}),draw($('#mindeye-giraffe-recon'),{asset:'training-figure',rect:selected.rect})])}catch(e){toast(e.message,true)} |
| } |
| root.querySelectorAll('[data-mindeye-training]').forEach(button=>button.onclick=async()=>{ |
| const next=button.dataset.mindeyeTraining;if(training===next)return;training=next;modeButtons(); |
| if(!game?.round){if(tab==='explore')return showExample(example);return} |
| const req=++requestId;loading('Loading this training comparison…'); |
| try{const data=await api('mindeye/training',{round_id:game.round.id,training});if(!mounted()||req!==requestId)return;game=data;if(tab==='explore')await showExample(example);else await renderRound()} |
| catch(e){if(mounted()){loading(e.message);training=game.round.training;modeButtons()}} |
| }); |
| $('#mindeye-play-tab').onclick=async()=>{setTab('play');if(game?.round)await renderRound()}; |
| $('#mindeye-explore-tab').onclick=()=>setTab('explore');$('#mindeye-live-tab').onclick=()=>setTab('live'); |
| $('#mindeye-next').onclick=newRound;$('#mindeye-difficulty').onchange=newRound; |
| $('#mindeye-reset').onclick=async()=>{try{game=await api('mindeye/reset',{});if(mounted())await newRound()}catch(e){toast(e.message,true)}}; |
| $('#mindeye-show-original').onclick=()=>{example=game.round.comparison.index;setTab('explore')}; |
| $('#mindeye-example-go').onclick=()=>showExample(Number($('#mindeye-example').value)-1); |
| $('#mindeye-example').onkeydown=e=>{if(e.key==='Enter')showExample(Number(e.target.value)-1)}; |
| $('#mindeye-example-back').onclick=()=>showExample((example+999)%1000);$('#mindeye-example-next').onclick=()=>showExample((example+1)%1000); |
| $('#mindeye-wipe-position').oninput=drawWipe;$('#mindeye-wipe-details').ontoggle=()=>{if($('#mindeye-wipe-details').open)drawWipe()}; |
| $('#mindeye-practice-details').ontoggle=()=>{if($('#mindeye-practice-details').open)drawPractice()};$('#mindeye-practice').oninput=drawPractice; |
| try{ |
| const data=await Promise.all([api('mindeye'),api('mindeye/state')]);if(!mounted())return ()=>{}; |
| info=data[0];game=data[1];for(const item of info.modes)urls[item.id]=item.url;urls['training-figure']=info.training_example.url; |
| if(game.round){$('#mindeye-difficulty').value=game.round.choice_count;await renderRound()}else await newRound(); |
| }catch(e){if(mounted())loading(e.message)} |
| return ()=>{alive=false;renderId++;requestId++;live.dispose()}; |
| } |
|
|
