import {api,card,escape,toast} from './common.js';
import {liveFMRI} from './mindeye-live.js?v=brain-hub-37';
// Reuse the authors' unmodified figure files across rounds and navigation.
// Each canvas is a viewport onto a published tile, not a generated reconstruction.
const images=new Map();
function loadImage(key,url){
if(!images.has(key))images.set(key,new Promise((resolve,reject)=>{
const image=new Image();image.decoding='async';image.onload=()=>resolve(image);
image.onerror=()=>{images.delete(key);reject(Error('The published research images could not load. Try again.'))};image.src=url;
}));
return images.get(key);
}
export async function mindsEye(root){
root.innerHTML=card('What did this person see?',`
PUBLISHED RESEARCH REPLAY
A person looked at a photograph inside an MRI scanner. MindEye2 reconstructed an image from their recorded brain responses. Can you match it to the photograph?
Your matches: 0 / 0
Training with this person
The model was already pretrained on seven other participants. These buttons choose saved research results.
Loading the published examples…
THE AI’S RECONSTRUCTION1 hour of participant training
Which photograph was it?
Your matches count your guesses in this game. They are not a measurement of MindEye2’s accuracy.
WHAT THE PERSON SAW
WHAT MINDEYE2 RECONSTRUCTED
Look for the clues. Which objects, colors, and positions survived? Which details changed?
Slide between the two imagesTry the training-data experiment
The same giraffe photograph, with different amounts of training data from one participant. Move through the authors’ actual saved results.
THE ORIGINAL
THE RECONSTRUCTION
10 min30 min1 h2 h10 h40 h
An individual example can still contain mistakes. The time labels describe training data, not how long you need to wait.
How does a brain pattern become a picture?
01Record the response
fMRI records patterns while a volunteer views photographs.
→
02Find visual features
A learned mapping predicts an image representation in CLIP space.
→
03Build an image
A diffusion model turns the predicted features into a reconstruction.
This exhibit uses 1,000 published pairs for NSD participant 1. Each reconstruction is an enhanced output released by the MindEye2 authors. The AI ran in their study; the game and comparison controls run here. Raw brain recordings are not included in this replay.
`,'MINDEYE2 · NSD PARTICIPANT 1');
const $=selector=>root.querySelector(selector);
let alive=true,info,game,training='one-hour',tab='play',example=2,comparison=null,renderId=0,requestId=0;
const paints=new WeakMap();
const mounted=()=>alive&&root.isConnected;
const urls={};
const live=liveFMRI($('#mindeye-live'),async(canvas,index)=>{const data=await api('mindeye/examples/'+index);await draw(canvas,data.original)});
function loading(message=''){if(mounted()){$('#mindeye-loading').textContent=message;$('#mindeye-loading').hidden=!message}}
async function draw(canvas,view,seen=renderId){
const painting=Symbol();paints.set(canvas,painting);
const pending=canvas.getContext('2d');pending.fillStyle='#edf1e7';pending.fillRect(0,0,canvas.width,canvas.height);
const image=await loadImage(view.asset,urls[view.asset]);
if(!mounted()||!canvas.isConnected||paints.get(canvas)!==painting)return;
const ctx=canvas.getContext('2d');ctx.imageSmoothingEnabled=true;ctx.imageSmoothingQuality='high';
ctx.clearRect(0,0,canvas.width,canvas.height);ctx.drawImage(image,...view.rect,0,0,canvas.width,canvas.height);
}
function score(){const s=game?.score||{matches:0,attempts:0};$('#mindeye-score').textContent=`Your matches: ${s.matches} / ${s.attempts}`}
function modeButtons(){root.querySelectorAll('[data-mindeye-training]').forEach(b=>b.classList.toggle('selected',b.dataset.mindeyeTraining===training));$('#mindeye-training-caption').textContent=(training==='one-hour'?'1 hour':'40 hours')+' of participant training'}
function setTab(which){
requestId++;renderId++;
tab=which;$('#mindeye-play').hidden=which!=='play';$('#mindeye-explore').hidden=which!=='explore';$('#mindeye-live').hidden=which!=='live';
for(const selector of ['#mindeye-score','#mindeye-training-controls','#mindeye-training-note','#mindeye-practice-details','#mindeye-method'])$(selector).hidden=which==='live';
$('#mindeye-source-badge').textContent=which==='live'?'LOCAL fMRI DECODER':'PUBLISHED RESEARCH REPLAY';
$('#mindeye-intro-text').textContent=which==='live'?'Choose a recorded example or submit a compatible fMRI array. MindEye2 makes a new coarse image from the recorded brain responses.':'A person looked at a photograph inside an MRI scanner. MindEye2 reconstructed an image from their recorded brain responses. Can you match it to the photograph?';
$('#mindeye-live-tab').setAttribute('aria-selected',String(which==='live'));if(which==='live'){loading();live.show()}
$('#mindeye-play-tab').setAttribute('aria-selected',String(which==='play'));$('#mindeye-explore-tab').setAttribute('aria-selected',String(which==='explore'));
if(which==='explore')showExample(example);
}
async function renderRound(){
if(!mounted()||!game?.round)return;
const round=game.round,seen=++renderId;training=round.training;modeButtons();score();
$('#mindeye-choices').innerHTML=round.choices.map((c,i)=>``).join('');
const feedback=$('#mindeye-feedback');feedback.hidden=!round.answered;feedback.classList.toggle('matched',Boolean(round.correct));
if(round.answered)feedback.textContent=round.correct?'You matched it. This is the photograph the participant saw.':'The outlined photograph is the one the participant saw. Compare the details to find the clues.';
$('#mindeye-show-original').hidden=!round.answered;
root.querySelectorAll('[data-mindeye-choice]').forEach(button=>button.onclick=async()=>{
root.querySelectorAll('[data-mindeye-choice]').forEach(b=>b.disabled=true);const req=++requestId;
try{const data=await api('mindeye/answer',{round_id:round.id,choice_id:button.dataset.mindeyeChoice});if(!mounted()||req!==requestId)return;game=data;await renderRound()}
catch(e){if(mounted()){toast(e.message,true);await renderRound()}}
});
loading('Loading published images…');
try{
await Promise.all([draw($('#mindeye-reconstruction'),round.reconstruction,seen),...round.choices.map((c,i)=>draw(root.querySelectorAll('[data-mindeye-choice] canvas')[i],c.image,seen))]);
if(mounted()&&seen===renderId){loading();$('#mindeye-next').disabled=false}
}catch(e){if(mounted()){loading(e.message);$('#mindeye-next').disabled=false}}
}
async function newRound(){
const req=++requestId;$('#mindeye-next').disabled=true;loading('Choosing another published example…');
try{const data=await api('mindeye/round',{training,choices:Number($('#mindeye-difficulty').value)});if(!mounted()||req!==requestId)return;game=data;await renderRound()}
catch(e){if(mounted()){loading(e.message);$('#mindeye-next').disabled=false}}
}
async function showExample(index){
index=Math.max(0,Math.min(999,Math.trunc(index)));const req=++requestId;loading('Loading the comparison…');
try{
const data=await api('mindeye/examples/'+index);if(!mounted()||req!==requestId)return;
comparison=data;example=index;$('#mindeye-example').value=index+1;const seen=++renderId;
await Promise.all([draw($('#mindeye-original'),data.original,seen),draw($('#mindeye-compared'),data.reconstructions[training],seen)]);
if(mounted()&&seen===renderId){loading();if($('#mindeye-wipe-details').open)await drawWipe()}
}catch(e){if(mounted())loading(e.message)}
}
async function drawWipe(){
if(!comparison||!mounted())return;
const seen=renderId,data=comparison;
const [original,reconstruction]=await Promise.all([loadImage('one-hour',urls['one-hour']),loadImage(training,urls[training])]);
if(!mounted()||seen!==renderId)return;
const canvas=$('#mindeye-wipe'),ctx=canvas.getContext('2d'),edge=Number($('#mindeye-wipe-position').value)*canvas.width/100;
ctx.drawImage(reconstruction,...data.reconstructions[training].rect,0,0,canvas.width,canvas.height);
ctx.save();ctx.beginPath();ctx.rect(0,0,edge,canvas.height);ctx.clip();ctx.drawImage(original,...data.original.rect,0,0,canvas.width,canvas.height);ctx.restore();
ctx.strokeStyle='#fffefa';ctx.lineWidth=5;ctx.beginPath();ctx.moveTo(edge,0);ctx.lineTo(edge,canvas.height);ctx.stroke();
}
async function drawPractice(){
if(!info||!mounted())return;
const data=info.training_example,selected=data.settings[Number($('#mindeye-practice').value)];
$('#mindeye-practice-value').textContent=selected.label;
try{await Promise.all([draw($('#mindeye-giraffe-original'),{asset:'training-figure',rect:data.original}),draw($('#mindeye-giraffe-recon'),{asset:'training-figure',rect:selected.rect})])}catch(e){toast(e.message,true)}
}
root.querySelectorAll('[data-mindeye-training]').forEach(button=>button.onclick=async()=>{
const next=button.dataset.mindeyeTraining;if(training===next)return;training=next;modeButtons();
if(!game?.round){if(tab==='explore')return showExample(example);return}
const req=++requestId;loading('Loading this training comparison…');
try{const data=await api('mindeye/training',{round_id:game.round.id,training});if(!mounted()||req!==requestId)return;game=data;if(tab==='explore')await showExample(example);else await renderRound()}
catch(e){if(mounted()){loading(e.message);training=game.round.training;modeButtons()}}
});
$('#mindeye-play-tab').onclick=async()=>{setTab('play');if(game?.round)await renderRound()};
$('#mindeye-explore-tab').onclick=()=>setTab('explore');$('#mindeye-live-tab').onclick=()=>setTab('live');
$('#mindeye-next').onclick=newRound;$('#mindeye-difficulty').onchange=newRound;
$('#mindeye-reset').onclick=async()=>{try{game=await api('mindeye/reset',{});if(mounted())await newRound()}catch(e){toast(e.message,true)}};
$('#mindeye-show-original').onclick=()=>{example=game.round.comparison.index;setTab('explore')};
$('#mindeye-example-go').onclick=()=>showExample(Number($('#mindeye-example').value)-1);
$('#mindeye-example').onkeydown=e=>{if(e.key==='Enter')showExample(Number(e.target.value)-1)};
$('#mindeye-example-back').onclick=()=>showExample((example+999)%1000);$('#mindeye-example-next').onclick=()=>showExample((example+1)%1000);
$('#mindeye-wipe-position').oninput=drawWipe;$('#mindeye-wipe-details').ontoggle=()=>{if($('#mindeye-wipe-details').open)drawWipe()};
$('#mindeye-practice-details').ontoggle=()=>{if($('#mindeye-practice-details').open)drawPractice()};$('#mindeye-practice').oninput=drawPractice;
try{
const data=await Promise.all([api('mindeye'),api('mindeye/state')]);if(!mounted())return ()=>{};
info=data[0];game=data[1];for(const item of info.modes)urls[item.id]=item.url;urls['training-figure']=info.training_example.url;
if(game.round){$('#mindeye-difficulty').value=game.round.choice_count;await renderRound()}else await newRound();
}catch(e){if(mounted())loading(e.message)}
return ()=>{alive=false;renderId++;requestId++;live.dispose()};
}