Spaces:
Running
Running
| ; | |
| let detailLayout=''; | |
| const detailNode=(tag,cls,text)=>{const n=document.createElement(tag);if(cls)n.className=cls;if(text!==undefined)n.textContent=text;return n;}; | |
| function openDetail(kind,index){ | |
| if(!S.config)return;if(kind==='head'&&Number.isInteger(index))S.head=Math.min(headCount()-1,index); | |
| if(kind==='expert')S.expert=Number.isInteger(index)?index:(S.expert??current()[S.layer]?.experts?.[0]??0); | |
| if((kind==='expert'||kind==='shared')&&!isMoe())kind='ffn';if(kind==='ffn'&&isMoe())kind='block';S.detail=kind; | |
| if(!$('detailDialog').open){$('detailDialog').showModal();document.body.classList.add('detail-open');}update(); | |
| } | |
| function closeDetail(){$('detailDialog').close();} | |
| function detailTokens(){return [...(S.start?.tokens||[]),...S.frames.map(f=>f.text)];} | |
| function metric(id,value){$(id).textContent=value;} | |
| function stat(id,label){return `<div><strong id="${id}"></strong><span>${label}</span></div>`;} | |
| function circuitNode(title,subtitle,cls=''){return `<div class="circuit-node ${cls}"><strong>${title}</strong><span>${subtitle}</span></div>`;} | |
| function circuitLink(label=''){return `<div class="circuit-link"><span>${label}</span><b>→</b></div>`;} | |
| function denseCircuit(){return '<div class="component-circuit">'+circuitNode(S.config.hidden_size,'Input')+circuitLink()+circuitNode(S.config.intermediate_size,'Gated feed-forward')+circuitLink()+circuitNode(S.config.hidden_size,'Output')+'</div>';} | |
| function makeHeadButtons(dest){for(let i=0;i<headCount();i++){const b=detailNode('button','detail-head');b.dataset.head=i;b.ariaLabel='Open '+(isLinear()?'linear value':'attention')+' head '+i;b.append(detailNode('span','head-meter'),detailNode('b','',pad(i)),detailNode('small','',''));b.onclick=()=>openDetail('head',i);dest.append(b);}} | |
| function makeExpertButtons(dest){for(let i=0;i<S.config.num_experts;i++){const b=detailNode('button','detail-expert',String(i).padStart(3,'0'));b.dataset.expert=i;b.ariaLabel='Open expert '+i;b.onclick=()=>openDetail('expert',i);dest.append(b);}} | |
| function buildDetail(){ | |
| const kind=S.detail,dense=!isMoe(),linear=isLinear(),hidden=S.config.hidden_size,body=$('detailBody');S.detailMetric='heads'; | |
| if(kind==='block'){ | |
| body.innerHTML=`<div class="detail-stats">${stat('dAttentionMode','Attention')}${stat('dHeadCount',linear?'Value / key heads':'Query / KV heads')}${stat('dRoutingCount',dense?'Feed-forward width':'Routed experts')}${stat('dResidual','Block output RMS')}</div> | |
| <div class="block-diagram"><div class="flow-pillar">Hidden state <b>${hidden}</b></div><div class="flow-connector">↓ <span>RMSNorm</span></div> | |
| <section class="detail-card"><div class="detail-card-heading"><h3>${linear?'Recurrent linear attention':'Attention'}</h3><span id="dAttentionCaption"></span></div><div id="dBlockHeads" class="detail-head-grid"></div><div class="card-caption">Each head produces ${headDimension()} values. Click a head to inspect ${linear?'its recurrent memory':'its attention'}.</div></section> | |
| <div class="flow-connector">↓ <span>Output projection + residual → RMSNorm</span></div> | |
| <section class="detail-card"><div class="detail-card-heading"><h3>${dense?'Dense feed-forward':'Mixture of experts'}</h3><span>${dense?'All units participate':S.config.num_experts_per_tok+' routed + 1 shared'}</span></div> | |
| ${dense?denseCircuit()+'<button id="dBlockFfn" class="shared-detail-link">Inspect feed-forward ↗</button>':'<div id="dBlockRoutes" class="route-cards"></div><div class="expert-map-heading"><span>'+S.config.num_experts+' experts</span><button id="dBlockShared" class="shared-detail-link">Shared expert ↗</button></div><div id="dBlockExperts" class="detail-expert-grid"></div>'}</section> | |
| <div class="flow-connector">↓ <span>+ residual</span></div><div class="flow-pillar">Block output <b id="dBlockOutput"></b></div></div>`; | |
| makeHeadButtons($('dBlockHeads'));if(dense)$('dBlockFfn').onclick=()=>openDetail('ffn');else{makeExpertButtons($('dBlockExperts'));$('dBlockShared').onclick=()=>openDetail('shared');} | |
| }else if(kind==='ffn'){ | |
| body.innerHTML=`<div class="detail-stats">${stat('dFfnInput','Hidden width')}${stat('dFfnWidth','Intermediate width')}${stat('dFfnRms','Feed-forward output RMS')}${stat('dFfnResidual','Block output RMS')}</div><section class="detail-card"><div class="detail-card-heading"><h3>Dense feed-forward</h3><span>SiLU gated network</span></div>${denseCircuit()}<div class="card-caption">Gate and value projections expand the hidden state. SiLU gating combines them, and the output projection returns to the hidden width. The dense branch runs for every token.</div></section><section class="detail-card timeline-card"><div class="detail-card-heading"><h3>Feed-forward output over tokens</h3><span>RMS</span></div><div id="dTimeline" class="detail-timeline"></div></section>`; | |
| }else if(kind==='head'){ | |
| body.innerHTML=`<div class="component-selector"><label for="dHeadSelect">${linear?'Value head':'Query head'}</label><select id="dHeadSelect"></select><span id="dKvGroup"></span></div> | |
| <div class="detail-stats">${stat('dHeadRms','Head output RMS')}${stat('dHeadDimension','Head dimension')}${stat('dKvHead',linear?'Shared key head':'Shared KV head')}${linear?stat('dStateRms','Recurrent state RMS'):stat('dEntropy','Token entropy, nats')}</div> | |
| <div class="component-circuit head-circuit">${linear?circuitNode('Previous state',S.config.linear_key_head_dim+' × '+S.config.linear_value_head_dim)+circuitLink('× retention')+circuitNode('Delta update','Write gate β')+circuitLink('Query read')+circuitNode('Gated output',headDimension()+' values'):circuitNode('Query','','query-node')+circuitLink('Q × K')+circuitNode('Attention',S.model.learned_sink?'Softmax + learned sink':'Softmax + output gate')+circuitLink('Weighted V')+circuitNode('Head output',headDimension()+' values')}</div> | |
| <div class="detail-columns"><section class="detail-card"><div class="detail-card-heading"><h3>${linear?'Recurrent memory':'Attention positions'}</h3><span>${linear?'Gated DeltaNet':'8 strongest'}</span></div>${linear?'<div id="dMemory" class="detail-recurrent"></div><p class="recurrent-explanation">Retention scales the previous memory. The write gate controls its delta update. The query reads the updated state, then normalization and gating shape the output.</p>':'<div id="dAttentionRows" class="detail-attention-rows"></div><div id="dAttentionMass" class="attention-mass"></div>'}</section> | |
| <section class="detail-card"><div class="detail-card-heading"><h3>${linear?'All value heads':'All query heads'}</h3><span>${headCount()} ${linear?'V':'Q'} / ${keyHeadCount()} ${linear?'K':'KV'}</span></div><div id="dAllHeads" class="detail-head-grid compact"></div><div class="card-caption">Colour and bar height show measured output RMS. All heads execute.</div></section></div> | |
| <section class="detail-card timeline-card"><div class="detail-card-heading"><h3>Head activity over tokens</h3>${linear?'<select id="dTimelineMetric" class="timeline-metric-select" aria-label="Head timeline measurement"><option value="heads">Output RMS</option><option value="state_rms">State RMS</option><option value="write_gate">Write gate</option><option value="retention">Retention</option></select>':'<span>Output RMS</span>'}</div><div id="dTimeline" class="detail-timeline"></div></section>`; | |
| for(let i=0;i<headCount();i++)$('dHeadSelect').add(new Option('Head '+pad(i),i));$('dHeadSelect').onchange=()=>openDetail('head',Number($('dHeadSelect').value));makeHeadButtons($('dAllHeads'));if(linear)$('dTimelineMetric').onchange=()=>{S.detailMetric=$('dTimelineMetric').value;paintTimeline('head');}; | |
| }else{ | |
| body.innerHTML=`<div class="component-selector"><label for="dExpertSelect">Expert</label><select id="dExpertSelect"></select><span id="dExpertArchitecture"></span></div> | |
| <div class="detail-stats">${stat('dExpertStatus','Current token')}${stat('dExpertWeight',kind==='shared'?'Output RMS':'Routing weight')}${stat('dExpertRank',kind==='shared'?'Participation':'Router rank')}${stat('dExpertFrequency','Selected tokens')}</div> | |
| <div class="component-circuit expert-circuit">${circuitNode(hidden,'Input')}${circuitLink()}${circuitNode(kind==='shared'?S.config.shared_expert_intermediate_size:S.config.moe_intermediate_size,'Gated feed-forward','expert-core')}${circuitLink()}${circuitNode(hidden,'Expert output')}${circuitLink(kind==='shared'?'Shared branch':'× routing weight')}${circuitNode('Combine','Routed + shared')}</div> | |
| <div class="detail-columns"><section class="detail-card"><div class="detail-card-heading"><h3>Current routing</h3><span>${S.config.num_experts_per_tok} of ${S.config.num_experts}</span></div><div id="dExpertRoutes" class="route-cards vertical"></div><div class="mixture-readings"><span>Routed mix <b id="dRoutedRms"></b></span><button id="dSharedLink">Shared expert <b id="dSharedRms"></b> ↗</button></div></section> | |
| <section class="detail-card"><div class="detail-card-heading"><h3>Expert map</h3><span>Layer <b id="dExpertLayer"></b></span></div><div id="dAllExperts" class="detail-expert-grid"></div></section></div> | |
| <section class="detail-card timeline-card"><div class="detail-card-heading"><h3 id="dExpertTimelineTitle"></h3><span id="dExpertTimelineMetric"></span></div><div id="dTimeline" class="detail-timeline"></div></section>`; | |
| for(let i=0;i<S.config.num_experts;i++)$('dExpertSelect').add(new Option('Expert '+String(i).padStart(3,'0'),i));$('dExpertSelect').add(new Option('Shared expert','shared')); | |
| $('dExpertSelect').onchange=()=>openDetail($('dExpertSelect').value==='shared'?'shared':'expert',Number($('dExpertSelect').value));makeExpertButtons($('dAllExperts'));$('dSharedLink').onclick=()=>openDetail('shared'); | |
| }body.scrollTop=0; | |
| } | |
| function paintHeads(dest,data){ | |
| const max=scaleFor(current());for(const b of dest.children){const h=Number(b.dataset.head),v=data?.heads?.[h];b.classList.toggle('selected',S.detail==='head'&&h===S.head);b.style.background=Number.isFinite(v)?colour(v,max):'';b.style.color=Number.isFinite(v)&&v/max>.6?'#f4f9eb':'';b.style.setProperty('--level',Number.isFinite(v)?Math.min(100,v/max*100)+'%':'0%');b.querySelector('small').textContent=Number.isFinite(v)?fmt(v):'·';b.title=`Head ${pad(h)} · ${isLinear()?'K':'KV'} ${keyHead(h)} · ${Number.isFinite(v)?fmt(v)+' output RMS':'awaiting trace'}`;b.ariaPressed=String(S.detail==='head'&&h===S.head);} | |
| } | |
| function paintExperts(dest,data){ | |
| for(const b of dest.children){const id=Number(b.dataset.expert),rank=data?.experts?.indexOf(id)??-1;b.classList.toggle('routed',rank>=0);b.classList.toggle('selected',S.detail==='expert'&&id===S.expert);b.style.background=rank>=0?colour(data.weights[rank],.27):'';b.style.color=rank>=0&&data.weights[rank]/.27>.6?'#f4f9eb':'';b.title=`Expert ${id} · ${rank>=0?'selected · '+pct(data.weights[rank]):data?'not selected':'awaiting routing'}`;b.ariaPressed=String(S.detail==='expert'&&id===S.expert);} | |
| } | |
| function paintRoutes(dest,data){ | |
| const signature=JSON.stringify([data?.experts,data?.weights,S.expert,S.detail]);if(dest.dataset.signature===signature)return;dest.dataset.signature=signature;dest.replaceChildren();if(!data?.experts){dest.append(detailNode('span','detail-empty','Run a prompt or open a saved trace to inspect routing.'));return;} | |
| data.experts.forEach((id,i)=>{const b=detailNode('button','route-card');b.dataset.expert=id;b.classList.toggle('selected',S.detail==='expert'&&id===S.expert);b.style.setProperty('--weight',pct(data.weights[i]));b.append(detailNode('span','','Expert '+String(id).padStart(3,'0')),detailNode('strong','',pct(data.weights[i])),detailNode('i',''));b.onclick=()=>openDetail('expert',id);dest.append(b);}); | |
| } | |
| function paintAttention(data){ | |
| const a=data?.attention?.[S.head],dest=$('dAttentionRows'),signature=JSON.stringify([a,S.head,S.frame,S.live,S.frames.length]);if(dest.dataset.signature===signature)return;dest.dataset.signature=signature;dest.replaceChildren();$('dAttentionMass').replaceChildren(); | |
| if(!a){dest.append(detailNode('span','detail-empty','Run a prompt or select a measured token to inspect attention.'));return;} | |
| const tokens=detailTokens();for(const [pos,weight]of a.top){const row=detailNode('div','detail-attention-row');row.style.setProperty('--weight',pct(weight));const token=detailNode('code','',tokens[pos]?.replace(/\n/g,'↵')??'#'+pos);token.title=tokens[pos]??'';row.append(detailNode('span','attention-position','#'+pos),token,detailNode('strong','',pct(weight)));dest.append(row);} | |
| const top=a.top.reduce((v,item)=>v+item[1],0);if(S.model.learned_sink)$('dAttentionMass').append(detailNode('span','','Learned sink '+pct(a.sink)));$('dAttentionMass').append(detailNode('span','','Other token positions '+pct(Math.max(0,1-(S.model.learned_sink?a.sink:0)-top)))); | |
| } | |
| function paintMemory(data){ | |
| const dest=$('dMemory');if(!dest.children.length)for(const [key,label]of [['write_gate','Write gate'],['retention','Retention'],['state_rms','State RMS']]){const card=detailNode('div','memory-metric');card.dataset.metric=key;card.append(detailNode('strong',''),detailNode('span','',label));if(key!=='state_rms'){const meter=detailNode('div','memory-meter');meter.append(detailNode('i',''));card.append(meter);}dest.append(card);} | |
| for(const card of dest.children){const v=data?.[card.dataset.metric]?.[S.head];card.querySelector('strong').textContent=Number.isFinite(v)?card.dataset.metric==='state_rms'?fmt(v):pct(v):'unmeasured';card.style.setProperty('--value',Number.isFinite(v)?pct(v):'0%');} | |
| } | |
| function paintTimeline(kind){ | |
| const dest=$('dTimeline'),field=isLinear()&&kind==='head'?S.detailMetric||'heads':'heads',percentage=kind==='expert'||['write_gate','retention'].includes(field),values=S.frames.map(f=>{const l=f.layers[S.layer];if(kind==='head')return l[field]?.[S.head];if(kind==='ffn')return l.ffn;if(kind==='shared')return l.shared;const i=l.experts?.indexOf(S.expert)??-1;return i>=0?l.weights[i]:0;}),max=Math.max(...values.filter(Number.isFinite),.00001); | |
| dest.querySelector('.detail-empty')?.remove();while(dest.children.length>values.length)dest.lastChild.remove(); | |
| for(let i=0;i<values.length;i++){let b=dest.children[i];if(!b){b=detailNode('button','timeline-token');b.append(detailNode('span','timeline-bar'),detailNode('code',''),detailNode('small',''));b.onclick=()=>selectFrame(i);dest.append(b);}const v=values[i],label=S.frames[i].text.replace(/\n/g,'↵')||'∅';b.classList.toggle('selected',i===S.frame);b.classList.toggle('inactive',v===0);b.style.setProperty('--level',Number.isFinite(v)?Math.max(v>0?3:0,v/max*100)+'%':'0%');b.querySelector('code').textContent=label;b.querySelector('small').textContent=String(i+1);b.title=`Token ${i+1}: ${label} · ${Number.isFinite(v)?percentage?pct(v):fmt(v)+' RMS':'unmeasured'}`;b.ariaLabel=b.title;b.ariaPressed=String(i===S.frame);} | |
| if(!values.length)dest.append(detailNode('span','detail-empty','A measured timeline appears after opening a capture.')); | |
| } | |
| function renderDetail(){ | |
| if(!$('detailDialog').open||!S.detail||!S.config)return; | |
| const kind=S.detail,data=current()[S.layer],dense=!isMoe(),linear=isLinear(),key=S.model.id+':'+kind+':'+S.layer;if(key!==detailLayout){detailLayout=key;buildDetail();} | |
| $('detailTitle').textContent=`Block ${pad(S.layer)}${kind==='head'?' / '+(linear?'Linear head ':'Attention head ')+pad(S.head):kind==='expert'?' / Expert '+String(S.expert).padStart(3,'0'):kind==='shared'?' / Shared expert':kind==='ffn'?' / Feed-forward':''}`; | |
| $('detailSubtitle').textContent=S.model.short_name+' · '+attentionLabel()+' · '+(dense?'Dense feed-forward':'Mixture of experts')+' · hidden size '+S.config.hidden_size; | |
| document.querySelectorAll('[data-detail]').forEach(b=>{const selected=b.dataset.detail===(kind==='shared'||kind==='ffn'?'expert':kind);b.classList.toggle('selected',selected);b.ariaPressed=String(selected);b.disabled=false;if(b.dataset.detail==='expert')b.textContent=dense?'Feed-forward':'Experts';if(b.dataset.detail==='head')b.textContent=linear?'Linear attention':'Attention';}); | |
| $('detailPrev').disabled=S.layer===0;$('detailNext').disabled=S.layer===layerCount()-1;$('detailMode').textContent=S.running&&S.live?'Live capture':S.timer?'Playing measured trace':'Paused inspection'; | |
| if(kind==='block'){ | |
| metric('dAttentionMode',linear?'Gated DeltaNet':S.config.layer_types[S.layer]==='full_attention'?'Full context':S.config.sliding_window+'-token window');metric('dHeadCount',headCount()+' / '+keyHeadCount());metric('dRoutingCount',dense?String(S.config.intermediate_size):S.config.num_experts_per_tok+' / '+S.config.num_experts);metric('dResidual',fmt(data?.residual));metric('dBlockOutput',Number.isFinite(data?.residual)?fmt(data.residual)+' RMS':String(S.config.hidden_size));metric('dAttentionCaption',linear?'Recurrent state':attentionLabel());paintHeads($('dBlockHeads'),data); | |
| if(!dense){paintRoutes($('dBlockRoutes'),data);paintExperts($('dBlockExperts'),data);$('dBlockShared').textContent='Shared expert · '+fmt(data?.shared)+' RMS ↗';} | |
| }else if(kind==='ffn'){ | |
| metric('dFfnInput',String(S.config.hidden_size));metric('dFfnWidth',String(S.config.intermediate_size));metric('dFfnRms',fmt(data?.ffn));metric('dFfnResidual',fmt(data?.residual));paintTimeline(kind); | |
| }else if(kind==='head'){ | |
| $('dHeadSelect').value=String(S.head);const group=headCount()/keyHeadCount(),start=keyHead()*group;metric('dKvGroup',linear?`Value head ${pad(S.head)} reads key head ${keyHead()} (${headCount()} V / ${keyHeadCount()} K)`:`Heads ${pad(start)} to ${pad(start+group-1)} share K and V`);metric('dHeadRms',fmt(data?.heads?.[S.head]));metric('dHeadDimension',String(headDimension()));metric('dKvHead',String(keyHead())); | |
| if(linear){metric('dStateRms',fmt(data?.state_rms?.[S.head]));paintMemory(data);}else{metric('dEntropy',fmt(data?.attention?.[S.head]?.entropy));$('detailBody').querySelector('.query-node span').textContent='Head '+pad(S.head)+' · KV '+keyHead();paintAttention(data);}paintHeads($('dAllHeads'),data);paintTimeline(kind); | |
| }else{ | |
| const shared=kind==='shared',rank=data?.experts?.indexOf(S.expert)??-1,selected=shared?Boolean(data):rank>=0,count=S.frames.filter(f=>shared||f.layers[S.layer].experts?.includes(S.expert)).length; | |
| $('dExpertSelect').value=shared?'shared':String(S.expert);metric('dExpertArchitecture',shared?'Always-active shared branch':'Layer-local routed expert');metric('dExpertStatus',data?(selected?'Selected':'Not selected'):'Awaiting trace');$('dExpertStatus').classList.toggle('is-active',selected);metric('dExpertWeight',shared?fmt(data?.shared):data?pct(rank>=0?data.weights[rank]:0):'unmeasured');metric('dExpertRank',shared?'Every token':rank>=0?String(rank+1)+' / '+S.config.num_experts_per_tok:data?'Outside top '+S.config.num_experts_per_tok:'unmeasured');metric('dExpertFrequency',count+' / '+S.frames.length);metric('dExpertLayer',pad(S.layer));metric('dRoutedRms',fmt(data?.routed)+' RMS');metric('dSharedRms',fmt(data?.shared)+' RMS');metric('dExpertTimelineTitle',shared?'Shared output over tokens':'Expert selection over tokens');metric('dExpertTimelineMetric',shared?'RMS':'Routing weight');$('detailBody').querySelector('.expert-core').classList.toggle('engaged',selected);paintRoutes($('dExpertRoutes'),data);paintExperts($('dAllExperts'),data);paintTimeline(kind); | |
| }renderDetailPlayback(); | |
| } | |
| function renderDetailPlayback(){ | |
| const n=S.frames.length,f=S.frames[S.frame],partial=S.live&&S.running&&S.partial.length;$('detailFrameLabel').textContent=`TOKEN ${Math.max(0,S.frame+1)} / ${n}`;$('detailToken').textContent=partial?'Capturing next token…':f?f.text.replace(/\n/g,'↵')||'∅':'Awaiting trace';$('detailToken').title=f?.text||'';$('detailInputPosition').textContent=partial?'Forward pass in progress':f?'Input position '+f.input_position:''; | |
| $('detailPlay').disabled=!n;$('detailPlay').textContent=S.timer?'Ⅱ':'▶';$('detailPlay').ariaLabel=S.timer?'Pause trace':'Play trace';$('detailStepBack').disabled=S.frame<=0;$('detailStepForward').disabled=S.frame>=n-1;$('detailSpeed').value=String(S.speed);$('detailLive').classList.toggle('selected',S.live);$('detailScrubber').disabled=!n;$('detailScrubber').max=Math.max(0,n-1);$('detailScrubber').value=Math.max(0,S.frame);$('detailPlaybackNote').textContent=S.live&&S.running?'Choose a speed to slow the captured frames.':S.running?'Replay · inference continues':S.timer?'Measured trace playback':'Step, scrub or play the measured trace.'; | |
| } | |
| function initDetail(){ | |
| $('detailSpeed').innerHTML=$('playSpeed').innerHTML;$('detailBack').onclick=closeDetail;$('detailClose').onclick=closeDetail;$('detailDialog').addEventListener('close',()=>{S.detail=null;detailLayout='';document.body.classList.remove('detail-open');});$('detailPrev').onclick=()=>selectLayer(S.layer-1);$('detailNext').onclick=()=>selectLayer(S.layer+1); | |
| document.querySelectorAll('[data-detail]').forEach(b=>b.onclick=()=>openDetail(b.dataset.detail==='expert'&&!isMoe()?'ffn':b.dataset.detail));$('detailPlay').onclick=togglePlayback;$('detailStepBack').onclick=()=>selectFrame(S.frame-1);$('detailStepForward').onclick=()=>selectFrame(S.frame+1);$('detailScrubber').oninput=()=>selectFrame(Number($('detailScrubber').value));$('detailSpeed').onchange=()=>changeSpeed($('detailSpeed').value);$('detailLive').onclick=()=>selectFrame(S.frames.length-1); | |
| $('detailDialog').addEventListener('keydown',e=>{if(/INPUT|SELECT|BUTTON/.test(e.target.tagName))return;if(e.code==='Space'){e.preventDefault();togglePlayback();}if(e.key==='ArrowLeft'){e.preventDefault();selectFrame(S.frame-1);}if(e.key==='ArrowRight'){e.preventDefault();selectFrame(S.frame+1);}}); | |
| } | |
| initDetail(); | |