Release site-v1.0.5 (part 6)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- lecture-06/assets/interactive-DDMF3hhv.css +1 -0
- lecture-06/assets/md-1POnEWPg2.js +1 -0
- lecture-06/assets/md-45uBB7cb.js +1 -0
- lecture-06/assets/md-5WEP6Md82.js +1 -0
- lecture-06/assets/md-9BRr5fKR.js +1 -0
- lecture-06/assets/md-B3cw-l9c.js +45 -0
- lecture-06/assets/md-B4dh1Gjy2.js +1 -0
- lecture-06/assets/md-B5Abi94E.js +15 -0
- lecture-06/assets/md-B5uG_D2l.js +1 -0
- lecture-06/assets/md-B815c0ps.js +1 -0
- lecture-06/assets/md-BJ1CN0D8.js +12 -0
- lecture-06/assets/md-BJLgyYot2.js +1 -0
- lecture-06/assets/md-BOLARYwC2.js +1 -0
- lecture-06/assets/md-BRWSRKtj2.js +1 -0
- lecture-06/assets/md-BTnUhx7N.css +1 -0
- lecture-06/assets/md-BX32i5rK.js +1 -0
- lecture-06/assets/md-BYKniqFA.js +12 -0
- lecture-06/assets/md-Bccft7aT.js +1 -0
- lecture-06/assets/md-Be8qHbLU.js +1 -0
- lecture-06/assets/md-BekE5qL_.js +1 -0
- lecture-06/assets/md-BexnRWui.js +1 -0
- lecture-06/assets/md-BgsK6PGZ2.js +1 -0
- lecture-06/assets/md-BjO-yXuT.js +1 -0
- lecture-06/assets/md-BkNuaaSP.js +1 -0
- lecture-06/assets/md-BmMhfMUg.js +14 -0
- lecture-06/assets/md-BpUXPIXm.js +1 -0
- lecture-06/assets/md-BqhSluwG.js +1 -0
- lecture-06/assets/md-BsLhH0zv.js +1 -0
- lecture-06/assets/md-BscC0r3L.js +8 -0
- lecture-06/assets/md-BtxXgyX-.js +1 -0
- lecture-06/assets/md-Bu64PrEi.js +1 -0
- lecture-06/assets/md-C2-OA9fh2.js +1 -0
- lecture-06/assets/md-C6qfUyBt2.js +1 -0
- lecture-06/assets/md-C8OiMOzj.js +1 -0
- lecture-06/assets/md-CB6QZfkK2.js +1 -0
- lecture-06/assets/md-CBhrP8n32.js +1 -0
- lecture-06/assets/md-CBiTRCiA2.js +1 -0
- lecture-06/assets/md-CGRw6CdD.js +1 -0
- lecture-06/assets/md-CHhgCgN4.js +1 -0
- lecture-06/assets/md-CIA7piby2.js +1 -0
- lecture-06/assets/md-CKfbaq-X.js +1 -0
- lecture-06/assets/md-CLUR6rvJ.js +1 -0
- lecture-06/assets/md-CNDLpstn2.js +1 -0
- lecture-06/assets/md-CSrnKzHt.js +1 -0
- lecture-06/assets/md-CV1HWu48.js +1 -0
- lecture-06/assets/md-CVYuYWEV.js +1 -0
- lecture-06/assets/md-CW3GRSxh2.js +1 -0
- lecture-06/assets/md-CWLf7t6o.js +1 -0
- lecture-06/assets/md-CZ0TTTDs.js +12 -0
- lecture-06/assets/md-Cbf_c7qC.js +12 -0
lecture-06/assets/interactive-DDMF3hhv.css
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
.dl-widget[data-v-0f64089e]{flex-direction:column;gap:.6rem;width:100%;height:100%;min-height:0;display:flex}.dl-widget.is-controls-first[data-v-0f64089e]{flex-direction:column-reverse;justify-content:flex-end}.dl-widget__stage[data-v-0f64089e]{flex:auto;justify-content:center;align-items:center;min-height:0;display:flex}.dl-widget__controls[data-v-0f64089e]{flex-wrap:wrap;flex:none;align-items:center;gap:.5rem 1.2rem;display:flex}.dl-widget__controls[data-v-0f64089e]:empty,.dl-widget__readout[data-v-0f64089e]:empty{display:none}.dl-widget__readout[data-v-0f64089e]{color:var(--dl-muted);font-variant-numeric:tabular-nums;flex:none;font-size:.82rem}.dl-interactive[data-v-0c449bbd]{flex-direction:column;display:flex}.dl-interactive__title[data-v-0c449bbd]{margin-bottom:.7rem}.dl-interactive__grid[data-v-0c449bbd]{flex:auto;align-items:stretch;gap:1.6rem;min-height:0;display:flex}.dl-interactive__grid.is-aside-left[data-v-0c449bbd]{flex-direction:row-reverse}.dl-interactive__stage[data-v-0c449bbd]{flex:auto;justify-content:center;align-items:center;min-width:0;display:flex}.dl-interactive__aside[data-v-0c449bbd]{font-size:var(--dl-fs-secondary);flex-grow:0;flex-shrink:0;overflow-y:auto}.dl-interactive__aside[data-v-0c449bbd]:empty{display:none}
|
lecture-06/assets/md-1POnEWPg2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,W as r,_ as i,_t as a,b as o,v as s,x as c,yt as l,z as u}from"./modules/shiki-BFOesOFp.js";import{n as d,t as f}from"./slidev/context-DrkI8qiD.js";import{t as p}from"./default-sHmABwVd.js";var m={class:`grid grid-cols-[1.3fr_1fr] gap-8 mt-2 items-start`},h={viewBox:`0 0 420 184`,class:`dl-diagram`,style:{"max-height":`14rem`},role:`img`,"aria-label":`The three words of the river bank, each with a bar chart of its four embedding numbers: the is 1 0 0 0, river is 0 2 0 1, bank is 0 0 0 2`},g={class:`dl-tight`},_={class:`mt-3 dl-callout`},v={class:`mt-3`},y={class:`mt-3 dl-secondary`},b={__name:`slides.md__slidev_25`,setup(b){let{$slidev:x,$nav:S,$clicksContext:C,$clicks:w,$page:T,$renderContext:E,$frontmatter:D}=d();return C.setup(),(d,b)=>{let x=r(`click`);return u(),s(p,l(t(a(f)(a(D),24))),{default:n(()=>[b[27]||=i(`h1`,null,`The sentence we will use all lecture`,-1),i(`div`,m,[i(`div`,null,[(u(),o(`svg`,h,[b[2]||=i(`rect`,{class:`dl-dg-box`,x:`10`,y:`8`,width:`112`,height:`28`,rx:`4`},null,-1),b[3]||=i(`rect`,{class:`dl-dg-box`,x:`150`,y:`8`,width:`112`,height:`28`,rx:`4`},null,-1),b[4]||=i(`rect`,{class:`dl-dg-box is-accent`,x:`290`,y:`8`,width:`112`,height:`28`,rx:`4`},null,-1),b[5]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`66`,y:`27`,"text-anchor":`middle`,style:{"font-style":`italic`}},`the`,-1),b[6]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`206`,y:`27`,"text-anchor":`middle`,style:{"font-style":`italic`}},`river`,-1),b[7]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`346`,y:`27`,"text-anchor":`middle`,style:{"font-style":`italic`}},`bank`,-1),b[8]||=i(`line`,{class:`dl-dg-split`,x1:`10`,y1:`150`,x2:`122`,y2:`150`},null,-1),b[9]||=i(`line`,{class:`dl-dg-split`,x1:`150`,y1:`150`,x2:`262`,y2:`150`},null,-1),b[10]||=i(`line`,{class:`dl-dg-split`,x1:`290`,y1:`150`,x2:`402`,y2:`150`},null,-1),b[11]||=i(`text`,{class:`dl-dg-small`,x:`24`,y:`166`,"text-anchor":`middle`},`det`,-1),b[12]||=i(`text`,{class:`dl-dg-small`,x:`52`,y:`180`,"text-anchor":`middle`},`water`,-1),b[13]||=i(`text`,{class:`dl-dg-small`,x:`80`,y:`166`,"text-anchor":`middle`},`money`,-1),b[14]||=i(`text`,{class:`dl-dg-small`,x:`108`,y:`180`,"text-anchor":`middle`},`thing`,-1),b[15]||=i(`text`,{class:`dl-dg-small`,x:`164`,y:`166`,"text-anchor":`middle`},`det`,-1),b[16]||=i(`text`,{class:`dl-dg-small`,x:`192`,y:`180`,"text-anchor":`middle`},`water`,-1),b[17]||=i(`text`,{class:`dl-dg-small`,x:`220`,y:`166`,"text-anchor":`middle`},`money`,-1),b[18]||=i(`text`,{class:`dl-dg-small`,x:`248`,y:`180`,"text-anchor":`middle`},`thing`,-1),b[19]||=i(`text`,{class:`dl-dg-small`,x:`304`,y:`166`,"text-anchor":`middle`},`det`,-1),b[20]||=i(`text`,{class:`dl-dg-small`,x:`332`,y:`180`,"text-anchor":`middle`},`water`,-1),b[21]||=i(`text`,{class:`dl-dg-small`,x:`360`,y:`166`,"text-anchor":`middle`},`money`,-1),b[22]||=i(`text`,{class:`dl-dg-small`,x:`388`,y:`180`,"text-anchor":`middle`},`thing`,-1),e((u(),o(`g`,null,[...b[0]||=[i(`rect`,{class:`dl-dg-bar`,x:`14`,y:`110`,width:`20`,height:`40`},null,-1),i(`text`,{class:`dl-dg-small`,x:`24`,y:`104`,"text-anchor":`middle`},`1`,-1),i(`rect`,{class:`dl-dg-bar`,x:`182`,y:`70`,width:`20`,height:`80`},null,-1),i(`text`,{class:`dl-dg-small`,x:`192`,y:`64`,"text-anchor":`middle`},`2`,-1),i(`rect`,{class:`dl-dg-bar`,x:`238`,y:`110`,width:`20`,height:`40`},null,-1),i(`text`,{class:`dl-dg-small`,x:`248`,y:`104`,"text-anchor":`middle`},`1`,-1)]])),[[x]]),e((u(),o(`g`,null,[...b[1]||=[i(`rect`,{class:`dl-dg-bar is-q`,x:`378`,y:`70`,width:`20`,height:`80`},null,-1),i(`text`,{class:`dl-dg-small`,x:`388`,y:`64`,"text-anchor":`middle`},`2`,-1)]])),[[x]])]))]),i(`div`,g,[b[25]||=i(`p`,null,[c(`Four embedding numbers per word. We pretend they mean `),i(`strong`,null,`det`),c(` (a word like `),i(`em`,null,`the`),c(`), `),i(`strong`,null,`water`),c(`, `),i(`strong`,null,`money`),c(` and `),i(`strong`,null,`thing`),c(`.`)],-1),e((u(),o(`div`,_,[...b[23]||=[i(`p`,null,[i(`em`,null,`bank`),c(` is a thing, and `),i(`strong`,null,`nothing else`),c(`. Nothing says riverbank or high-street bank.`)],-1)]])),[[x]]),e((u(),o(`div`,v,[...b[24]||=[i(`p`,null,`So the sense has to come from a neighbour. Attention is how it gets there.`,-1)]])),[[x]])])]),e((u(),o(`div`,y,[...b[26]||=[i(`p`,null,`Hand-picked numbers, so you can check the arithmetic. Real coordinates have no names.`,-1)]])),[[x]])]),_:1},16)}}};export{b as default};
|
lecture-06/assets/md-45uBB7cb.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_54`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),53))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: Attention does not know what order the words came in`,-1),n(`div`,{class:`dl-describe`},[n(`p`,null,`The widget shows each word’s output vector from one attention layer. The button shuffles the order of the words.`),n(`ul`,null,[n(`li`,null,`After a shuffle, every word gets exactly the same output as before. Only its place in the list changes.`),n(`li`,null,`The reason: each output is a weighted sum over all the words. A sum gives the same answer in any order.`),n(`li`,null,[a(`So, to attention alone, `),n(`em`,null,`the river bank`),a(` and `),n(`em`,null,`bank river the`),a(` are the same input.`)]),n(`li`,null,`A recurrent network knew the order because it read one word at a time. Deleting the recurrence removed that knowledge.`)]),n(`p`,null,`Takeaway: the model needs another way to know where each word is.`)],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-5WEP6Md82.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,g as l,lt as u,m as d,v as f,vt as p,w as m,x as h,xt as g,y as _,yt as v,z as y}from"./modules/shiki-BFOesOFp.js";import{t as b}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as x,t as S}from"./slidev/context-DrkI8qiD.js";import{t as C}from"./slidev/VClicks-CGAOkBil.js";import{n as w,t as T}from"./interactive-ClWDKV9B.js";import{t as E}from"./StepButton-BcP33F-1.js";var D=[`viewBox`],O=[`x`,`width`],k=[`x`],A=[`x`,`width`],j=[`x`],M=[`d`],N=[`cx`],P=[`x`],F=[`x`],I=[`x`,`y`],L=[`x`,`y`],R=[`x`,`y`],z=640,B=372,V=218,H=56,U=366,W=30,G=b(m({__name:`TransformerStack`,props:{layers:{default:6}},setup(e){let{$slidev:t,$nav:a,$clicksContext:s,$clicks:m,$page:v,$renderContext:b,$frontmatter:S}=x(),C=e,T=[{id:`enc-emb`,x:H,y:300,label:`Input embedding`,sub:`one vector per source token`},{id:`enc-sa`,x:H,y:218,label:`Self-attention`,sub:`every token sees every token`},{id:`enc-ff`,x:H,y:180,label:`Feed-forward`,sub:`each position on its own`},{id:`dec-emb`,x:U,y:300,label:`Output embedding`,sub:`what has been written so far`},{id:`dec-sa`,x:U,y:218,label:`Masked self-attention`,sub:`no peeking at the future`},{id:`dec-ca`,x:U,y:180,label:`Cross-attention`,sub:`Q from here, K and V from the encoder`},{id:`dec-ff`,x:U,y:142,label:`Feed-forward`},{id:`dec-lin`,x:U,y:96,label:`Linear`,sub:`d → vocabulary size`},{id:`dec-sm`,x:U,y:58,label:`Softmax`,sub:`a probability per word in the vocabulary`}],G=[{title:`Embed the source`,lit:[`enc-in`,`enc-emb`],note:"The source sentence is split into tokens and each one is looked up in an embedding table (`nn.Embedding`), which turns a token id into a learned vector. Nothing here knows anything about order yet."},{title:`Add position`,lit:[`enc-in`,`enc-emb`,`enc-pe`],note:`A fixed positional vector is added to each embedding. Without it the encoder would treat the sentence as a bag of words, because a weighted sum has no order in it.`},{title:`Encoder self-attention`,lit:[`enc-sa`],note:`Every source token attends to every other, in both directions — there is no mask here, because the whole source is available at once. This is the step that makes each vector *contextual*.`},{title:`Encoder block, ×N`,lit:[`enc-sa`,`enc-ff`,`enc-stack`],note:`Attention and feed-forward, each wrapped in a residual and a LayerNorm, repeated N times. The output is one vector per source token — not one vector for the whole sentence, which is precisely the bottleneck this architecture removes.`},{title:`The decoder starts`,lit:[`dec-in`,`dec-emb`,`dec-pe`],note:`The decoder is fed what has been produced so far, shifted right by one and starting from a begin-of-sequence token. During training that is the true target sentence, so every position trains at once. This is teacher forcing: training on the true previous words instead of the model’s own guesses.`},{title:`Masked self-attention`,lit:[`dec-sa`],note:`The decoder attends to its own output, with position t forbidden from seeing anything after t. Without the mask the model could read the answer off its own input and the loss would collapse to nothing.`},{title:`Cross-attention`,lit:[`dec-ca`,`cross`],note:`The only place the two stacks meet. Queries come from the decoder — “what do I need next?” — and keys and values from the encoder’s output. This is Bahdanau’s 2014 attention for translation, with the recurrence removed from both sides.`},{title:`Decoder block, ×N`,lit:[`dec-sa`,`dec-ca`,`dec-ff`,`dec-stack`],note:`Three sub-layers now rather than two, again each with a residual and a norm, again repeated N times.`},{title:`Predict the next token`,lit:[`dec-lin`,`dec-sm`,`dec-out`],note:`A linear layer to vocabulary size and a softmax: a probability for every word. Take one, append it, and run the decoder again. The encoder runs once; the decoder runs once per output token — which is where “transformers are parallel” gets its asterisk.`}],K=u(0),q=l(()=>K.value>=G.length),J=l(()=>K.value===0?null:G[K.value-1]);function Y(){K.value=q.value?0:K.value+1}function X(e){return K.value===0?`idle`:J.value?.lit.includes(e)?`current`:G.slice(0,K.value).some(t=>t.lit.includes(e))?`seen`:`idle`}return(e,t)=>{let a=E,s=w;return y(),f(s,{"max-width":`43rem`},{controls:n(()=>[r(a,{label:q.value?`Start over`:K.value===0?`Walk the architecture`:`Next step`,glyph:`▸`,onClick:Y},null,8,[`label`])]),readout:n(()=>[J.value?(y(),c(d,{key:0},[o(`strong`,null,g(K.value)+`/`+g(G.length)+` · `+g(J.value.title),1),h(` — `+g(J.value.note),1)],64)):(y(),c(d,{key:1},[t[1]||=h(` Six boxes you have already met, arranged into two stacks. Press `,-1),t[2]||=o(`em`,null,`Walk the architecture`,-1),t[3]||=h(` and follow the data: up the encoder once, then up the decoder once for every word it writes. `,-1)],64))]),default:n(()=>[(y(),c(`svg`,{class:`dl-stack`,viewBox:`0 0 ${z} ${B}`,preserveAspectRatio:`xMidYMid meet`,role:`img`,"aria-label":`The 2017 encoder-decoder transformer: an encoder stack reading the source sentence and a decoder stack that attends to its own output and to the encoder's`},[t[0]||=o(`defs`,null,[o(`marker`,{id:`dl-stack-arrow`,viewBox:`0 0 8 8`,refX:`7`,refY:`4`,markerWidth:`5`,markerHeight:`5`,orient:`auto-start-reverse`},[o(`path`,{d:`M0,0 L8,4 L0,8 z`,fill:`currentColor`})])],-1),_(` Column spines, drawn first so the boxes sit on top of them. `),o(`line`,{class:`dl-stack__spine`,x1:165,y1:336,x2:165,y2:176}),o(`line`,{class:`dl-stack__spine`,x1:475,y1:336,x2:475,y2:54,"marker-end":`url(#dl-stack-arrow)`}),_(` Repeated-block brackets. `),o(`rect`,{class:p([`dl-stack__repeat`,`is-${X(`enc-stack`)}`]),x:44,y:`172`,width:242,height:`84`,rx:`8`},null,10,O),o(`text`,{class:p([`dl-stack__repeatlabel`,`is-${X(`enc-stack`)}`]),x:38,y:`214`,"text-anchor":`end`},`×`+g(C.layers),11,k),o(`rect`,{class:p([`dl-stack__repeat`,`is-${X(`dec-stack`)}`]),x:354,y:`134`,width:242,height:`122`,rx:`8`},null,10,A),o(`text`,{class:p([`dl-stack__repeatlabel`,`is-${X(`dec-stack`)}`]),x:602,y:`198`,"text-anchor":`start`},`×`+g(C.layers),11,j),_(` The one place the two stacks touch. `),o(`path`,{class:p([`dl-stack__cross`,`is-${X(`cross`)}`]),d:`M 165 176 V 158 H 320 V 195 H 362`,"marker-end":`url(#dl-stack-arrow)`},null,10,M),o(`text`,{class:p([`dl-stack__crosslabel`,`is-${X(`cross`)}`]),x:`318`,y:`152`,"text-anchor":`middle`},` K, V from the encoder `,2),_(` Positional-encoding adders. `),(y(!0),c(d,null,i([{id:`enc-pe`,x:165},{id:`dec-pe`,x:475}],e=>(y(),c(`g`,{key:e.id},[o(`circle`,{class:p([`dl-stack__add`,`is-${X(e.id)}`]),cx:e.x,cy:`272`,r:`11`},null,10,N),o(`text`,{class:`dl-stack__plus`,x:e.x,y:`277`,"text-anchor":`middle`},`+`,8,P),o(`text`,{class:p([`dl-stack__pe`,`is-${X(e.id)}`]),x:e.x-18,y:`276`,"text-anchor":`end`},`position`,10,F)]))),128)),_(` The boxes. `),(y(),c(d,null,i(T,e=>o(`g`,{key:e.id},[o(`rect`,{class:p([`dl-stack__box`,`is-${X(e.id)}`]),x:e.x,y:e.y,width:V,height:W,rx:`5`},null,10,I),o(`text`,{class:p([`dl-stack__label`,`is-${X(e.id)}`]),x:e.x+V/2,y:e.y+(e.sub?14:20),"text-anchor":`middle`},g(e.label),11,L),e.sub?(y(),c(`text`,{key:0,class:p([`dl-stack__sub`,`is-${X(e.id)}`]),x:e.x+V/2,y:e.y+25,"text-anchor":`middle`},g(e.sub),11,R)):_(`v-if`,!0)])),64)),_(` Ports. `),o(`text`,{class:p([`dl-stack__port`,`is-${X(`enc-in`)}`]),x:165,y:`350`,"text-anchor":`middle`},` source sentence `,2),o(`text`,{class:p([`dl-stack__port`,`is-${X(`dec-in`)}`]),x:475,y:`350`,"text-anchor":`middle`},` output so far, shifted right `,2),o(`text`,{class:p([`dl-stack__port`,`is-${X(`dec-out`)}`]),x:475,y:`44`,"text-anchor":`middle`},` probability of every next word `,2),o(`text`,{class:`dl-stack__side`,x:H,y:`18`},`ENCODER — reads once`),o(`text`,{class:`dl-stack__side`,x:U,y:`18`},`DECODER — runs once per output token`)],8,D))]),_:1})}}}),[[`__scopeId`,`data-v-66f48009`]]),K={class:`mt-2 dl-callout`},q={__name:`slides.md__slidev_16`,setup(i){let{$slidev:l,$nav:u,$clicksContext:d,$clicks:p,$page:m,$renderContext:g,$frontmatter:_}=x();return d.setup(),(i,l)=>{let u=G,d=C,p=a(`click`);return y(),f(T,v(t(s(S)(s(_),15))),{aside:n(t=>[l[2]||=o(`p`,null,[h(`Press `),o(`strong`,null,`Walk the architecture`),h(` and follow the data: nine steps, no new boxes.`)],-1),r(d,null,{default:n(()=>[...l[0]||=[o(`ul`,null,[o(`li`,null,[h(`up the encoder `),o(`strong`,null,`once`)]),o(`li`,null,[h(`then up the decoder `),o(`strong`,null,`once per word it writes`)]),o(`li`,null,`cross-attention is the only wire between them`)],-1)]]),_:1}),e((y(),c(`div`,K,[...l[1]||=[o(`p`,null,`The famous figure, in the order the data actually moves.`,-1)]])),[[p]])]),default:n(()=>[r(u,{layers:6})]),_:1},16)}}};export{q as default};
|
lecture-06/assets/md-9BRr5fKR.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`slides.md__slidev_46`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),45))),{default:t(()=>[p[0]||=r(`h1`,null,`From one head to one block`,-1),p[1]||=r(`p`,null,`Many heads, one block.`,-1),r(`div`,f,[n(m,{kind:`block`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/assets/md-B3cw-l9c.js
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_39`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),38))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: Why divide by the square root`,-1),n(`div`,{class:`dl-describe`},[n(`ul`,null,[n(`li`,null,[a(`A score is a dot product: `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`k`)])]),n(`annotation`,{encoding:`application/x-tex`},`d_k`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3361em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(` products added together. Wider keys give bigger scores, typically growing like `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msqrt`,null,[n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`k`)])])]),n(`annotation`,{encoding:`application/x-tex`},`\\sqrt{d_k}`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1.04em`,"vertical-align":`-0.1828em`}}),n(`span`,{class:`mord sqrt`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.8572em`}},[n(`span`,{class:`svg-align`,style:{top:`-3em`}},[n(`span`,{class:`pstrut`,style:{height:`3em`}}),n(`span`,{class:`mord`,style:{"padding-left":`0.833em`}},[n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3361em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])]),n(`span`,{style:{top:`-2.8172em`}},[n(`span`,{class:`pstrut`,style:{height:`3em`}}),n(`span`,{class:`hide-tail`,style:{"min-width":`0.853em`,height:`1.08em`}},[n(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`1.08em`,viewBox:`0 0 400000 1080`,preserveAspectRatio:`xMinYMin slice`},[n(`path`,{d:`M95,702
|
| 2 |
+
c-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14
|
| 3 |
+
c0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54
|
| 4 |
+
c44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10
|
| 5 |
+
s173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429
|
| 6 |
+
c69,-144,104.5,-217.7,106.5,-221
|
| 7 |
+
l0 -0
|
| 8 |
+
c5.3,-9.3,12,-14,20,-14
|
| 9 |
+
H400000v40H845.2724
|
| 10 |
+
s-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7
|
| 11 |
+
c-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z
|
| 12 |
+
M834 80h400000v40h-400000z`})])])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.1828em`}},[n(`span`)])])])])])])]),a(`.`)]),n(`li`,null,`The left bars are the softmax of the scores 0, 2, 0: weights 0.11, 0.79, 0.11. The right bars use the same scores eight times bigger, 0, 16, 0. The weights become 0.00, 1.00, 0.00.`),n(`li`,null,[a(`That softmax is `),n(`strong`,null,`saturated`),a(`. Its gradient is almost zero, so the layer stops learning where to look.`)]),n(`li`,null,[a(`The fix is `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`α`),n(`mrow`,null,[n(`mi`,null,`i`),n(`mi`,null,`j`)])]),n(`mo`,null,`=`),n(`msub`,null,[n(`mrow`,null,[n(`mi`,{mathvariant:`normal`},`softmax`),n(`mo`,null,``)]),n(`mi`,null,`j`)]),n(`mo`,{stretchy:`false`},`(`),n(`msub`,null,[n(`mi`,{mathvariant:`bold`},`q`),n(`mi`,null,`i`)]),n(`mo`,null,`⋅`),n(`msub`,null,[n(`mi`,{mathvariant:`bold`},`k`),n(`mi`,null,`j`)]),n(`mi`,{mathvariant:`normal`},`/`),n(`msqrt`,null,[n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`k`)])]),n(`mo`,{stretchy:`false`},`)`)]),n(`annotation`,{encoding:`application/x-tex`},`\\alpha_{ij} = \\operatorname{softmax}_j(\\mathbf{q}_i \\cdot \\mathbf{k}_j / \\sqrt{d_k})`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.7167em`,"vertical-align":`-0.2861em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0037em`}},`α`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3117em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`-0.0037em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0572em`}},`ij`)])])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.2861em`}},[n(`span`)])])])])]),n(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),n(`span`,{class:`mrel`},`=`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1.0361em`,"vertical-align":`-0.2861em`}}),n(`span`,{class:`mop`},[n(`span`,{class:`mop`},[n(`span`,{class:`mord mathrm`},`softmax`)]),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3117em`}},[n(`span`,{style:{top:`-2.55em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0572em`}},`j`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.2861em`}},[n(`span`)])])])])]),n(`span`,{class:`mopen`},`(`),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathbf`},`q`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3117em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`},`i`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])]),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),n(`span`,{class:`mbin`},`⋅`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1.1433em`,"vertical-align":`-0.2861em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathbf`},`k`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3117em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0572em`}},`j`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.2861em`}},[n(`span`)])])])])]),n(`span`,{class:`mord`},`/`),n(`span`,{class:`mord sqrt`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.8572em`}},[n(`span`,{class:`svg-align`,style:{top:`-3em`}},[n(`span`,{class:`pstrut`,style:{height:`3em`}}),n(`span`,{class:`mord`,style:{"padding-left":`0.833em`}},[n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3361em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])]),n(`span`,{style:{top:`-2.8172em`}},[n(`span`,{class:`pstrut`,style:{height:`3em`}}),n(`span`,{class:`hide-tail`,style:{"min-width":`0.853em`,height:`1.08em`}},[n(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`1.08em`,viewBox:`0 0 400000 1080`,preserveAspectRatio:`xMinYMin slice`},[n(`path`,{d:`M95,702
|
| 13 |
+
c-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14
|
| 14 |
+
c0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54
|
| 15 |
+
c44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10
|
| 16 |
+
s173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429
|
| 17 |
+
c69,-144,104.5,-217.7,106.5,-221
|
| 18 |
+
l0 -0
|
| 19 |
+
c5.3,-9.3,12,-14,20,-14
|
| 20 |
+
H400000v40H845.2724
|
| 21 |
+
s-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7
|
| 22 |
+
c-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z
|
| 23 |
+
M834 80h400000v40h-400000z`})])])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.1828em`}},[n(`span`)])])])]),n(`span`,{class:`mclose`},`)`)])])]),a(`. Dividing by `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msqrt`,null,[n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`k`)])])]),n(`annotation`,{encoding:`application/x-tex`},`\\sqrt{d_k}`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1.04em`,"vertical-align":`-0.1828em`}}),n(`span`,{class:`mord sqrt`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.8572em`}},[n(`span`,{class:`svg-align`,style:{top:`-3em`}},[n(`span`,{class:`pstrut`,style:{height:`3em`}}),n(`span`,{class:`mord`,style:{"padding-left":`0.833em`}},[n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3361em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])]),n(`span`,{style:{top:`-2.8172em`}},[n(`span`,{class:`pstrut`,style:{height:`3em`}}),n(`span`,{class:`hide-tail`,style:{"min-width":`0.853em`,height:`1.08em`}},[n(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`1.08em`,viewBox:`0 0 400000 1080`,preserveAspectRatio:`xMinYMin slice`},[n(`path`,{d:`M95,702
|
| 24 |
+
c-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14
|
| 25 |
+
c0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54
|
| 26 |
+
c44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10
|
| 27 |
+
s173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429
|
| 28 |
+
c69,-144,104.5,-217.7,106.5,-221
|
| 29 |
+
l0 -0
|
| 30 |
+
c5.3,-9.3,12,-14,20,-14
|
| 31 |
+
H400000v40H845.2724
|
| 32 |
+
s-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7
|
| 33 |
+
c-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z
|
| 34 |
+
M834 80h400000v40h-400000z`})])])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.1828em`}},[n(`span`)])])])])])])]),a(` cancels the growth.`)]),n(`li`,null,[a(`For `),n(`em`,null,`bank`),a(` (`),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`k`)]),n(`mo`,null,`=`),n(`mn`,null,`2`)]),n(`annotation`,{encoding:`application/x-tex`},`d_k = 2`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3361em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])]),n(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),n(`span`,{class:`mrel`},`=`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),n(`span`,{class:`mord`},`2`)])])]),a(`): `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mn`,null,`2`),n(`mi`,{mathvariant:`normal`},`/`),n(`msqrt`,null,[n(`mn`,null,`2`)]),n(`mo`,null,`=`),n(`mn`,null,`1.41`)]),n(`annotation`,{encoding:`application/x-tex`},`2 / \\sqrt{2} = 1.41`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1.1572em`,"vertical-align":`-0.25em`}}),n(`span`,{class:`mord`},`2/`),n(`span`,{class:`mord sqrt`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.9072em`}},[n(`span`,{class:`svg-align`,style:{top:`-3em`}},[n(`span`,{class:`pstrut`,style:{height:`3em`}}),n(`span`,{class:`mord`,style:{"padding-left":`0.833em`}},[n(`span`,{class:`mord`},`2`)])]),n(`span`,{style:{top:`-2.8672em`}},[n(`span`,{class:`pstrut`,style:{height:`3em`}}),n(`span`,{class:`hide-tail`,style:{"min-width":`0.853em`,height:`1.08em`}},[n(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`1.08em`,viewBox:`0 0 400000 1080`,preserveAspectRatio:`xMinYMin slice`},[n(`path`,{d:`M95,702
|
| 35 |
+
c-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14
|
| 36 |
+
c0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54
|
| 37 |
+
c44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10
|
| 38 |
+
s173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429
|
| 39 |
+
c69,-144,104.5,-217.7,106.5,-221
|
| 40 |
+
l0 -0
|
| 41 |
+
c5.3,-9.3,12,-14,20,-14
|
| 42 |
+
H400000v40H845.2724
|
| 43 |
+
s-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7
|
| 44 |
+
c-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z
|
| 45 |
+
M834 80h400000v40h-400000z`})])])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.1328em`}},[n(`span`)])])])]),n(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),n(`span`,{class:`mrel`},`=`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),n(`span`,{class:`mord`},`1.41`)])])]),a(`, and the weight on `),n(`em`,null,`river`),a(` is 0.67.`)]),n(`li`,null,[a(`In the 2017 model, `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`k`)]),n(`mo`,null,`=`),n(`mn`,null,`64`)]),n(`annotation`,{encoding:`application/x-tex`},`d_k = 64`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3361em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])]),n(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),n(`span`,{class:`mrel`},`=`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),n(`span`,{class:`mord`},`64`)])])]),a(`, so the divisor is 8.`)])])],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-B4dh1Gjy2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_24`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),23))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: Three roles, one vector`,-1),n(`div`,{class:`dl-describe`},[n(`ul`,null,[n(`li`,null,[a(`The box `),n(`strong`,null,`x`),a(` on the left is one token’s vector.`)]),n(`li`,null,[a(`It goes through three different learned matrices, `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`W`),n(`mi`,null,`Q`)])]),n(`annotation`,{encoding:`application/x-tex`},`W_Q`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.9694em`,"vertical-align":`-0.2861em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3283em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`},`Q`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.2861em`}},[n(`span`)])])])])])])])]),a(`, `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`W`),n(`mi`,null,`K`)])]),n(`annotation`,{encoding:`application/x-tex`},`W_K`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3283em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0715em`}},`K`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(` and `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`W`),n(`mi`,null,`V`)])]),n(`annotation`,{encoding:`application/x-tex`},`W_V`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3283em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.2222em`}},`V`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(` (the trapezoids). Each one gives a different short vector.`)]),n(`li`,null,[a(`The `),n(`strong`,null,`query`),a(),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,{mathvariant:`bold`},`q`)]),n(`annotation`,{encoding:`application/x-tex`},`\\mathbf{q}`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6389em`,"vertical-align":`-0.1944em`}}),n(`span`,{class:`mord mathbf`},`q`)])])]),a(` says what this token is looking for. It is compared with every token’s key.`)]),n(`li`,null,[a(`The `),n(`strong`,null,`key`),a(),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,{mathvariant:`bold`},`k`)]),n(`annotation`,{encoding:`application/x-tex`},`\\mathbf{k}`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6944em`}}),n(`span`,{class:`mord mathbf`},`k`)])])]),a(` says what this token advertises, so that others can find it.`)]),n(`li`,null,[a(`The `),n(`strong`,null,`value`),a(),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,{mathvariant:`bold`},`v`)]),n(`annotation`,{encoding:`application/x-tex`},`\\mathbf{v}`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.4444em`}}),n(`span`,{class:`mord mathbf`,style:{"margin-right":`0.016em`}},`v`)])])]),a(` is what this token hands over when another token attends to it. Values are averaged into the output.`)]),n(`li`,null,[a(`Every token plays all three roles at once, even when it looks at itself. `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`W`),n(`mi`,null,`Q`)])]),n(`annotation`,{encoding:`application/x-tex`},`W_Q`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.9694em`,"vertical-align":`-0.2861em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3283em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`},`Q`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.2861em`}},[n(`span`)])])])])])])])]),a(`, `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`W`),n(`mi`,null,`K`)])]),n(`annotation`,{encoding:`application/x-tex`},`W_K`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3283em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0715em`}},`K`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(` and `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`W`),n(`mi`,null,`V`)])]),n(`annotation`,{encoding:`application/x-tex`},`W_V`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3283em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.2222em`}},`V`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(` are the only new learned parameters.`)])])],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-B5Abi94E.js
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,M as n,Q as r,S as i,W as a,_ as o,_t as s,b as c,v as l,x as u,yt as d,z as f}from"./modules/shiki-BFOesOFp.js";import{n as p,t as m}from"./slidev/context-DrkI8qiD.js";import{t as h}from"./default-sHmABwVd.js";import{t as g}from"./slidev/CodeBlockWrapper-C_5MjYK7.js";var _={class:`grid grid-cols-[1fr_15rem] gap-4 items-center`},v={class:`mt-2 dl-secondary`},y={__name:`slides.md__slidev_101`,setup(y){let{$slidev:b,$nav:x,$clicksContext:S,$clicks:C,$page:w,$renderContext:T,$frontmatter:E}=p();return S.setup(),(p,y)=>{let b=g,x=a(`click`);return f(),l(h,d(t(s(m)(s(E),100))),{default:r(()=>[y[3]||=o(`h1`,null,`A block, and a stack`,-1),o(`div`,_,[o(`div`,null,[i(b,n({lines:!0},{title:``,ranges:[`all`,`4-8`,`11-12`,`15`,`all`]}),{default:r(()=>[...y[0]||=[o(`pre`,{class:`shiki shiki-themes vitesse-dark vitesse-light slidev-code`,style:{"--shiki-dark":`#dbd7caee`,"--shiki-light":`#393a34`,"--shiki-dark-bg":`#121212`,"--shiki-light-bg":`#ffffff`}},[o(`code`,{class:`language-python`},[o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#CB7676`,"--shiki-light":`#AB5959`}},`class`),o(`span`,{style:{"--shiki-dark":`#5DA994`,"--shiki-light":`#2E8F82`}},` Block`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#80A665`,"--shiki-light":`#59873A`}},`nn`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#80A665`,"--shiki-light":`#59873A`}},`Module`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`):`)]),u(`
|
| 2 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#CB7676`,"--shiki-light":`#AB5959`}},` def`),o(`span`,{style:{"--shiki-dark":`#B8A965`,"--shiki-light":`#998418`}},` __init__`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` d_model`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` n_heads`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` d_ff`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`):`)]),u(`
|
| 3 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#B8A965`,"--shiki-light":`#998418`}},` super`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`().`),o(`span`,{style:{"--shiki-dark":`#B8A965`,"--shiki-light":`#998418`}},`__init__`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`()`)]),u(`
|
| 4 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`n1 `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` nn`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`LayerNorm`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`d_model`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`)]),u(`
|
| 5 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`attn `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` SelfAttention`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`d_model`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` n_heads`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`)]),u(`
|
| 6 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`n2 `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` nn`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`LayerNorm`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`d_model`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`)]),u(`
|
| 7 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`ff `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` nn`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`Sequential`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`nn`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`Linear`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`d_model`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` d_ff`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`),`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` nn`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`GELU`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(),`)]),u(`
|
| 8 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` nn`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`Linear`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`d_ff`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` d_model`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`))`)]),u(`
|
| 9 |
+
`),o(`span`,{class:`line`}),u(`
|
| 10 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#CB7676`,"--shiki-light":`#AB5959`}},` def`),o(`span`,{style:{"--shiki-dark":`#80A665`,"--shiki-light":`#59873A`}},` forward`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` x`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`):`)]),u(`
|
| 11 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` x `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` x `),o(`span`,{style:{"--shiki-dark":`#CB7676`,"--shiki-light":`#AB5959`}},`+`),o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`attn`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},`self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`n1`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`x`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`))`),o(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # pre-LN: normalise the copy…`)]),u(`
|
| 12 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` x `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` x `),o(`span`,{style:{"--shiki-dark":`#CB7676`,"--shiki-light":`#AB5959`}},`+`),o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`ff`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},`self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`n2`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`x`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`))`),o(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # …never the residual stream itself`)]),u(`
|
| 13 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},` return`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` x`)]),u(`
|
| 14 |
+
`),o(`span`,{class:`line`}),u(`
|
| 15 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`model `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` nn`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`Sequential`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#CB7676`,"--shiki-light":`#AB5959`}},`*`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`[`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`Block`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},`512`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},` 8`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},` 2048`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`),o(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},` for`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` _ `),o(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`in`),o(`span`,{style:{"--shiki-dark":`#B8A965`,"--shiki-light":`#998418`}},` range`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},`6`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)])`)])])],-1)]]),_:1},16)]),y[1]||=o(`div`,null,[o(`svg`,{class:`dl-diagram`,viewBox:`0 0 230 290`,role:`img`,"aria-label":`One pre-LN block, read from the bottom. The residual stream x runs straight up. A copy branches off, goes through a norm and attention, and is added back at the first plus sign. A second copy goes through a norm and the feed-forward network and is added back at the second plus sign. Six of these blocks make the stack.`},[o(`defs`,null,[o(`marker`,{id:`l6s5-a3`,markerWidth:`8`,markerHeight:`8`,refX:`7`,refY:`4`,orient:`auto`},[o(`path`,{d:`M0 0 L8 4 L0 8 z`,class:`dl-dg-head`})])]),o(`path`,{d:`M30 280 V214`,class:`dl-dg-line`}),o(`path`,{d:`M30 198 V104`,class:`dl-dg-line`}),o(`path`,{d:`M30 88 V22`,class:`dl-dg-line`,"marker-end":`url(#l6s5-a3)`}),o(`text`,{x:`44`,y:`282`,class:`dl-dg-in`},`x`),o(`text`,{x:`44`,y:`22`,class:`dl-dg-small`},`× 6 blocks`),o(`circle`,{cx:`30`,cy:`206`,r:`8`,class:`dl-dg-op`}),o(`text`,{x:`30`,y:`210.5`,class:`dl-dg-lab is-sm`,"text-anchor":`middle`},`+`),o(`circle`,{cx:`30`,cy:`96`,r:`8`,class:`dl-dg-op`}),o(`text`,{x:`30`,y:`100.5`,class:`dl-dg-lab is-sm`,"text-anchor":`middle`},`+`),o(`path`,{d:`M30 256 H62`,class:`dl-dg-arrow`}),o(`rect`,{x:`62`,y:`244`,width:`50`,height:`24`,rx:`3`,class:`dl-dg-box`}),o(`text`,{x:`87`,y:`261`,class:`dl-dg-small`,"text-anchor":`middle`},`norm`),o(`path`,{d:`M112 256 H124`,class:`dl-dg-arrow`}),o(`polygon`,{points:`124,242 220,242 212,270 132,270`,class:`dl-dg-net`}),o(`text`,{x:`172`,y:`261`,class:`dl-dg-small`,"text-anchor":`middle`},`attention`),o(`path`,{d:`M172 242 V206 H42`,class:`dl-dg-arrow`,"marker-end":`url(#l6s5-a3)`}),o(`path`,{d:`M30 146 H62`,class:`dl-dg-arrow`}),o(`rect`,{x:`62`,y:`134`,width:`50`,height:`24`,rx:`3`,class:`dl-dg-box`}),o(`text`,{x:`87`,y:`151`,class:`dl-dg-small`,"text-anchor":`middle`},`norm`),o(`path`,{d:`M112 146 H124`,class:`dl-dg-arrow`}),o(`polygon`,{points:`124,132 220,132 212,160 132,160`,class:`dl-dg-net`}),o(`text`,{x:`172`,y:`151`,class:`dl-dg-small`,"text-anchor":`middle`},`feed-forward`),o(`path`,{d:`M172 132 V96 H42`,class:`dl-dg-arrow`,"marker-end":`url(#l6s5-a3)`})])],-1)]),e((f(),c(`div`,v,[...y[2]||=[o(`p`,null,[o(`code`,null,`x = x + sublayer(norm(x))`),u(`, twice. This is the pre-LN (layer norm first) block from "One wire moved, and deep stacks started training". `),o(`code`,null,`nn.Sequential`),u(` is the stack.`)],-1)]])),[[x]])]),_:1},16)}}};export{y as default};
|
lecture-06/assets/md-B5uG_D2l.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`slides.md__slidev_109`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),108))),{default:t(()=>[p[0]||=r(`h1`,null,`Where that leaves us`,-1),p[1]||=r(`p`,null,`One idea, applied twice per block.`,-1),r(`div`,f,[n(m,{kind:`block`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/assets/md-B815c0ps.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`slides.md__slidev_4`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),3))),{default:t(()=>[p[0]||=r(`h1`,null,`Delete the recurrence`,-1),r(`div`,f,[n(m,{kind:`cut`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/assets/md-BJ1CN0D8.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_113`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),112))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: Reading, and the lab`,-1),n(`div`,{class:`dl-describe`},[n(`ol`,null,[n(`li`,null,`Read "The Illustrated Transformer" first. It explains the architecture with pictures.`),n(`li`,null,`Read "Self-attention from scratch" next. It works through the same kind of example as the lecture.`),n(`li`,null,`Look at nanoGPT for a complete decoder-only model. Its attention code matches "Self-attention, written out".`)]),n(`p`,null,[a(`In the lab you build the block from "A block, and a stack". You train it on character-level text. Then you remove three parts, one at a time: the causal mask, the `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msqrt`,null,[n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`k`)])])]),n(`annotation`,{encoding:`application/x-tex`},`\\sqrt{d_k}`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1.04em`,"vertical-align":`-0.1828em`}}),n(`span`,{class:`mord sqrt`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.8572em`}},[n(`span`,{class:`svg-align`,style:{top:`-3em`}},[n(`span`,{class:`pstrut`,style:{height:`3em`}}),n(`span`,{class:`mord`,style:{"padding-left":`0.833em`}},[n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3361em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])]),n(`span`,{style:{top:`-2.8172em`}},[n(`span`,{class:`pstrut`,style:{height:`3em`}}),n(`span`,{class:`hide-tail`,style:{"min-width":`0.853em`,height:`1.08em`}},[n(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`1.08em`,viewBox:`0 0 400000 1080`,preserveAspectRatio:`xMinYMin slice`},[n(`path`,{d:`M95,702
|
| 2 |
+
c-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14
|
| 3 |
+
c0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54
|
| 4 |
+
c44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10
|
| 5 |
+
s173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429
|
| 6 |
+
c69,-144,104.5,-217.7,106.5,-221
|
| 7 |
+
l0 -0
|
| 8 |
+
c5.3,-9.3,12,-14,20,-14
|
| 9 |
+
H400000v40H845.2724
|
| 10 |
+
s-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7
|
| 11 |
+
c-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z
|
| 12 |
+
M834 80h400000v40h-400000z`})])])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.1828em`}},[n(`span`)])])])])])])]),a(` scaling and the positional encoding. Predict each result first, then report what each part was doing.`)])],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-BJLgyYot2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_30`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),29))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: Why a key and a value are different things`,-1),n(`div`,{class:`dl-describe`},[n(`ul`,null,[n(`li`,null,[n(`strong`,null,`Top`),a(`, a library book’s spine: `),n(`em`,null,`river`),a(`’s key, made with `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`W`),n(`mi`,null,`K`)])]),n(`annotation`,{encoding:`application/x-tex`},`W_K`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3283em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0715em`}},`K`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(`, is `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mo`,{stretchy:`false`},`[`),n(`mn`,null,`2`),n(`mo`,{separator:`true`},`,`),n(`mn`,null,`0`),n(`mo`,{stretchy:`false`},`]`)]),n(`annotation`,{encoding:`application/x-tex`},`[2, 0]`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),n(`span`,{class:`mopen`},`[`),n(`span`,{class:`mord`},`2`),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},`0`),n(`span`,{class:`mclose`},`]`)])])]),a(`. It makes the book easy to find, here by the word "water".`)]),n(`li`,null,[n(`strong`,null,`Bottom:`),a(),n(`em`,null,`river`),a(`’s value, made with `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`W`),n(`mi`,null,`V`)])]),n(`annotation`,{encoding:`application/x-tex`},`W_V`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3283em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.2222em`}},`V`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(`, is `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mo`,{stretchy:`false`},`[`),n(`mn`,null,`2.3`),n(`mo`,{separator:`true`},`,`),n(`mn`,null,`0.3`),n(`mo`,{stretchy:`false`},`]`)]),n(`annotation`,{encoding:`application/x-tex`},`[2.3, 0.3]`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),n(`span`,{class:`mopen`},`[`),n(`span`,{class:`mord`},`2.3`),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},`0.3`),n(`span`,{class:`mclose`},`]`)])])]),a(`. It is what is inside the book: what `),n(`em`,null,`river`),a(` hands over once `),n(`em`,null,`bank`),a(` has found it.`)]),n(`li`,null,`A good spine label is short and says what the book is about. That is not what you want to read. So the two jobs get two different matrices.`),n(`li`,null,[a(`Using one matrix for both (`),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`W`),n(`mi`,null,`K`)]),n(`mo`,null,`=`),n(`msub`,null,[n(`mi`,null,`W`),n(`mi`,null,`V`)])]),n(`annotation`,{encoding:`application/x-tex`},`W_K = W_V`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3283em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0715em`}},`K`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])]),n(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),n(`span`,{class:`mrel`},`=`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3283em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.2222em`}},`V`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(`) still works, but slightly worse.`)])])],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-BOLARYwC2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,yt as a,z as o}from"./modules/shiki-BFOesOFp.js";import{n as s,t as c}from"./slidev/context-DrkI8qiD.js";import{t as l}from"./default-sHmABwVd.js";var u={__name:`slides.md__slidev_83`,setup(u){let{$slidev:d,$nav:f,$clicksContext:p,$clicks:m,$page:h,$renderContext:g,$frontmatter:_}=s();return p.setup(),(s,u)=>(o(),i(l,a(e(r(c)(r(_),82))),{default:t(()=>[...u[0]||=[n(`h1`,null,`In words: Why decoder-only took over`,-1),n(`div`,{class:`dl-describe`},[n(`ul`,null,[n(`li`,null,`The picture shows ordinary text along the bottom: "the river bank was flooded".`),n(`li`,null,`Each arrow points from a word to the word after it. That next word is the target the model must predict.`),n(`li`,null,`So five words of text give four training examples. Nobody had to label anything.`),n(`li`,null,`Any text can be used this way, and every position in it is an example.`),n(`li`,null,`Many tasks can be written as "continue this text", so one objective covers them all.`),n(`li`,null,`Remember: decoder-only won because its training data costs nothing, not because the layer is better.`)])],-1)]]),_:1},16))}};export{u as default};
|
lecture-06/assets/md-BRWSRKtj2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,m as l,v as u,x as d,xt as f,yt as p,z as m}from"./modules/shiki-BFOesOFp.js";import{n as h,t as g}from"./slidev/context-DrkI8qiD.js";import{t as _}from"./default-sHmABwVd.js";import{t as v}from"./slidev/VClicks-CGAOkBil.js";var y={class:`grid grid-cols-2 gap-8 mt-2 dl-tight`},b={viewBox:`0 0 400 190`,class:`dl-diagram`,role:`img`,"aria-label":`Five words of text give four next-word training examples: each word predicts the word after it`},x=[`x`],S=[`x`],C=[`d`],w=[`x`],T=[`x`],E={class:`mt-3 dl-callout`},D={__name:`slides.md__slidev_82`,setup(D){let{$slidev:O,$nav:k,$clicksContext:A,$clicks:j,$page:M,$renderContext:N,$frontmatter:P}=h();return A.setup(),(h,D)=>{let O=v,k=a(`click`);return m(),u(_,p(t(s(g)(s(P),81))),{default:n(()=>[D[5]||=o(`h1`,null,`Why decoder-only took over`,-1),D[6]||=o(`p`,null,[d(`Not the most expressive design. The one with the most `),o(`strong`,null,`training data`),d(`.`)],-1),o(`div`,y,[o(`div`,null,[(m(),c(`svg`,b,[D[0]||=o(`defs`,null,[o(`marker`,{id:`l6s4-next`,markerWidth:`6`,markerHeight:`6`,refX:`5`,refY:`3`,orient:`auto`},[o(`path`,{class:`dl-dg-head`,d:`M0 0 L6 3 L0 6 z`})])],-1),D[1]||=o(`text`,{class:`dl-dg-small`,x:`6`,y:`24`},`predicts the next word`,-1),(m(),c(l,null,i([`river`,`bank`,`was`,`flooded`],(e,t)=>o(`g`,{key:`p`+t},[o(`rect`,{class:`dl-dg-fill`,x:6+t*78,y:`36`,width:`70`,height:`30`,rx:`4`},null,8,x),o(`text`,{class:`dl-dg-in`,x:41+t*78,y:`56`,"text-anchor":`middle`},f(e),9,S),o(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s4-next)`,d:`M${41+t*78} 128 V72`},null,8,C)])),64)),(m(),c(l,null,i([`the`,`river`,`bank`,`was`,`flooded`],(e,t)=>o(`g`,{key:`t`+t},[o(`rect`,{class:`dl-dg-box`,x:6+t*78,y:`130`,width:`70`,height:`30`,rx:`4`},null,8,w),o(`text`,{class:`dl-dg-lab is-sm`,x:41+t*78,y:`150`,"text-anchor":`middle`},f(e),9,T)])),64)),D[2]||=o(`text`,{class:`dl-dg-small is-good`,x:`6`,y:`182`},`5 words of text → 4 training examples, no labels`,-1)]))]),o(`div`,null,[r(O,null,{default:n(()=>[...D[3]||=[o(`ul`,null,[o(`li`,null,[d(`Next-token prediction needs `),o(`strong`,null,`no labels`),d(`: any text is training data`)]),o(`li`,null,[o(`strong`,null,`Every position`),d(` is an example, so one document gives thousands`)]),o(`li`,null,`Any task written as text becomes next-token prediction`),o(`li`,null,`One stack instead of two: simpler to scale and to serve`)],-1)]]),_:1})])]),e((m(),c(`div`,E,[...D[4]||=[o(`p`,null,`The design that won was the one whose training data was free.`,-1)]])),[[k]])]),_:1},16)}}};export{D as default};
|
lecture-06/assets/md-BTnUhx7N.css
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
.dl-stack[data-v-66f48009]{width:100%;height:100%}.dl-stack__spine[data-v-66f48009]{stroke:var(--dl-border);stroke-width:2px;color:var(--dl-border)}.dl-stack__box[data-v-66f48009]{fill:var(--dl-bg);stroke:var(--dl-border);stroke-width:1.3px;transition:fill .2s,stroke .2s}.dl-stack__box.is-seen[data-v-66f48009]{fill:var(--dl-surface);stroke:var(--dl-body)}.dl-stack__box.is-current[data-v-66f48009]{fill:var(--dl-accent-soft);stroke:var(--dl-accent);stroke-width:2.2px}.dl-stack__label[data-v-66f48009]{fill:var(--dl-muted);font-size:11.5px;font-weight:600;transition:fill .2s}.dl-stack__label.is-seen[data-v-66f48009],.dl-stack__label.is-current[data-v-66f48009]{fill:var(--dl-heading)}.dl-stack__sub[data-v-66f48009]{fill:var(--dl-muted);font-size:8.8px}.dl-stack__sub.is-current[data-v-66f48009]{fill:var(--dl-accent-strong)}.dl-stack__repeat[data-v-66f48009]{fill:none;stroke:var(--dl-border);stroke-width:1.2px;stroke-dasharray:5 4;transition:stroke .2s}.dl-stack__repeat.is-current[data-v-66f48009]{stroke:var(--dl-accent)}.dl-stack__repeatlabel[data-v-66f48009]{fill:var(--dl-muted);font-size:11px;font-weight:700}.dl-stack__repeatlabel.is-current[data-v-66f48009]{fill:var(--dl-accent)}.dl-stack__cross[data-v-66f48009]{fill:none;stroke:var(--dl-border);stroke-width:1.6px;color:var(--dl-border);transition:stroke .2s}.dl-stack__cross.is-seen[data-v-66f48009]{stroke:var(--dl-body);color:var(--dl-body)}.dl-stack__cross.is-current[data-v-66f48009]{stroke:var(--dl-accent);color:var(--dl-accent);stroke-width:2.2px}.dl-stack__crosslabel[data-v-66f48009]{fill:var(--dl-muted);opacity:0;font-size:9px;transition:opacity .2s}.dl-stack__crosslabel.is-seen[data-v-66f48009],.dl-stack__crosslabel.is-current[data-v-66f48009]{opacity:1}.dl-stack__crosslabel.is-current[data-v-66f48009]{fill:var(--dl-accent);font-weight:700}.dl-stack__add[data-v-66f48009]{fill:var(--dl-bg);stroke:var(--dl-border);stroke-width:1.4px}.dl-stack__add.is-seen[data-v-66f48009]{stroke:var(--dl-body)}.dl-stack__add.is-current[data-v-66f48009]{fill:var(--dl-accent-soft);stroke:var(--dl-accent);stroke-width:2.2px}.dl-stack__plus[data-v-66f48009]{fill:var(--dl-heading);font-size:12px;font-weight:700}.dl-stack__pe[data-v-66f48009]{fill:var(--dl-muted);font-size:9px}.dl-stack__pe.is-current[data-v-66f48009]{fill:var(--dl-accent);font-weight:700}.dl-stack__port[data-v-66f48009]{fill:var(--dl-muted);font-size:10px;transition:fill .2s}.dl-stack__port.is-current[data-v-66f48009]{fill:var(--dl-accent);font-weight:700}.dl-stack__side[data-v-66f48009]{fill:var(--dl-muted);letter-spacing:.09em;font-size:9.5px}
|
lecture-06/assets/md-BX32i5rK.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_61`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),60))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: Two wrappers that make it stackable`,-1),n(`div`,{class:`dl-describe`},[n(`p`,null,[a(`Read the diagram from the bottom. The input `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,{mathvariant:`bold`},`x`)]),n(`annotation`,{encoding:`application/x-tex`},`\\mathbf{x}`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.4444em`}}),n(`span`,{class:`mord mathbf`},`x`)])])]),a(` goes up two paths.`)]),n(`ul`,null,[n(`li`,null,`One path goes through the sub-layer: attention or the feed-forward network.`),n(`li`,null,[a(`The other path copies `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,{mathvariant:`bold`},`x`)]),n(`annotation`,{encoding:`application/x-tex`},`\\mathbf{x}`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.4444em`}}),n(`span`,{class:`mord mathbf`},`x`)])])]),a(` unchanged. At the + sign the two are added: `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,{mathvariant:`bold`},`x`),n(`mo`,null,`+`),n(`mi`,{mathvariant:`normal`},`SubLayer`),n(`mo`,null,``),n(`mo`,{stretchy:`false`},`(`),n(`mi`,{mathvariant:`bold`},`x`),n(`mo`,{stretchy:`false`},`)`)]),n(`annotation`,{encoding:`application/x-tex`},`\\mathbf{x} + \\operatorname{SubLayer}(\\mathbf{x})`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6667em`,"vertical-align":`-0.0833em`}}),n(`span`,{class:`mord mathbf`},`x`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),n(`span`,{class:`mbin`},`+`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),n(`span`,{class:`mop`},[n(`span`,{class:`mord mathrm`},`SubLayer`)]),n(`span`,{class:`mopen`},`(`),n(`span`,{class:`mord mathbf`},`x`),n(`span`,{class:`mclose`},`)`)])])]),a(`.`)]),n(`li`,null,[a(`The sub-layer only proposes a change. It never replaces `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,{mathvariant:`bold`},`x`)]),n(`annotation`,{encoding:`application/x-tex`},`\\mathbf{x}`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.4444em`}}),n(`span`,{class:`mord mathbf`},`x`)])])]),a(`.`)]),n(`li`,null,`The copy has no matrix on it, so gradients flow back along it easily, as in a ResNet.`),n(`li`,null,`LayerNorm then re-centres and re-scales each position’s vector on its own. Unlike BatchNorm, it never mixes examples or time steps.`)]),n(`p`,null,`Takeaway: the output has the same shape as the input, so identical blocks can be stacked.`)],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-BYKniqFA.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_43`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),42))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: The whole layer, in one line`,-1),n(`div`,{class:`dl-describe`},[n(`p`,null,[a(`The equation does all four steps for every word at once. `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`Q`)]),n(`annotation`,{encoding:`application/x-tex`},`Q`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8778em`,"vertical-align":`-0.1944em`}}),n(`span`,{class:`mord mathnormal`},`Q`)])])]),a(`, `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`K`)]),n(`annotation`,{encoding:`application/x-tex`},`K`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0715em`}},`K`)])])]),a(` and `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`V`)]),n(`annotation`,{encoding:`application/x-tex`},`V`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.2222em`}},`V`)])])]),a(` hold the queries, keys and values, one row per token. `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`T`)]),n(`annotation`,{encoding:`application/x-tex`},`T`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`)])])]),a(` counts the tokens.`)]),n(`ol`,null,[n(`li`,null,[n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`Q`)]),n(`annotation`,{encoding:`application/x-tex`},`Q`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8778em`,"vertical-align":`-0.1944em`}}),n(`span`,{class:`mord mathnormal`},`Q`)])])]),a(` (`),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`T`),n(`mo`,null,`×`),n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`k`)])]),n(`annotation`,{encoding:`application/x-tex`},`T \\times d_k`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.7667em`,"vertical-align":`-0.0833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),n(`span`,{class:`mbin`},`×`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3361em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(`) times `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msup`,null,[n(`mi`,null,`K`),n(`mi`,{mathvariant:`normal`},`⊤`)])]),n(`annotation`,{encoding:`application/x-tex`},`K^\\top`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8491em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0715em`}},`K`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.8491em`}},[n(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mtight`},`⊤`)])])])])])])])])])]),a(` (`),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`k`)]),n(`mo`,null,`×`),n(`mi`,null,`T`)]),n(`annotation`,{encoding:`application/x-tex`},`d_k \\times T`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3361em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])]),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),n(`span`,{class:`mbin`},`×`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`)])])]),a(`) gives a `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`T`),n(`mo`,null,`×`),n(`mi`,null,`T`)]),n(`annotation`,{encoding:`application/x-tex`},`T \\times T`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.7667em`,"vertical-align":`-0.0833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),n(`span`,{class:`mbin`},`×`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`)])])]),a(` grid of scores: every query against every key.`)]),n(`li`,null,[a(`Every score is divided by `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msqrt`,null,[n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`k`)])])]),n(`annotation`,{encoding:`application/x-tex`},`\\sqrt{d_k}`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1.04em`,"vertical-align":`-0.1828em`}}),n(`span`,{class:`mord sqrt`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.8572em`}},[n(`span`,{class:`svg-align`,style:{top:`-3em`}},[n(`span`,{class:`pstrut`,style:{height:`3em`}}),n(`span`,{class:`mord`,style:{"padding-left":`0.833em`}},[n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3361em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])]),n(`span`,{style:{top:`-2.8172em`}},[n(`span`,{class:`pstrut`,style:{height:`3em`}}),n(`span`,{class:`hide-tail`,style:{"min-width":`0.853em`,height:`1.08em`}},[n(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`1.08em`,viewBox:`0 0 400000 1080`,preserveAspectRatio:`xMinYMin slice`},[n(`path`,{d:`M95,702
|
| 2 |
+
c-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14
|
| 3 |
+
c0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54
|
| 4 |
+
c44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10
|
| 5 |
+
s173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429
|
| 6 |
+
c69,-144,104.5,-217.7,106.5,-221
|
| 7 |
+
l0 -0
|
| 8 |
+
c5.3,-9.3,12,-14,20,-14
|
| 9 |
+
H400000v40H845.2724
|
| 10 |
+
s-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7
|
| 11 |
+
c-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z
|
| 12 |
+
M834 80h400000v40h-400000z`})])])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.1828em`}},[n(`span`)])])])])])])]),a(`.`)]),n(`li`,null,`Softmax runs along each row, so every row of weights adds up to 1.`),n(`li`,null,[a(`The weights (`),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`T`),n(`mo`,null,`×`),n(`mi`,null,`T`)]),n(`annotation`,{encoding:`application/x-tex`},`T \\times T`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.7667em`,"vertical-align":`-0.0833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),n(`span`,{class:`mbin`},`×`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`)])])]),a(`) times `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`V`)]),n(`annotation`,{encoding:`application/x-tex`},`V`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.2222em`}},`V`)])])]),a(` (`),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`T`),n(`mo`,null,`×`),n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`v`)])]),n(`annotation`,{encoding:`application/x-tex`},`T \\times d_v`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.7667em`,"vertical-align":`-0.0833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),n(`span`,{class:`mbin`},`×`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.1514em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0359em`}},`v`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(`) give the output: one new vector per token, `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`T`),n(`mo`,null,`×`),n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`v`)])]),n(`annotation`,{encoding:`application/x-tex`},`T \\times d_v`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.7667em`,"vertical-align":`-0.0833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),n(`span`,{class:`mbin`},`×`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.1514em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0359em`}},`v`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(`.`)])]),n(`p`,null,[a(`For "the river bank", `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`T`),n(`mo`,null,`=`),n(`mn`,null,`3`)]),n(`annotation`,{encoding:`application/x-tex`},`T = 3`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),n(`span`,{class:`mrel`},`=`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),n(`span`,{class:`mord`},`3`)])])]),a(`, so the grid of scores is `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mn`,null,`3`),n(`mo`,null,`×`),n(`mn`,null,`3`)]),n(`annotation`,{encoding:`application/x-tex`},`3 \\times 3`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.7278em`,"vertical-align":`-0.0833em`}}),n(`span`,{class:`mord`},`3`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),n(`span`,{class:`mbin`},`×`),n(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),n(`span`,{class:`mord`},`3`)])])]),a(`.`)])],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-Bccft7aT.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_6`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),5))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: Two walls in reading one word at a time`,-1),n(`div`,{class:`dl-describe`},[n(`p`,null,`The picture shows a recurrent encoder–decoder and its two walls.`),n(`ul`,null,[n(`li`,null,[n(`strong`,null,`Left, wall one.`),a(` The encoder read the source into states `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,{mathvariant:`bold`},`h`),n(`mn`,null,`1`)]),n(`mo`,{separator:`true`},`,`),n(`msub`,null,[n(`mi`,{mathvariant:`bold`},`h`),n(`mn`,null,`2`)]),n(`mo`,{separator:`true`},`,`),n(`msub`,null,[n(`mi`,{mathvariant:`bold`},`h`),n(`mn`,null,`3`)])]),n(`annotation`,{encoding:`application/x-tex`},`\\mathbf{h}_1, \\mathbf{h}_2, \\mathbf{h}_3`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8889em`,"vertical-align":`-0.1944em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathbf`},`h`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3011em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mtight`},`1`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])]),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathbf`},`h`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3011em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mtight`},`2`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])]),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathbf`},`h`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3011em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mtight`},`3`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(`. The plain decoder saw only one fixed vector, `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`c`)]),n(`annotation`,{encoding:`application/x-tex`},`c`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.4306em`}}),n(`span`,{class:`mord mathnormal`},`c`)])])]),a(`. Everything had to squeeze through that one box.`)]),n(`li`,null,[n(`strong`,null,`Attention fixed wall one.`),a(` The curved lines let each decoder step score every encoder state and take a weighted sum. That gives a fresh context at every step.`)]),n(`li`,null,[n(`strong`,null,`Right, wall two.`),a(` Each state `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,{mathvariant:`bold`},`h`),n(`mi`,null,`t`)])]),n(`annotation`,{encoding:`application/x-tex`},`\\mathbf{h}_t`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathbf`},`h`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.2806em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`},`t`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(` needs the one before it. So 200 tokens means 200 steps in a row.`)]),n(`li`,null,`Attention sat on top of this chain, so wall two stayed.`)]),n(`p`,null,`This lecture removes the chain completely.`)],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-Be8qHbLU.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_108`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),107))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: Four bugs, and what each one looks like`,-1),n(`div`,{class:`dl-describe`},[n(`p`,null,`The slide lists four common bugs in transformer code, with their symptoms.`),n(`ol`,null,[n(`li`,null,[n(`strong`,null,`No causal mask:`),a(` the loss drops almost to zero, but generation is gibberish. The model copies the next token from its input.`)]),n(`li`,null,[n(`strong`,null,[n(`code`,null,`batch_first`),a(` left out:`)]),a(` PyTorch reads the batch as time. There is no error.`)]),n(`li`,null,[n(`strong`,null,`Padding mask forgotten:`),a(` attention puts weight on pad tokens. There is no error.`)]),n(`li`,null,[n(`strong`,null,[n(`code`,null,`norm_first`),a(` left as the default:`)]),a(` the old post-LN wiring. A deep stack diverges early unless you add a warmup.`)])]),n(`p`,null,`Three of the four bugs give no error message. A loss that looks too good means: check the mask first.`)],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-BekE5qL_.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_10`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),9))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: which wall attention solves`,-1),n(`div`,{class:`dl-describe`},[n(`ul`,null,[n(`li`,null,`The poll asks which of the two walls attention removes when it is added on top of a recurrent model.`),n(`li`,null,[n(`strong`,null,`The answer is the bottleneck only.`),a(` Attention replaces the single fixed vector with a fresh weighted sum at every output step.`)]),n(`li`,null,`It does not give parallelism. In the sketch, attention (the circle) still sits on top of the recurrent chain. Each state still waits for the one before it.`),n(`li`,null,`"Both" is the common wrong answer. Attention and parallelism are often taught together, but they are two separate steps.`)]),n(`p`,null,`Today’s point: attention does not need the recurrence. Remove the chain, and the second wall falls too.`)],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-BexnRWui.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_91`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),90))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: What generation actually costs`,-1),n(`div`,{class:`dl-describe`},[n(`ul`,null,[n(`li`,null,`A model writes one token at a time. Each new token attends to every earlier token.`),n(`li`,null,[a(`The keys and values of earlier tokens do not change. So the model keeps them in a `),n(`strong`,null,`KV (key–value) cache`),a(`.`)]),n(`li`,null,`In the picture, grey cells are cached and the accent cell is new. Each step adds one column and needs only one new query.`),n(`li`,null,[a(`The cache costs memory: layers × heads × tokens × `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`d`),n(`mi`,null,`k`)])]),n(`annotation`,{encoding:`application/x-tex`},`d_k`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`},`d`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3361em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(` × 2, for every sequence.`)]),n(`li`,null,`With grouped-query attention, four query heads share one key/value head, so the cache is four times smaller.`),n(`li`,null,`Remember: at long context, the cache, not the weights, fills the GPU.`)])],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-BgsK6PGZ2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_8`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),7))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: The move`,-1),n(`div`,{class:`dl-describe`},[n(`ul`,null,[n(`li`,null,[n(`strong`,null,`Left: the recurrent design.`),a(` A recurrent chain does the work. Attention sits on top as a passenger and reads from every state.`)]),n(`li`,null,[n(`strong`,null,`Right: the 2017 design`),a(`, from `),n(`em`,null,`Attention Is All You Need`),a(` (Vaswani et al.). The chain is gone. Each word connects to every other word through attention only.`)]),n(`li`,null,`With no chain, no position waits for another. The whole sentence is processed at once.`),n(`li`,null,`The authors removed the recurrent neural network (RNN) as an experiment. Translation quality went up, not down.`)]),n(`p`,null,`Keep the weighted sum and delete what it was attached to. Every later model builds on that idea.`)],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-BjO-yXuT.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,H as t,Q as n,S as r,_ as i,_t as a,b as o,v as s,w as c,x as l,xt as u,y as d,yt as f,z as p}from"./modules/shiki-BFOesOFp.js";import{t as m}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as h,t as g}from"./slidev/context-DrkI8qiD.js";import{t as _}from"./TransformerGlyph-D9TehqNe.js";var v={class:`slidev-layout dl-title`},y={class:`dl-title__eyebrow`},b={key:0,class:`dl-title__code`},x={key:1,class:`dl-title__lecture`},S={class:`dl-title__body`},C={class:`dl-title__meta`},w={key:0,class:`dl-title__author`},T=[`href`],E={key:2,class:`dl-title__date`},D=m(c({__name:`title`,props:{courseCode:{},lecture:{},author:{},email:{},date:{}},setup(e){let{$slidev:n,$nav:r,$clicksContext:a,$clicks:s,$page:c,$renderContext:l,$frontmatter:f}=h(),m=e;return(e,n)=>(p(),o(`div`,v,[i(`div`,y,[m.courseCode?(p(),o(`span`,b,u(m.courseCode),1)):d(`v-if`,!0),m.lecture?(p(),o(`span`,x,`Lecture `+u(m.lecture),1)):d(`v-if`,!0)]),i(`div`,S,[t(e.$slots,`default`,{},void 0,!0)]),n[0]||=i(`div`,{class:`dl-rule dl-title__rule`},null,-1),i(`div`,C,[m.author?(p(),o(`div`,w,u(m.author),1)):d(`v-if`,!0),m.email?(p(),o(`a`,{key:1,href:`mailto:${m.email}`,class:`dl-title__email`},u(m.email),9,T)):d(`v-if`,!0),m.date?(p(),o(`div`,E,u(m.date),1)):d(`v-if`,!0)])]))}}),[[`__scopeId`,`data-v-dc59a472`]]),O={class:`mt-6 flex items-center gap-6`},k={__name:`slides.md__slidev_1`,setup(t){let{$slidev:o,$nav:c,$clicksContext:u,$clicks:d,$page:m,$renderContext:v,$frontmatter:y}=h();return u.setup(),(t,o)=>{let c=_;return p(),s(D,f(e(a(g)(a(y),0))),{default:n(()=>[o[3]||=i(`h1`,null,`Transformers`,-1),o[4]||=i(`p`,null,`A network that reads a sentence one word at a time is slow to train and forgets the start. Let every word look at every other word at once instead. Every large language model since 2017 is built on that idea.`,-1),i(`div`,O,[r(c,{kind:`cut`,size:110},{default:n(()=>[...o[0]||=[l(`read in order`,-1)]]),_:1}),o[2]||=i(`span`,{class:`text-3xl`,style:{color:`var(--dl-accent)`}},`→`,-1),r(c,{kind:`block`,size:110},{default:n(()=>[...o[1]||=[l(`look everywhere`,-1)]]),_:1})])]),_:1},16)}}};export{k as default};
|
lecture-06/assets/md-BkNuaaSP.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`slides.md__slidev_20`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),19))),{default:t(()=>[p[0]||=r(`h1`,null,`Query, key, value`,-1),p[1]||=r(`p`,null,`A dictionary lookup, made soft.`,-1),r(`div`,f,[n(m,{kind:`qkv`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/assets/md-BmMhfMUg.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,M as n,Q as r,S as i,W as a,_ as o,_t as s,b as c,v as l,x as u,yt as d,z as f}from"./modules/shiki-BFOesOFp.js";import{n as p,t as m}from"./slidev/context-DrkI8qiD.js";import{t as h}from"./default-sHmABwVd.js";import{t as g}from"./slidev/CodeBlockWrapper-C_5MjYK7.js";var _={class:`dl-diagram mt-1`,viewBox:`0 0 820 64`,style:{"max-width":`46rem`},role:`img`,"aria-label":`The shape of x through the layer: (B, T, d), then three tensors of (B, T, d) after qkv and split, then (B, h, T, d_k) after view and transpose, then (B, h, T, d_k) after attention, then (B, T, d) after merging the heads and the output projection.`},v={class:`mt-1 dl-secondary`},y={__name:`slides.md__slidev_99`,setup(y){let{$slidev:b,$nav:x,$clicksContext:S,$clicks:C,$page:w,$renderContext:T,$frontmatter:E}=p();return S.setup(),(p,y)=>{let b=g,x=a(`click`);return f(),l(h,d(t(s(m)(s(E),98))),{default:r(()=>[y[3]||=o(`h1`,null,`Self-attention, written out`,-1),i(b,n({lines:!0},{title:``,ranges:[`all`,`5-6`,`10-11`,`12`,`13-14`,`all`]}),{default:r(()=>[...y[0]||=[o(`pre`,{class:`shiki shiki-themes vitesse-dark vitesse-light slidev-code`,style:{"--shiki-dark":`#dbd7caee`,"--shiki-light":`#393a34`,"--shiki-dark-bg":`#121212`,"--shiki-light-bg":`#ffffff`}},[o(`code`,{class:`language-python`},[o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#CB7676`,"--shiki-light":`#AB5959`}},`class`),o(`span`,{style:{"--shiki-dark":`#5DA994`,"--shiki-light":`#2E8F82`}},` SelfAttention`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#80A665`,"--shiki-light":`#59873A`}},`nn`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#80A665`,"--shiki-light":`#59873A`}},`Module`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`):`)]),u(`
|
| 2 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#CB7676`,"--shiki-light":`#AB5959`}},` def`),o(`span`,{style:{"--shiki-dark":`#B8A965`,"--shiki-light":`#998418`}},` __init__`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` d_model`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` n_heads`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`):`)]),u(`
|
| 3 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#B8A965`,"--shiki-light":`#998418`}},` super`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`().`),o(`span`,{style:{"--shiki-dark":`#B8A965`,"--shiki-light":`#998418`}},`__init__`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`()`)]),u(`
|
| 4 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`h`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`dk `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` n_heads`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` d_model `),o(`span`,{style:{"--shiki-dark":`#CB7676`,"--shiki-light":`#AB5959`}},`//`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` n_heads`)]),u(`
|
| 5 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`qkv `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` nn`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`Linear`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`d_model`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},` 3`),o(`span`,{style:{"--shiki-dark":`#CB7676`,"--shiki-light":`#AB5959`}},` *`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` d_model`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#BD976A`,"--shiki-light":`#B07D48`}},` bias`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`False`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`),o(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # W_Q, W_K, W_V`)]),u(`
|
| 6 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`proj `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` nn`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`Linear`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`d_model`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` d_model`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#BD976A`,"--shiki-light":`#B07D48`}},` bias`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`False`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`),o(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # W_O`)]),u(`
|
| 7 |
+
`),o(`span`,{class:`line`}),u(`
|
| 8 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#CB7676`,"--shiki-light":`#AB5959`}},` def`),o(`span`,{style:{"--shiki-dark":`#80A665`,"--shiki-light":`#59873A`}},` forward`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` x`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`):`),o(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # (B, T, d)`)]),u(`
|
| 9 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` B`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` T`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` d `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` x`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`shape`)]),u(`
|
| 10 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` q`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` k`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` v `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`qkv`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`x`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`).`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`split`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`d`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#BD976A`,"--shiki-light":`#B07D48`}},` dim`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#CB7676`,"--shiki-light":`#AB5959`}},`-`),o(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},`1`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`),o(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # three of (B, T, d)`)]),u(`
|
| 11 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` q`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` k`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` v `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},` (`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`t`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`view`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`B`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` T`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`h`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`dk`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`).`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`transpose`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},`1`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},` 2`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`),o(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},` for`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` t `),o(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`in`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},` (`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`q`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` k`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` v`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`))`)]),u(`
|
| 12 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` y `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` F`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`scaled_dot_product_attention`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`q`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` k`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` v`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#BD976A`,"--shiki-light":`#B07D48`}},` is_causal`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`True`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`)]),u(`
|
| 13 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` y `),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` y`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`transpose`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},`1`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},` 2`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`).`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`reshape`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`B`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` T`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` d`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`),o(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # heads back together`)]),u(`
|
| 14 |
+
`),o(`span`,{class:`line`},[o(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},` return`),o(`span`,{style:{"--shiki-dark":`#C99076`,"--shiki-light":`#A65E2B`}},` self`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`proj`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),o(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`y`),o(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`),o(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # (B, T, d)`)])])],-1)]]),_:1},16),(f(),c(`svg`,_,[...y[1]||=[o(`defs`,null,[o(`marker`,{id:`l6s5-a2`,markerWidth:`8`,markerHeight:`8`,refX:`7`,refY:`4`,orient:`auto`},[o(`path`,{d:`M0 0 L8 4 L0 8 z`,class:`dl-dg-head`})])],-1),o(`rect`,{x:`2`,y:`4`,width:`120`,height:`30`,rx:`3`,class:`dl-dg-box`},null,-1),o(`text`,{x:`62`,y:`24`,class:`dl-dg-in`,"text-anchor":`middle`},`x (B, T, d)`,-1),o(`path`,{d:`M124 19 H168`,class:`dl-dg-arrow`,"marker-end":`url(#l6s5-a2)`},null,-1),o(`text`,{x:`146`,y:`54`,class:`dl-dg-small`,"text-anchor":`middle`},`qkv, split`,-1),o(`rect`,{x:`172`,y:`4`,width:`120`,height:`30`,rx:`3`,class:`dl-dg-box`},null,-1),o(`text`,{x:`232`,y:`24`,class:`dl-dg-in`,"text-anchor":`middle`},`3 × (B, T, d)`,-1),o(`path`,{d:`M294 19 H338`,class:`dl-dg-arrow`,"marker-end":`url(#l6s5-a2)`},null,-1),o(`text`,{x:`316`,y:`54`,class:`dl-dg-small`,"text-anchor":`middle`},`view, transpose`,-1),o(`rect`,{x:`342`,y:`4`,width:`120`,height:`30`,rx:`3`,class:`dl-dg-box is-accent`},null,-1),o(`text`,{x:`402`,y:`24`,class:`dl-dg-in`,"text-anchor":`middle`},`(B, h, T, d_k)`,-1),o(`path`,{d:`M464 19 H508`,class:`dl-dg-arrow`,"marker-end":`url(#l6s5-a2)`},null,-1),o(`text`,{x:`486`,y:`54`,class:`dl-dg-small`,"text-anchor":`middle`},`attention`,-1),o(`rect`,{x:`512`,y:`4`,width:`120`,height:`30`,rx:`3`,class:`dl-dg-box is-accent`},null,-1),o(`text`,{x:`572`,y:`24`,class:`dl-dg-in`,"text-anchor":`middle`},`(B, h, T, d_k)`,-1),o(`path`,{d:`M634 19 H678`,class:`dl-dg-arrow`,"marker-end":`url(#l6s5-a2)`},null,-1),o(`text`,{x:`656`,y:`54`,class:`dl-dg-small`,"text-anchor":`middle`},`merge, W_O`,-1),o(`rect`,{x:`682`,y:`4`,width:`120`,height:`30`,rx:`3`,class:`dl-dg-box`},null,-1),o(`text`,{x:`742`,y:`24`,class:`dl-dg-in`,"text-anchor":`middle`},`(B, T, d)`,-1)]])),e((f(),c(`div`,v,[...y[2]||=[o(`p`,null,[u(`The heads are a `),o(`strong`,null,`reshape`),u(`, not a loop. Eight heads cost what one would.`)],-1)]])),[[x]])]),_:1},16)}}};export{y as default};
|
lecture-06/assets/md-BpUXPIXm.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_85`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),84))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: One wire moved, and deep stacks started training`,-1),n(`div`,{class:`dl-describe`},[n(`ul`,null,[n(`li`,null,[a(`The widget draws one transformer block. The `),n(`strong`,null,`Rewire`),a(` button switches between the 2017 wiring and the modern one.`)]),n(`li`,null,`In the 2017 wiring (post-LN), the LayerNorm comes after each add. It sits on the residual path itself.`),n(`li`,null,`In the modern wiring (pre-LN), the LayerNorm comes before each sub-layer. It works on a copy, and the residual path is untouched.`),n(`li`,null,`So in a pre-LN stack, the path from bottom to top is only additions. The gradient can travel down it without being rescaled.`),n(`li`,null,`Remember: a deep post-LN stack needs a learning-rate warmup to train. A pre-LN stack does not.`)])],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-BqhSluwG.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_104`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),103))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: Or use the one in the box`,-1),n(`div`,{class:`dl-describe`},[n(`p`,null,[a(`The slide shows PyTorch’s ready-made encoder layer, `),n(`code`,null,`nn.TransformerEncoderLayer`),a(`.`)]),n(`ul`,null,[n(`li`,null,[n(`strong`,null,`Reading the sketch:`),a(` a `),n(`code`,null,`(B, T, 512)`),a(` input goes through six stacked layers. The output has the same shape.`)]),n(`li`,null,[n(`strong`,null,`Two settings to change:`),a(),n(`code`,null,`batch_first=True`),a(` puts the batch first, as `),n(`code`,null,`(B, T, d)`),a(`. `),n(`code`,null,`norm_first=True`),a(` gives the pre-LN wiring.`)]),n(`li`,null,[n(`strong`,null,`Why it matters:`),a(` both defaults are wrong for a modern model, and neither one gives an error message.`)]),n(`li`,null,[n(`strong`,null,`The check:`),a(` the printed count is 18 914 304 parameters. That is the 18.9 M from "What a block costs".`)])]),n(`p`,null,`Always set both keyword arguments when you use this class.`)],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-BsLhH0zv.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_45`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),44))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: Where self-attention’s keys come from`,-1),n(`div`,{class:`dl-describe`},[n(`ul`,null,[n(`li`,null,[a(`In self-attention over one sentence, the keys come from `),n(`strong`,null,`the same sentence`),a(`. Each key is the previous layer’s output for one token, multiplied by `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`msub`,null,[n(`mi`,null,`W`),n(`mi`,null,`K`)])]),n(`annotation`,{encoding:`application/x-tex`},`W_K`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),n(`span`,{class:`mord`},[n(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),n(`span`,{class:`msupsub`},[n(`span`,{class:`vlist-t vlist-t2`},[n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.3283em`}},[n(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[n(`span`,{class:`pstrut`,style:{height:`2.7em`}}),n(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[n(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0715em`}},`K`)])])]),n(`span`,{class:`vlist-s`},``)]),n(`span`,{class:`vlist-r`},[n(`span`,{class:`vlist`,style:{height:`0.15em`}},[n(`span`)])])])])])])])]),a(`.`)]),n(`li`,null,[a(`In the picture, `),n(`em`,null,`bank`),a(`’s query looks at three keys. After the answer, arrows show that each key is made from one of the three words below it. The queries and values come from the same three words.`)]),n(`li`,null,`There is no stored memory from training. Nothing is kept from one sentence to the next.`),n(`li`,null,[a(`That is what `),n(`em`,null,`self`),a(` means: one sequence gives the queries, the keys and the values. Cross-attention, in section 04, takes them from two different places.`)])])],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-BscC0r3L.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{E as e,M as t,Q as n,S as r,V as i,_ as a,_t as o,b as s,m as c,v as l,x as u,yt as d,z as f}from"./modules/shiki-BFOesOFp.js";import{n as p,t as m}from"./slidev/context-DrkI8qiD.js";import{t as h}from"./default-sHmABwVd.js";import{t as g}from"./slidev/CodeBlockWrapper-C_5MjYK7.js";import{t as _}from"./slidev/VClicks-CGAOkBil.js";var v={class:`grid grid-cols-[1fr_1fr] gap-6 mt-2 items-center`},y={class:`dl-diagram`,viewBox:`0 0 420 120`,role:`img`,"aria-label":`An input of shape (B, T, 512) goes into a stack of six encoder layers and comes out with the same shape. The stack has 18 914 304 parameters.`},b=[`y`],x={class:`dl-tight`},S={__name:`slides.md__slidev_103`,setup(S){let{$slidev:C,$nav:w,$clicksContext:T,$clicks:E,$page:D,$renderContext:O,$frontmatter:k}=p();return T.setup(),(p,S)=>{let C=g,w=_;return f(),l(h,d(e(o(m)(o(k),102))),{default:n(()=>[S[12]||=a(`h1`,null,`Or use the one in the box`,-1),r(C,t({lines:!0},{title:``,ranges:[`all`,`1-3`,`5`,`7-8`,`all`]}),{default:n(()=>[...S[0]||=[a(`pre`,{class:`shiki shiki-themes vitesse-dark vitesse-light slidev-code`,style:{"--shiki-dark":`#dbd7caee`,"--shiki-light":`#393a34`,"--shiki-dark-bg":`#121212`,"--shiki-light-bg":`#ffffff`}},[a(`code`,{class:`language-python`},[a(`span`,{class:`line`},[a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`layer `),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` nn`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`TransformerEncoderLayer`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),a(`span`,{style:{"--shiki-dark":`#BD976A`,"--shiki-light":`#B07D48`}},`d_model`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),a(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},`512`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),a(`span`,{style:{"--shiki-dark":`#BD976A`,"--shiki-light":`#B07D48`}},` nhead`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),a(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},`8`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),a(`span`,{style:{"--shiki-dark":`#BD976A`,"--shiki-light":`#B07D48`}},` dim_feedforward`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),a(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},`2048`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`)]),u(`
|
| 2 |
+
`),a(`span`,{class:`line`},[a(`span`,{style:{"--shiki-dark":`#BD976A`,"--shiki-light":`#B07D48`}},` batch_first`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),a(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`True`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),a(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # (B, T, d), not (T, B, d)`)]),u(`
|
| 3 |
+
`),a(`span`,{class:`line`},[a(`span`,{style:{"--shiki-dark":`#BD976A`,"--shiki-light":`#B07D48`}},` norm_first`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),a(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`True`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`),a(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # pre-LN, not the default`)]),u(`
|
| 4 |
+
`),a(`span`,{class:`line`}),u(`
|
| 5 |
+
`),a(`span`,{class:`line`},[a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`encoder `),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` nn`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`TransformerEncoder`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`layer`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),a(`span`,{style:{"--shiki-dark":`#BD976A`,"--shiki-light":`#B07D48`}},` num_layers`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),a(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},`6`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`)]),u(`
|
| 6 |
+
`),a(`span`,{class:`line`}),u(`
|
| 7 |
+
`),a(`span`,{class:`line`},[a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`out `),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` encoder`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`src`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),a(`span`,{style:{"--shiki-dark":`#BD976A`,"--shiki-light":`#B07D48`}},` src_key_padding_mask`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`pad_mask`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`),a(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # (B, T, 512) → (B, T, 512)`)]),u(`
|
| 8 |
+
`),a(`span`,{class:`line`},[a(`span`,{style:{"--shiki-dark":`#B8A965`,"--shiki-light":`#998418`}},`print`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),a(`span`,{style:{"--shiki-dark":`#B8A965`,"--shiki-light":`#998418`}},`sum`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`p`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`numel`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`()`),a(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},` for`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` p `),a(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`in`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` encoder`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`parameters`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`()))`),a(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # 18 914 304`)])])],-1)]]),_:1},16),a(`div`,v,[a(`div`,null,[(f(),s(`svg`,y,[S[1]||=a(`defs`,null,[a(`marker`,{id:`l6s5-a4`,markerWidth:`8`,markerHeight:`8`,refX:`7`,refY:`4`,orient:`auto`},[a(`path`,{d:`M0 0 L8 4 L0 8 z`,class:`dl-dg-head`})])],-1),S[2]||=a(`rect`,{x:`2`,y:`36`,width:`96`,height:`28`,rx:`3`,class:`dl-dg-box`},null,-1),S[3]||=a(`text`,{x:`50`,y:`55`,class:`dl-dg-in`,"text-anchor":`middle`},`(B, T, 512)`,-1),S[4]||=a(`path`,{d:`M100 50 H136`,class:`dl-dg-arrow`,"marker-end":`url(#l6s5-a4)`},null,-1),(f(),s(c,null,i(6,e=>a(`rect`,{key:e,x:`142`,y:4+(e-1)*15,width:`134`,height:`12`,rx:`2`,class:`dl-dg-fill`},null,8,b)),64)),S[5]||=a(`text`,{x:`209`,y:`112`,class:`dl-dg-small`,"text-anchor":`middle`},`6 encoder layers`,-1),S[6]||=a(`path`,{d:`M280 50 H316`,class:`dl-dg-arrow`,"marker-end":`url(#l6s5-a4)`},null,-1),S[7]||=a(`rect`,{x:`320`,y:`36`,width:`96`,height:`28`,rx:`3`,class:`dl-dg-box is-accent`},null,-1),S[8]||=a(`text`,{x:`368`,y:`55`,class:`dl-dg-in`,"text-anchor":`middle`},`(B, T, 512)`,-1),S[9]||=a(`text`,{x:`368`,y:`84`,class:`dl-dg-small`,"text-anchor":`middle`},`18 914 304`,-1),S[10]||=a(`text`,{x:`368`,y:`98`,class:`dl-dg-small`,"text-anchor":`middle`},`parameters`,-1)]))]),a(`div`,x,[r(w,null,{default:n(()=>[...S[11]||=[a(`ul`,null,[a(`li`,null,[a(`code`,null,`batch_first=True`),u(`: without it, the batch axis is read as time`)]),a(`li`,null,[a(`code`,null,`norm_first=True`),u(`: the default is the 2017 wiring, which needs warmup`)]),a(`li`,null,[u(`The count matches "What a block costs": `),a(`strong`,null,`18.9 M`)])],-1)]]),_:1})])])]),_:1},16)}}};export{S as default};
|
lecture-06/assets/md-BtxXgyX-.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,m as l,v as u,x as d,yt as f,z as p}from"./modules/shiki-BFOesOFp.js";import{n as m,t as h}from"./slidev/context-DrkI8qiD.js";import{t as g}from"./default-sHmABwVd.js";import{t as _}from"./slidev/KaTexBlockWrapper-BKHEVd16.js";var v={class:`grid grid-cols-2 gap-8 mt-2 items-start`},y={viewBox:`0 0 300 250`,class:`dl-diagram`,role:`img`,"aria-label":`An input 512 wide is projected and cut into 8 heads of 64; the 8 outputs are concatenated back to 512 and mixed by W_O into a 512-wide output`,style:{"max-height":`16rem`}},b=[`x`],x=[`x`],S={class:`dl-math-sm`},C={class:`mt-3 dl-callout`},w={class:`mt-3 dl-secondary`},T={__name:`slides.md__slidev_51`,setup(T){let{$slidev:E,$nav:D,$clicksContext:O,$clicks:k,$page:A,$renderContext:j,$frontmatter:M}=m();return O.setup(),(m,T)=>{let E=_,D=a(`click`);return p(),u(g,f(t(s(h)(s(M),50))),{default:n(()=>[T[18]||=o(`h1`,null,`Heads are free`,-1),T[19]||=o(`p`,null,[d(`Eight heads do `),o(`strong`,null,`not`),d(` cost eight times one head. Each head is `),o(`strong`,null,`narrower`),d(`.`)],-1),o(`div`,v,[o(`div`,null,[(p(),c(`svg`,y,[T[0]||=o(`defs`,null,[o(`marker`,{id:`l6s2-heads-arrow`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[o(`path`,{class:`dl-dg-head`,d:`M0 0 L7 3.5 L0 7 z`})])],-1),T[1]||=o(`rect`,{class:`dl-dg-box`,x:`22`,y:`214`,width:`256`,height:`24`,rx:`3`},null,-1),T[2]||=o(`text`,{class:`dl-dg-small`,x:`150`,y:`230`,"text-anchor":`middle`},`input: 512`,-1),T[3]||=o(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s2-heads-arrow)`,d:`M150 212 V190`},null,-1),(p(),c(l,null,i(8,e=>o(`g`,{key:e},[o(`rect`,{class:`dl-dg-box is-accent`,x:22+(e-1)*32.3,y:`156`,width:`30`,height:`30`,rx:`3`},null,8,b),o(`text`,{class:`dl-dg-small`,x:37+(e-1)*32.3,y:`175`,"text-anchor":`middle`},`64`,8,x)])),64)),T[4]||=o(`text`,{class:`dl-dg-small`,x:`150`,y:`148`,"text-anchor":`middle`},`8 heads × 64`,-1),T[5]||=o(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s2-heads-arrow)`,d:`M150 136 V118`},null,-1),T[6]||=o(`rect`,{class:`dl-dg-box`,x:`22`,y:`90`,width:`256`,height:`24`,rx:`3`},null,-1),T[7]||=o(`text`,{class:`dl-dg-small`,x:`150`,y:`106`,"text-anchor":`middle`},`8 × 64 = 512`,-1),T[8]||=o(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s2-heads-arrow)`,d:`M150 88 V70`},null,-1),T[9]||=o(`polygon`,{class:`dl-dg-net`,points:`110,66 190,66 200,42 100,42`},null,-1),T[10]||=o(`text`,{class:`dl-dg-in`,x:`150`,y:`59`,"text-anchor":`middle`},`W_O`,-1),T[11]||=o(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s2-heads-arrow)`,d:`M150 40 V30`},null,-1),T[12]||=o(`rect`,{class:`dl-dg-box`,x:`22`,y:`4`,width:`256`,height:`24`,rx:`3`},null,-1),T[13]||=o(`text`,{class:`dl-dg-small`,x:`150`,y:`20`,"text-anchor":`middle`},`output: 512`,-1)]))]),o(`div`,null,[e((p(),c(`div`,null,[o(`div`,S,[r(E,{ranges:[]},{default:n(()=>[...T[14]||=[o(`p`,null,[o(`span`,{class:`katex-display`},[o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`,display:`block`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,{mathvariant:`normal`},`MHA`),o(`mo`,null,``),o(`mo`,{stretchy:`false`},`(`),o(`mi`,null,`X`),o(`mo`,{stretchy:`false`},`)`),o(`mo`,null,`=`),o(`mrow`,null,[o(`mo`,{fence:`true`},`[`),o(`msub`,null,[o(`mrow`,null,[o(`mi`,{mathvariant:`normal`},`head`),o(`mo`,null,``)]),o(`mn`,null,`1`)]),o(`mtext`,null,` `),o(`mi`,{mathvariant:`normal`},`∥`),o(`mtext`,null,` `),o(`mo`,null,`⋯`),o(`mtext`,null,` `),o(`mi`,{mathvariant:`normal`},`∥`),o(`mtext`,null,` `),o(`msub`,null,[o(`mrow`,null,[o(`mi`,{mathvariant:`normal`},`head`),o(`mo`,null,``)]),o(`mi`,null,`h`)]),o(`mo`,{fence:`true`},`]`)]),o(`msub`,null,[o(`mi`,null,`W`),o(`mi`,null,`O`)])]),o(`annotation`,{encoding:`application/x-tex`},`\\operatorname{MHA}(X) = \\left[\\operatorname{head}_1 \\,\\|\\, \\cdots \\,\\|\\, \\operatorname{head}_h\\right] W_O`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),o(`span`,{class:`mop`},[o(`span`,{class:`mord mathrm`},`MHA`)]),o(`span`,{class:`mopen`},`(`),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0785em`}},`X`),o(`span`,{class:`mclose`},`)`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mrel`},`=`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),o(`span`,{class:`minner`},[o(`span`,{class:`mopen delimcenter`,style:{top:`0em`}},`[`),o(`span`,{class:`mop`},[o(`span`,{class:`mop`},[o(`span`,{class:`mord mathrm`},`head`)]),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3011em`}},[o(`span`,{style:{top:`-2.55em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`1`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])]),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mord`},`∥`),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`minner`},`⋯`),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mord`},`∥`),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mop`},[o(`span`,{class:`mop`},[o(`span`,{class:`mord mathrm`},`head`)]),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3361em`}},[o(`span`,{style:{top:`-2.55em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mathnormal mtight`},`h`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])]),o(`span`,{class:`mclose delimcenter`,style:{top:`0em`}},`]`)]),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3283em`}},[o(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0278em`}},`O`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])])])])])])],-1)]]),_:1})]),T[15]||=o(`div`,{class:`dl-secondary`},[o(`p`,null,[d(`Multi-head attention (MHA) of `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`X`)]),o(`annotation`,{encoding:`application/x-tex`},`X`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0785em`}},`X`)])])]),d(`, one row per token. `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`h`)]),o(`annotation`,{encoding:`application/x-tex`},`h`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6944em`}}),o(`span`,{class:`mord mathnormal`},`h`)])])]),d(` heads, 8 here; `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`msub`,null,[o(`mrow`,null,[o(`mi`,{mathvariant:`normal`},`head`),o(`mo`,null,``)]),o(`mi`,null,`i`)])]),o(`annotation`,{encoding:`application/x-tex`},`\\operatorname{head}_i`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),o(`span`,{class:`mop`},[o(`span`,{class:`mop`},[o(`span`,{class:`mord mathrm`},`head`)]),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3117em`}},[o(`span`,{style:{top:`-2.55em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mathnormal mtight`},`i`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])])])])]),d(` is head `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`i`)]),o(`annotation`,{encoding:`application/x-tex`},`i`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6595em`}}),o(`span`,{class:`mord mathnormal`},`i`)])])]),d(`’s output. `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,{mathvariant:`normal`},`∥`)]),o(`annotation`,{encoding:`application/x-tex`},`\\|`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),o(`span`,{class:`mord`},`∥`)])])]),d(`: side by side.`)])],-1)])),[[D]]),e((p(),c(`div`,C,[...T[16]||=[o(`p`,null,[d(`Four `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`d`),o(`mo`,null,`×`),o(`mi`,null,`d`)]),o(`annotation`,{encoding:`application/x-tex`},`d \\times d`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.7778em`,"vertical-align":`-0.0833em`}}),o(`span`,{class:`mord mathnormal`},`d`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),o(`span`,{class:`mbin`},`×`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6944em`}}),o(`span`,{class:`mord mathnormal`},`d`)])])]),d(` matrices — `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`msub`,null,[o(`mi`,null,`W`),o(`mi`,null,`Q`)])]),o(`annotation`,{encoding:`application/x-tex`},`W_Q`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.9694em`,"vertical-align":`-0.2861em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3283em`}},[o(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mathnormal mtight`},`Q`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.2861em`}},[o(`span`)])])])])])])])]),d(`, `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`msub`,null,[o(`mi`,null,`W`),o(`mi`,null,`K`)])]),o(`annotation`,{encoding:`application/x-tex`},`W_K`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3283em`}},[o(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0715em`}},`K`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])])])])]),d(`, `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`msub`,null,[o(`mi`,null,`W`),o(`mi`,null,`V`)])]),o(`annotation`,{encoding:`application/x-tex`},`W_V`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3283em`}},[o(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.2222em`}},`V`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])])])])]),d(`, `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`msub`,null,[o(`mi`,null,`W`),o(`mi`,null,`O`)])]),o(`annotation`,{encoding:`application/x-tex`},`W_O`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3283em`}},[o(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0278em`}},`O`)])])]),o(`span`,{class:`vlist-s`},`��`)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])])])])]),d(` — so `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mn`,null,`4`),o(`msup`,null,[o(`mi`,null,`d`),o(`mn`,null,`2`)])]),o(`annotation`,{encoding:`application/x-tex`},`4d^2`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8141em`}}),o(`span`,{class:`mord`},`4`),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`},`d`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.8141em`}},[o(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`2`)])])])])])])])])])]),d(` weights, `),o(`strong`,null,[d(`whatever `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`h`)]),o(`annotation`,{encoding:`application/x-tex`},`h`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6944em`}}),o(`span`,{class:`mord mathnormal`},`h`)])])]),d(` is`)]),d(`.`)],-1)]])),[[D]]),e((p(),c(`div`,w,[...T[17]||=[o(`p`,null,[d(`At `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`d`),o(`mo`,null,`=`),o(`mn`,null,`512`)]),o(`annotation`,{encoding:`application/x-tex`},`d = 512`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6944em`}}),o(`span`,{class:`mord mathnormal`},`d`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mrel`},`=`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),o(`span`,{class:`mord`},`512`)])])]),d(`: `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mn`,null,`4`),o(`mo`,null,`×`),o(`msup`,null,[o(`mn`,null,`512`),o(`mn`,null,`2`)]),o(`mo`,null,`=`),o(`mn`,null,`1`),o(`mtext`,null,` `),o(`mn`,null,`048`),o(`mtext`,null,` `),o(`mn`,null,`576`)]),o(`annotation`,{encoding:`application/x-tex`},`4 \\times 512^2 = 1\\,048\\,576`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.7278em`,"vertical-align":`-0.0833em`}}),o(`span`,{class:`mord`},`4`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),o(`span`,{class:`mbin`},`×`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8141em`}}),o(`span`,{class:`mord`},`51`),o(`span`,{class:`mord`},[o(`span`,{class:`mord`},`2`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.8141em`}},[o(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`2`)])])])])])])]),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mrel`},`=`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),o(`span`,{class:`mord`},`1`),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mord`},`048`),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mord`},`576`)])])]),d(`, for any `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`h`)]),o(`annotation`,{encoding:`application/x-tex`},`h`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6944em`}}),o(`span`,{class:`mord mathnormal`},`h`)])])]),d(`.`)],-1)]])),[[D]])])])]),_:1},16)}}};export{T as default};
|
lecture-06/assets/md-Bu64PrEi.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,x as l,yt as u,z as d}from"./modules/shiki-BFOesOFp.js";import{n as f,t as p}from"./slidev/context-DrkI8qiD.js";import{t as m}from"./slidev/VClicks-CGAOkBil.js";import{t as h}from"./interactive-ClWDKV9B.js";import{t as g}from"./AttentionMatrix-DXtaXAxa.js";var _={class:`mt-2 dl-callout`},v={__name:`slides.md__slidev_40`,setup(v){let{$slidev:y,$nav:b,$clicksContext:x,$clicks:S,$page:C,$renderContext:w,$frontmatter:T}=f();return x.setup(),(f,v)=>{let y=g,b=m,x=i(`click`);return d(),c(h,u(t(o(p)(o(T),39))),{aside:n(t=>[v[2]||=a(`p`,null,[l(`All three rows. Rows `),a(`strong`,null,`ask`),l(`, columns `),a(`strong`,null,`are read`),l(`.`)],-1),r(b,null,{default:n(()=>[...v[0]||=[a(`ul`,null,[a(`li`,null,[a(`em`,null,`the`),l(` asks nothing, so its row is flat: 0.33 each`)]),a(`li`,null,[a(`em`,null,`bank`),l(`’s row is the one we just derived`)]),a(`li`,null,[l(`Every row sums to `),a(`strong`,null,`1`)])],-1)]]),_:1}),e((d(),s(`div`,_,[...v[1]||=[a(`p`,null,[l(`No cell depends on another, so all nine are computed `),a(`strong`,null,`at the same time`),l(`. The wall comes down.`)],-1)]])),[[x]])]),default:n(()=>[r(y,{maskable:!1,switchable:``})]),_:1},16)}}};export{v as default};
|
lecture-06/assets/md-C2-OA9fh2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,yt as l,z as u}from"./modules/shiki-BFOesOFp.js";import{n as d,t as f}from"./slidev/context-DrkI8qiD.js";import{t as p}from"./TransformerGlyph-D9TehqNe.js";import{t as m}from"./default-sHmABwVd.js";var h={class:`grid grid-cols-4 gap-x-4 gap-y-2 mt-3`},g={class:`flex flex-col items-center text-center gap-1`},_={class:`flex flex-col items-center text-center gap-1`},v={class:`flex flex-col items-center text-center gap-1`},y={class:`flex flex-col items-center text-center gap-1`},b={class:`flex flex-col items-center text-center gap-1`},x={class:`flex flex-col items-center text-center gap-1`},S={class:`flex flex-col items-center text-center gap-1`},C={class:`flex flex-col items-center text-center gap-1`},w={class:`mt-3 dl-callout`},T={__name:`slides.md__slidev_2`,setup(T){let{$slidev:E,$nav:D,$clicksContext:O,$clicks:k,$page:A,$renderContext:j,$frontmatter:M}=d();return O.setup(),(d,T)=>{let E=p,D=i(`click`);return u(),c(m,l(t(o(f)(o(M),1))),{default:n(()=>[T[9]||=a(`h1`,null,`Today, in eight parts`,-1),T[10]||=a(`p`,null,`The whole machine first, then each part, then the kinds of transformer in use.`,-1),a(`div`,h,[a(`div`,g,[r(E,{kind:`cut`,size:64}),T[0]||=a(`span`,{class:`dl-secondary`},`00 Why a new design`,-1)]),a(`div`,_,[r(E,{kind:`encdec`,size:64}),T[1]||=a(`span`,{class:`dl-secondary`},`01 The whole architecture`,-1)]),a(`div`,v,[r(E,{kind:`qkv`,size:64}),T[2]||=a(`span`,{class:`dl-secondary`},`02 Attention: query, key, value`,-1)]),a(`div`,y,[r(E,{kind:`block`,size:64}),T[3]||=a(`span`,{class:`dl-secondary`},`03 One block`,-1)]),a(`div`,b,[r(E,{kind:`decoder`,size:64}),T[4]||=a(`span`,{class:`dl-secondary`},`04 Inside the decoder`,-1)]),a(`div`,x,[r(E,{kind:`fork`,size:64}),T[5]||=a(`span`,{class:`dl-secondary`},`05 Three kinds, and their uses`,-1)]),a(`div`,S,[r(E,{kind:`code`,size:64}),T[6]||=a(`span`,{class:`dl-secondary`},`06 In PyTorch`,-1)]),a(`div`,C,[r(E,{kind:`block`,size:64}),T[7]||=a(`span`,{class:`dl-secondary`},`07 Where that leaves us`,-1)])]),e((u(),s(`div`,w,[...T[8]||=[a(`p`,null,`Section 01 shows the full picture. Sections 02 to 04 open its boxes one by one.`,-1)]])),[[D]])]),_:1},16)}}};export{T as default};
|
lecture-06/assets/md-C6qfUyBt2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,m as l,v as u,x as d,yt as f,z as p}from"./modules/shiki-BFOesOFp.js";import{n as m,t as h}from"./slidev/context-DrkI8qiD.js";import{t as g}from"./default-sHmABwVd.js";import{t as _}from"./slidev/VClicks-CGAOkBil.js";var v={class:`grid grid-cols-2 gap-8 mt-2 items-start`},y={viewBox:`0 0 330 200`,class:`dl-diagram`,role:`img`,"aria-label":`Score grids: 4 words give a 4 by 4 grid of 16 scores; 8 words give an 8 by 8 grid of 64 scores, four times as many`,style:{"max-height":`15rem`}},b=[`x`,`y`],x=[`x`,`y`],S={class:`dl-tight`},C={class:`mt-3 dl-callout`},w={class:`mt-3 dl-secondary`},T={__name:`slides.md__slidev_66`,setup(T){let{$slidev:E,$nav:D,$clicksContext:O,$clicks:k,$page:A,$renderContext:j,$frontmatter:M}=m();return O.setup(),(m,T)=>{let E=_,D=a(`click`);return p(),u(g,f(t(s(h)(s(M),65))),{default:n(()=>[T[6]||=o(`h1`,null,`The bill for looking everywhere`,-1),T[7]||=o(`p`,null,[d(`The score matrix is `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`T`),o(`mo`,null,`×`),o(`mi`,null,`T`)]),o(`annotation`,{encoding:`application/x-tex`},`T \\times T`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.7667em`,"vertical-align":`-0.0833em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),o(`span`,{class:`mbin`},`×`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`)])])]),d(`: every position scores every position.`)],-1),o(`div`,v,[o(`div`,null,[(p(),c(`svg`,y,[T[1]||=o(`text`,{class:`dl-dg-small`,x:`10`,y:`16`},`query × key`,-1),(p(),c(l,null,i(16,e=>o(`rect`,{key:`a`+e,class:`dl-dg-box is-accent`,x:10+(e-1)%4*18,y:30+Math.floor((e-1)/4)*18,width:`16`,height:`16`,rx:`2`},null,8,b)),64)),T[2]||=o(`text`,{class:`dl-dg-small`,x:`10`,y:`122`},`T = 4: 16 scores`,-1),e((p(),c(`g`,null,[(p(),c(l,null,i(64,e=>o(`rect`,{key:`b`+e,class:`dl-dg-box`,x:160+(e-1)%8*18,y:30+Math.floor((e-1)/8)*18,width:`16`,height:`16`,rx:`2`},null,8,x)),64)),T[0]||=o(`text`,{class:`dl-dg-small`,x:`160`,y:`190`},`T = 8: 64 scores, 4 ×`,-1)])),[[D]])]))]),o(`div`,S,[r(E,null,{default:n(()=>[...T[3]||=[o(`ul`,null,[o(`li`,null,[d(`512 tokens → 262 144 scores `),o(`strong`,null,`per head, per layer`)])],-1)]]),_:1}),e((p(),c(`div`,C,[...T[4]||=[o(`p`,null,[d(`An RNN took `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`T`)]),o(`annotation`,{encoding:`application/x-tex`},`T`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`)])])]),d(` steps, one after another. Attention takes `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`msup`,null,[o(`mi`,null,`T`),o(`mn`,null,`2`)])]),o(`annotation`,{encoding:`application/x-tex`},`T^2`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8141em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.8141em`}},[o(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`2`)])])])])])])])])])]),d(`, all at once: more work, far less waiting.`)],-1)]])),[[D]]),e((p(),c(`div`,w,[...T[5]||=[o(`p`,null,[d(`Caveat: the feed-forward network grows like `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`T`),o(`msup`,null,[o(`mi`,null,`d`),o(`mn`,null,`2`)])]),o(`annotation`,{encoding:`application/x-tex`},`T d^2`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8141em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`},`d`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.8141em`}},[o(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`2`)])])])])])])])])])]),d(`, attention like `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`msup`,null,[o(`mi`,null,`T`),o(`mn`,null,`2`)]),o(`mi`,null,`d`)]),o(`annotation`,{encoding:`application/x-tex`},`T^2 d`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8141em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.8141em`}},[o(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`2`)])])])])])])]),o(`span`,{class:`mord mathnormal`},`d`)])])]),d(`. At `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`T`),o(`mo`,null,`=`),o(`mi`,null,`d`),o(`mo`,null,`=`),o(`mn`,null,`512`)]),o(`annotation`,{encoding:`application/x-tex`},`T = d = 512`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mrel`},`=`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6944em`}}),o(`span`,{class:`mord mathnormal`},`d`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mrel`},`=`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),o(`span`,{class:`mord`},`512`)])])]),d(`, attention is not yet the expensive part.`)],-1)]])),[[D]])])])]),_:1},16)}}};export{T as default};
|
lecture-06/assets/md-C8OiMOzj.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,bt as l,g as u,lt as d,m as f,v as p,vt as m,w as h,x as g,xt as _,y as v,yt as y,z as b}from"./modules/shiki-BFOesOFp.js";import{t as x}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as S,t as C}from"./slidev/context-DrkI8qiD.js";import{t as w}from"./slidev/VClicks-CGAOkBil.js";import{n as T,t as E}from"./interactive-ClWDKV9B.js";import{t as D}from"./StepButton-BcP33F-1.js";import{t as O}from"./useConvolution-qO1u8ksh.js";import{n as k,r as A}from"./useAttention-Blk0c2BZ.js";var j={class:`dl-mh`},M={class:`dl-mh__sentence`},N=[`onClick`],P={class:`dl-mh__rows`},F=[`onClick`],I={class:`dl-mh__headlabel`},L={class:`dl-mh__headreads`},R={class:`dl-mh__bars`},z=[`title`],B={class:`dl-mh__reads`},V={class:`dl-mh__concat`},H={class:`dl-mh__whole`},U={class:`dl-mh__whole is-out`},W=x(h({__name:`MultiHeadLab`,props:{query:{default:6},dModel:{default:512}},setup(e){let{$slidev:t,$nav:a,$clicksContext:h,$clicks:y,$page:x,$renderContext:C,$frontmatter:w}=S(),E=e,W=d(E.query),G=d(null),K=A,q=u(()=>k.map((e,t)=>({...e,index:t,weights:e.weights[W.value],top:e.weights[W.value].indexOf(Math.max(...e.weights[W.value]))}))),J=u(()=>G.value===null?q.value:[q.value[G.value]]),Y=u(()=>Math.round(E.dModel/k.length));function X(){W.value=(W.value+1)%K.length}function Z(e){return e<=W.value}return(e,t)=>{let a=D,u=T;return b(),p(u,{"max-width":`45rem`},{controls:n(()=>[r(a,{label:`Next query word`,glyph:`▸`,onClick:X}),G.value===null?v(`v-if`,!0):(b(),p(a,{key:0,label:`All heads`,variant:`ghost`,glyph:`▦`,onClick:t[0]||=e=>G.value=null}))]),readout:n(()=>[W.value===6?(b(),c(f,{key:0},[t[5]||=o(`strong`,null,`it`,-1),t[6]||=g(` asks four questions at once: head 1 takes the word before it, head 2 the nearby nouns, head 3 goes back five words to `,-1),t[7]||=o(`strong`,null,`cat`,-1),t[8]||=g(` — what `,-1),t[9]||=o(`em`,null,`it`,-1),t[10]||=g(` refers to — and head 4 parks on the first token. One softmax puts its mass in one place, so one head could answer only one of those. `,-1),t[11]||=o(`span`,{class:`dl-mh__note`},[g(` Head 4 is not a drawing error: heads that sit almost entirely on the first token turn up in every trained model. Nothing after `),o(`em`,null,`it`),g(` is drawn because this is a decoder, which may not look forward — section 03. `)],-1)],64)):(b(),c(f,{key:1},[o(`strong`,null,_(s(K)[W.value]),1),t[12]||=g(` asking, four heads answering differently: `,-1),(b(!0),c(f,null,i(q.value,(e,t)=>(b(),c(f,{key:e.name},[t>0?(b(),c(f,{key:0},[g(`, `)],64)):v(`v-if`,!0),g(` `+_(e.name)+` → `,1),o(`strong`,null,_(s(K)[e.top]),1)],64))),128)),t[13]||=g(`. Each head is its own W`,-1),t[14]||=o(`sub`,null,`Q`,-1),t[15]||=g(`, W`,-1),t[16]||=o(`sub`,null,`K`,-1),t[17]||=g(`, W`,-1),t[18]||=o(`sub`,null,`V`,-1),g(` at width `+_(Y.value)+` = `+_(E.dModel)+`/`+_(s(k).length)+`, so all `+_(s(k).length)+` together cost exactly what one head of width `+_(E.dModel)+` would. `,1)],64))]),default:n(()=>[o(`div`,j,[v(` The sentence, and which word is asking. `),o(`div`,M,[t[1]||=o(`span`,{class:`dl-mh__label`},`query`,-1),(b(!0),c(f,null,i(s(K),(e,t)=>(b(),c(`button`,{key:e+t,type:`button`,class:m([`dl-mh__word`,{"is-current":t===W.value}]),onClick:e=>W.value=t},_(e),11,N))),128))]),v(` One row per head: the same question, four different answers. `),o(`div`,P,[(b(!0),c(f,null,i(J.value,e=>(b(),c(`div`,{key:e.name,class:`dl-mh__row`},[o(`button`,{type:`button`,class:m([`dl-mh__headname`,{"is-solo":G.value===e.index}]),onClick:t=>G.value=G.value===e.index?null:e.index},[o(`span`,I,_(e.name),1),o(`span`,L,_(e.reads),1)],10,F),o(`div`,R,[(b(!0),c(f,null,i(e.weights,(t,n)=>(b(),c(`div`,{key:n,class:m([`dl-mh__bar`,{"is-top":n===e.top&&Z(n),"is-dead":!Z(n)}]),title:`${s(K)[n]}: ${s(O)(t)}`},[o(`span`,{class:`dl-mh__fill`,style:l({height:`${Z(n)?t*100:0}%`})},null,4)],10,z))),128))]),o(`div`,B,[e.weights[e.top]>.4?(b(),c(f,{key:0},[t[2]||=g(`→ `,-1),o(`strong`,null,_(s(K)[e.top]),1)],64)):(b(),c(f,{key:1},[g(`→ spread`)],64))])]))),128))]),v(` What happens to the four outputs. `),o(`div`,V,[(b(!0),c(f,null,i(q.value,e=>(b(),c(`span`,{key:`c${e.index}`,class:`dl-mh__slice`},_(Y.value),1))),128)),t[3]||=o(`span`,{class:`dl-mh__op`},`concat →`,-1),o(`span`,H,_(E.dModel),1),t[4]||=o(`span`,{class:`dl-mh__op`},[g(`× W`),o(`sub`,null,`O`),g(` →`)],-1),o(`span`,U,_(E.dModel),1)])])]),_:1})}}}),[[`__scopeId`,`data-v-d4d1022e`]]),G={class:`mt-2 dl-callout`},K={__name:`slides.md__slidev_49`,setup(i){let{$slidev:l,$nav:u,$clicksContext:d,$clicks:f,$page:m,$renderContext:h,$frontmatter:_}=S();return d.setup(),(i,l)=>{let u=W,d=w,f=a(`click`);return b(),p(E,y(t(s(C)(s(_),48))),{aside:n(t=>[l[2]||=o(`p`,null,[o(`em`,null,`it`),g(` is asking. Four heads, four different answers.`)],-1),r(d,null,{default:n(()=>[...l[0]||=[o(`ul`,null,[o(`li`,null,[o(`strong`,null,`head 1`),g(` — the previous word`)]),o(`li`,null,[o(`strong`,null,`head 3`),g(` — five words back, to `),o(`em`,null,`cat`)]),o(`li`,null,[o(`strong`,null,`head 4`),g(` — parks on token 1 and does nothing`)])],-1)]]),_:1}),e((b(),c(`div`,G,[...l[1]||=[o(`p`,null,`Head 4 is not a drawing error. Nearly-dead heads are in every model anyone has looked inside.`,-1)]])),[[f]])]),default:n(()=>[r(u)]),_:1},16)}}};export{K as default};
|
lecture-06/assets/md-CB6QZfkK2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_26`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),25))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: The sentence we will use all lecture`,-1),n(`div`,{class:`dl-describe`},[n(`ul`,null,[n(`li`,null,[a(`The running example is "the river bank". Each bar chart is one word’s embedding: four numbers, one bar each. We pretend the four positions mean det (a word like `),n(`em`,null,`the`),a(`), water, money and thing.`)]),n(`li`,null,[n(`em`,null,`the`),a(` = `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mo`,{stretchy:`false`},`[`),n(`mn`,null,`1`),n(`mo`,{separator:`true`},`,`),n(`mn`,null,`0`),n(`mo`,{separator:`true`},`,`),n(`mn`,null,`0`),n(`mo`,{separator:`true`},`,`),n(`mn`,null,`0`),n(`mo`,{stretchy:`false`},`]`)]),n(`annotation`,{encoding:`application/x-tex`},`[1, 0, 0, 0]`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),n(`span`,{class:`mopen`},`[`),n(`span`,{class:`mord`},`1`),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},`0`),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},`0`),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},`0`),n(`span`,{class:`mclose`},`]`)])])]),a(`: only the det bar.`)]),n(`li`,null,[n(`em`,null,`river`),a(` = `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mo`,{stretchy:`false`},`[`),n(`mn`,null,`0`),n(`mo`,{separator:`true`},`,`),n(`mn`,null,`2`),n(`mo`,{separator:`true`},`,`),n(`mn`,null,`0`),n(`mo`,{separator:`true`},`,`),n(`mn`,null,`1`),n(`mo`,{stretchy:`false`},`]`)]),n(`annotation`,{encoding:`application/x-tex`},`[0, 2, 0, 1]`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),n(`span`,{class:`mopen`},`[`),n(`span`,{class:`mord`},`0`),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},`2`),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},`0`),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},`1`),n(`span`,{class:`mclose`},`]`)])])]),a(`: a thing, and watery.`)]),n(`li`,null,[n(`em`,null,`bank`),a(` = `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mo`,{stretchy:`false`},`[`),n(`mn`,null,`0`),n(`mo`,{separator:`true`},`,`),n(`mn`,null,`0`),n(`mo`,{separator:`true`},`,`),n(`mn`,null,`0`),n(`mo`,{separator:`true`},`,`),n(`mn`,null,`2`),n(`mo`,{stretchy:`false`},`]`)]),n(`annotation`,{encoding:`application/x-tex`},`[0, 0, 0, 2]`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),n(`span`,{class:`mopen`},`[`),n(`span`,{class:`mord`},`0`),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},`0`),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},`0`),n(`span`,{class:`mpunct`},`,`),n(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),n(`span`,{class:`mord`},`2`),n(`span`,{class:`mclose`},`]`)])])]),a(`: only the thing bar. Its own vector does not say which kind of bank it is.`)]),n(`li`,null,[a(`So the meaning of `),n(`em`,null,`bank`),a(` must come from a neighbour, here `),n(`em`,null,`river`),a(`. Attention is the mechanism that moves it there.`)]),n(`li`,null,`The numbers are hand-picked so the arithmetic works by hand.`)])],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-CBhrP8n32.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,x as l,yt as u,z as d}from"./modules/shiki-BFOesOFp.js";import{n as f,t as p}from"./slidev/context-DrkI8qiD.js";import{t as m}from"./slidev/VClicks-CGAOkBil.js";import{t as h}from"./interactive-ClWDKV9B.js";import{t as g}from"./TransformerBlock-CZtncS4l.js";var _={class:`mt-2 dl-callout`},v={__name:`slides.md__slidev_62`,setup(v){let{$slidev:y,$nav:b,$clicksContext:x,$clicks:S,$page:C,$renderContext:w,$frontmatter:T}=f();return x.setup(),(f,v)=>{let y=g,b=m,x=i(`click`);return d(),c(h,u(t(o(p)(o(T),61))),{aside:n(t=>[v[2]||=a(`p`,null,[l(`Press `),a(`strong`,null,`Walk the block`),l(`. Six parts, all of them already met.`)],-1),r(b,null,{default:n(()=>[...v[0]||=[a(`ul`,null,[a(`li`,null,[l(`attention — the `),a(`strong`,null,`only`),l(` sideways step`)]),a(`li`,null,`add — the residual`),a(`li`,null,`norm`),a(`li`,null,`feed-forward — each position alone`),a(`li`,null,`add, norm`)],-1)]]),_:1}),e((d(),s(`div`,_,[...v[1]||=[a(`p`,null,`Same shape at the top as at the bottom. Stack it.`,-1)]])),[[x]])]),default:n(()=>[r(y,{variant:`post`,switchable:!1})]),_:1},16)}}};export{v as default};
|
lecture-06/assets/md-CBiTRCiA2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_63`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),62))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: The block, assembled`,-1),n(`div`,{class:`dl-describe`},[n(`p`,null,`The widget draws one transformer block as six steps, read from bottom to top.`),n(`ol`,null,[n(`li`,null,[n(`strong`,null,`Attention`),a(`: every position looks at every other position. This is the only step that moves information between words.`)]),n(`li`,null,[n(`strong`,null,`Add`),a(`: the residual adds the input back.`)]),n(`li`,null,[n(`strong`,null,`Norm`),a(`: LayerNorm re-scales each position’s vector.`)]),n(`li`,null,[n(`strong`,null,`Feed-forward`),a(`: each position goes through the small network alone.`)]),n(`li`,null,[n(`strong`,null,`Add`),a(` and 6. `),n(`strong`,null,`norm`),a(` again, around the feed-forward network.`)])]),n(`p`,null,`The vectors leave the top with the same shape they had at the bottom.`),n(`p`,null,`Takeaway: attention plus a small network, each wrapped in add and norm. The shape is kept, so blocks stack.`)],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-CGRw6CdD.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_102`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),101))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: A block, and a stack`,-1),n(`div`,{class:`dl-describe`},[n(`p`,null,`The slide shows one transformer block in PyTorch, and a stack of six blocks.`),n(`ul`,null,[n(`li`,null,[n(`strong`,null,`Reading the sketch:`),a(` start at the bottom. The accent line is the residual stream `),n(`code`,null,`x`),a(`.`)]),n(`li`,null,[n(`strong`,null,`First branch:`),a(` a copy of `),n(`code`,null,`x`),a(` is normalised and goes through attention. The result is added back at the first plus sign.`)]),n(`li`,null,[n(`strong`,null,`Second branch:`),a(` the same, with the feed-forward network, at the second plus sign.`)]),n(`li`,null,[n(`strong`,null,`In code:`),a(` each branch is one line, `),n(`code`,null,`x = x + sublayer(norm(x))`),a(`.`)]),n(`li`,null,[n(`strong`,null,`The stack:`),a(` six `),n(`code`,null,`Block(512, 8, 2048)`),a(` objects inside `),n(`code`,null,`nn.Sequential`),a(`.`)])]),n(`p`,null,`This is the pre-LN wiring: the norm acts on the copy, never on the stream itself.`)],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-CHhgCgN4.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,yt as a,z as o}from"./modules/shiki-BFOesOFp.js";import{n as s,t as c}from"./slidev/context-DrkI8qiD.js";import{t as l}from"./default-sHmABwVd.js";var u={__name:`slides.md__slidev_111`,setup(u){let{$slidev:d,$nav:f,$clicksContext:p,$clicks:m,$page:h,$renderContext:g,$frontmatter:_}=s();return p.setup(),(s,u)=>(o(),i(l,a(e(r(c)(r(_),110))),{default:t(()=>[...u[0]||=[n(`h1`,null,`In words: Where we got to`,-1),n(`div`,{class:`dl-describe`},[n(`p`,null,`Each glyph stands for one section, in lecture order: the whole, the parts, the uses.`),n(`ul`,null,[n(`li`,null,`With no recurrence, every position is processed at the same time.`),n(`li`,null,`The whole model is an encoder stack and a decoder stack, joined by cross-attention.`),n(`li`,null,`Attention is a soft lookup: a query meets every key, and the values are mixed.`),n(`li`,null,`A block adds heads, position, a per-position network, and residuals with norms.`),n(`li`,null,`The decoder adds a mask and cross-attention.`),n(`li`,null,`Encoder-only labels, decoder-only writes, encoder–decoder translates.`)]),n(`p`,null,`The sentence to remember: mix across positions, then think about each position alone.`)],-1)]]),_:1},16))}};export{u as default};
|
lecture-06/assets/md-CIA7piby2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`slides.md__slidev_77`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),76))),{default:t(()=>[p[0]||=r(`h1`,null,`Three kinds of transformer`,-1),p[1]||=r(`p`,null,`The same parts, wired three ways, for different jobs.`,-1),r(`div`,f,[n(m,{kind:`fork`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/assets/md-CKfbaq-X.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_56`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),55))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: So put the position into the vector`,-1),n(`div`,{class:`dl-describe`},[n(`p`,null,`The widget shows 12 positions as rows and 16 dimensions as columns. Each row is the position vector for that place in the sentence.`),n(`ul`,null,[n(`li`,null,`The numbers come from sines and cosines. Nothing is learned.`),n(`li`,null,`The left columns change quickly from row to row. The right columns change slowly.`),n(`li`,null,`Read together, the fast and slow columns give every position its own pattern.`),n(`li`,null,[a(`This vector is `),n(`strong`,null,`added`),a(` to the word’s embedding before the first block. The vector does not get any wider.`)])]),n(`p`,null,`Takeaway: after the addition, one vector says both which word it is and where it sits in the sentence.`)],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-CLUR6rvJ.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_41`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),40))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: Every query at once`,-1),n(`div`,{class:`dl-describe`},[n(`ul`,null,[n(`li`,null,`The grid is the full attention matrix for "the river bank". Each row is a word asking with its query. Each column is a word being read through its key.`),n(`li`,null,[a(`Row `),n(`em`,null,`the`),a(`: its query is zero, so every score is 0. The weights are flat, 0.33 each.`)]),n(`li`,null,[a(`Row `),n(`em`,null,`river`),a(`: weights 0.25, 0.50, 0.25.`)]),n(`li`,null,[a(`Row `),n(`em`,null,`bank`),a(`: 0.16, 0.67, 0.16, the row from the five-line derivation.`)]),n(`li`,null,`Every row adds up to 1.`),n(`li`,null,`No cell needs any other cell. So all nine are computed at the same time, in one matrix multiply. A recurrent network had to compute its states one after another, so this removes the second wall from section 00.`)])],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-CNDLpstn2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_81`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),80))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: Which kind for which job`,-1),n(`div`,{class:`dl-describe`},[n(`p`,null,`The three kinds of transformer fit three shapes of task.`),n(`ul`,null,[n(`li`,null,[n(`strong`,null,`Encoder-only`),a(` reads the whole input at once, with no mask. It suits jobs where the input is complete and one answer comes out: sentiment, spam, topic, search embeddings, named entities. Example: BERT (bidirectional encoder representations from transformers).`)]),n(`li`,null,[n(`strong`,null,`Decoder-only`),a(` writes left to right, one word at a time. It suits jobs where text comes out: chat, code completion, question answering. Examples: GPT (generative pre-trained transformer), Llama.`)]),n(`li`,null,[n(`strong`,null,`Encoder–decoder`),a(` reads one full sequence and writes another: translation, summarisation, speech to text. Examples: T5 (text-to-text transfer transformer), Whisper.`)])]),n(`p`,null,`Rule of thumb: label it, write it, or turn it into something else.`)],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-CSrnKzHt.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`slides.md__slidev_96`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),95))),{default:t(()=>[p[0]||=r(`h1`,null,`In PyTorch`,-1),p[1]||=r(`p`,null,`Everything you computed by hand, as code.`,-1),r(`div`,f,[n(m,{kind:`code`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/assets/md-CV1HWu48.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,W as r,_ as i,_t as a,b as o,v as s,x as c,yt as l,z as u}from"./modules/shiki-BFOesOFp.js";import{n as d,t as f}from"./slidev/context-DrkI8qiD.js";import{t as p}from"./default-sHmABwVd.js";var m={class:`mt-2 dl-callout`},h={__name:`slides.md__slidev_105`,setup(h){let{$slidev:g,$nav:_,$clicksContext:v,$clicks:y,$page:b,$renderContext:x,$frontmatter:S}=d();return v.setup(),(d,h)=>{let g=r(`click`);return u(),s(p,l(t(a(f)(a(S),104))),{default:n(()=>[h[1]||=i(`h1`,null,`Shapes, end to end`,-1),h[2]||=i(`div`,{class:`grid grid-cols-[1fr_12rem] gap-6 items-center`},[i(`div`,{class:`dl-ledger`},[i(`table`,null,[i(`thead`,null,[i(`tr`,null,[i(`th`,null,`step`),i(`th`,null,`shape`),i(`th`,null,`what it is`)])]),i(`tbody`,null,[i(`tr`,null,[i(`td`,null,[i(`code`,null,`ids`)]),i(`td`,null,[i(`code`,null,`(32, 128)`)]),i(`td`,null,`token ids, padded`)]),i(`tr`,null,[i(`td`,null,[i(`code`,null,`embed(ids)`),c(` + position`)]),i(`td`,null,[i(`code`,null,`(32, 128, 512)`)]),i(`td`,null,`added: same width`)]),i(`tr`,null,[i(`td`,null,[i(`code`,null,`qkv(x)`)]),i(`td`,null,[i(`code`,null,`(32, 128, 1536)`)]),i(`td`,null,`Q, K, V in one tensor`)]),i(`tr`,null,[i(`td`,null,`reshape`),i(`td`,null,[i(`code`,null,`(32, 8, 128, 64)`)]),i(`td`,null,`heads as batch`)]),i(`tr`,null,[i(`td`,null,[i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`mi`,null,`Q`),i(`msup`,null,[i(`mi`,null,`K`),i(`mi`,{mathvariant:`normal`},`⊤`)])]),i(`annotation`,{encoding:`application/x-tex`},`QK^\\top`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`1.0435em`,"vertical-align":`-0.1944em`}}),i(`span`,{class:`mord mathnormal`},`Q`),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0715em`}},`K`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.8491em`}},[i(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mtight`},`⊤`)])])])])])])])])])])]),i(`td`,null,[i(`code`,null,`(32, 8, 128, 128)`)]),i(`td`,null,[i(`strong`,null,`the scores`)])]),i(`tr`,null,[i(`td`,null,`weights @ V`),i(`td`,null,[i(`code`,null,`(32, 8, 128, 64)`)]),i(`td`,null,`output per head`)]),i(`tr`,null,[i(`td`,null,[c(`merged + `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`msub`,null,[i(`mi`,null,`W`),i(`mi`,null,`O`)])]),i(`annotation`,{encoding:`application/x-tex`},`W_O`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t vlist-t2`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.3283em`}},[i(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0278em`}},`O`)])])]),i(`span`,{class:`vlist-s`},``)]),i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.15em`}},[i(`span`)])])])])])])])])]),i(`td`,null,[i(`code`,null,`(32, 128, 512)`)]),i(`td`,null,`residual stream`)]),i(`tr`,null,[i(`td`,null,[i(`code`,null,`head(x)`)]),i(`td`,null,[i(`code`,null,`(32, 128, 50257)`)]),i(`td`,null,`logit per vocabulary entry`)])])])]),i(`div`,null,[i(`svg`,{class:`dl-diagram`,viewBox:`0 0 200 224`,role:`img`,"aria-label":`The scores tensor drawn to scale by area. At 128 tokens it is a tiny square of 4.2 million numbers in the corner. At 2048 tokens it is the large square, 1.07 billion numbers, 256 times the area.`},[i(`rect`,{x:`20`,y:`20`,width:`160`,height:`160`,class:`dl-dg-fill is-muted is-dashed`}),i(`rect`,{x:`20`,y:`20`,width:`10`,height:`10`,class:`dl-dg-fill`}),i(`text`,{x:`20`,y:`13`,class:`dl-dg-small is-good`},`T = 128: 4.2 M (corner)`),i(`text`,{x:`100`,y:`198`,class:`dl-dg-small`,"text-anchor":`middle`},`T = 2048: 1.07 billion`)])])],-1),e((u(),o(`div`,m,[...h[0]||=[i(`p`,null,[c(`Only the scores grow with `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`mi`,null,`T`)]),i(`annotation`,{encoding:`application/x-tex`},`T`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`)])])]),c(),i(`strong`,null,`twice`),c(`. That row is the long-context problem.`)],-1)]])),[[g]])]),_:1},16)}}};export{h as default};
|
lecture-06/assets/md-CVYuYWEV.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,_ as n,_t as r,v as i,x as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./default-sHmABwVd.js";var d={__name:`slides.md__slidev_74`,setup(d){let{$slidev:f,$nav:p,$clicksContext:m,$clicks:h,$page:g,$renderContext:_,$frontmatter:v}=c();return m.setup(),(c,d)=>(s(),i(u,o(e(r(l)(r(v),73))),{default:t(()=>[...d[0]||=[n(`h1`,null,`In words: How many decoder passes teacher forcing needs`,-1),n(`div`,{class:`dl-describe`},[n(`p`,null,[a(`The poll asked how many decoder passes it takes to train on a 100-token target. The answer is `),n(`strong`,null,`one`),a(`.`)]),n(`ul`,null,[n(`li`,null,[n(`strong`,null,`In training, the whole target sentence is the decoder’s input.`),a(` This is teacher forcing.`)]),n(`li`,null,[n(`strong`,null,[a(`The mask stops position `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`t`)]),n(`annotation`,{encoding:`application/x-tex`},`t`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6151em`}}),n(`span`,{class:`mord mathnormal`},`t`)])])]),a(` from seeing anything after `),n(`span`,{class:`katex`},[n(`span`,{class:`katex-mathml`},[n(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[n(`semantics`,null,[n(`mrow`,null,[n(`mi`,null,`t`)]),n(`annotation`,{encoding:`application/x-tex`},`t`)])])]),n(`span`,{class:`katex-html`,"aria-hidden":`true`},[n(`span`,{class:`katex-base`},[n(`span`,{class:`katex-strut`,style:{height:`0.6151em`}}),n(`span`,{class:`mord mathnormal`},`t`)])])]),a(`.`)]),a(` So each position is still a fair next-token prediction.`)]),n(`li`,null,[n(`strong`,null,`All 100 positions are therefore computed together, in one pass.`)]),n(`li`,null,[n(`strong`,null,`Generation is the opposite: 100 passes, one per token.`),a(` A token does not exist until it has been sampled.`)])]),n(`p`,null,`Training is parallel over the sequence; generation is not.`)],-1)]]),_:1},16))}};export{d as default};
|
lecture-06/assets/md-CW3GRSxh2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,x as l,yt as u,z as d}from"./modules/shiki-BFOesOFp.js";import{n as f,t as p}from"./slidev/context-DrkI8qiD.js";import{t as m}from"./default-sHmABwVd.js";import{t as h}from"./Citation-C9LcHAMd.js";var g={class:`mt-1 flex justify-center`},_={viewBox:`0 0 680 200`,class:`dl-diagram`,style:{"max-width":`34rem`},role:`img`,"aria-label":`Left: attention sits on top of a recurrent chain. Right: the chain is gone and the words the, river, bank connect directly through attention`},v={class:`dl-tight`},y={class:`mt-2 dl-callout`},b={class:`mt-1`},x={__name:`slides.md__slidev_7`,setup(x){let{$slidev:S,$nav:C,$clicksContext:w,$clicks:T,$page:E,$renderContext:D,$frontmatter:O}=f();return w.setup(),(f,x)=>{let S=h,C=i(`click`);return d(),c(m,u(t(o(p)(o(O),6))),{default:n(()=>[x[21]||=a(`h1`,null,`The move`,-1),x[22]||=a(`p`,null,[l(`Attention began as a `),a(`strong`,null,`passenger`),l(` on a recurrent model that did the real work.`)],-1),a(`div`,g,[(d(),s(`svg`,_,[x[1]||=a(`defs`,null,[a(`marker`,{id:`l6s0-move-head`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[a(`path`,{class:`dl-dg-head`,d:`M0 0 L7 3.5 L0 7 z`})])],-1),x[2]||=a(`text`,{class:`dl-dg-small`,x:`8`,y:`16`},`before — attention on top`,-1),x[3]||=a(`circle`,{class:`dl-dg-op`,cx:`95`,cy:`48`,r:`16`},null,-1),x[4]||=a(`text`,{class:`dl-dg-lab is-sm`,x:`95`,y:`53`,"text-anchor":`middle`},`Σ`,-1),x[5]||=a(`text`,{class:`dl-dg-small`,x:`120`,y:`52`},`weighted sum`,-1),x[6]||=a(`path`,{class:`dl-dg-line`,d:`M85 61 L31 128`},null,-1),x[7]||=a(`path`,{class:`dl-dg-line`,d:`M95 64 L99 128`},null,-1),x[8]||=a(`path`,{class:`dl-dg-line`,d:`M105 61 L167 128`},null,-1),x[9]||=a(`rect`,{class:`dl-dg-box`,x:`8`,y:`130`,width:`46`,height:`30`,rx:`4`},null,-1),x[10]||=a(`rect`,{class:`dl-dg-box`,x:`76`,y:`130`,width:`46`,height:`30`,rx:`4`},null,-1),x[11]||=a(`rect`,{class:`dl-dg-box`,x:`144`,y:`130`,width:`46`,height:`30`,rx:`4`},null,-1),x[12]||=a(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-move-head)`,d:`M54 145 H74`},null,-1),x[13]||=a(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-move-head)`,d:`M122 145 H142`},null,-1),x[14]||=a(`text`,{class:`dl-dg-small`,x:`31`,y:`184`,"text-anchor":`middle`},`the`,-1),x[15]||=a(`text`,{class:`dl-dg-small`,x:`99`,y:`184`,"text-anchor":`middle`},`river`,-1),x[16]||=a(`text`,{class:`dl-dg-small`,x:`167`,y:`184`,"text-anchor":`middle`},`bank`,-1),x[17]||=a(`text`,{class:`dl-dg-small`,x:`200`,y:`150`},`the chain does the work`,-1),x[18]||=a(`line`,{class:`dl-dg-split`,x1:`340`,y1:`8`,x2:`340`,y2:`192`},null,-1),e((d(),s(`g`,null,[...x[0]||=[a(`text`,{class:`dl-dg-small is-good`,x:`360`,y:`16`},`2017 — keep the weighted sum, delete the chain`,-1),a(`path`,{class:`dl-dg-line`,d:`M415 128 Q460 92 505 128`},null,-1),a(`path`,{class:`dl-dg-line`,d:`M505 128 Q550 92 595 128`},null,-1),a(`path`,{class:`dl-dg-line`,d:`M415 128 Q505 46 595 128`},null,-1),a(`rect`,{class:`dl-dg-box is-accent`,x:`380`,y:`130`,width:`70`,height:`30`,rx:`4`},null,-1),a(`rect`,{class:`dl-dg-box is-accent`,x:`470`,y:`130`,width:`70`,height:`30`,rx:`4`},null,-1),a(`rect`,{class:`dl-dg-box is-accent`,x:`560`,y:`130`,width:`70`,height:`30`,rx:`4`},null,-1),a(`text`,{class:`dl-dg-in`,x:`415`,y:`150`,"text-anchor":`middle`},`the`,-1),a(`text`,{class:`dl-dg-in`,x:`505`,y:`150`,"text-anchor":`middle`},`river`,-1),a(`text`,{class:`dl-dg-in`,x:`595`,y:`150`,"text-anchor":`middle`},`bank`,-1),a(`text`,{class:`dl-dg-small`,x:`505`,y:`184`,"text-anchor":`middle`},`all positions at once — nothing waits`,-1)]])),[[C]])]))]),e((d(),s(`div`,v,[...x[19]||=[a(`ul`,null,[a(`li`,null,[l(`They removed the RNN (recurrent neural network) as a test. Translation quality went `),a(`strong`,null,`up`),l(`.`)])],-1)]])),[[C]]),e((d(),s(`div`,y,[...x[20]||=[a(`p`,null,[a(`strong`,null,`Attention Is All You Need`),l(` (Vaswani et al., 2017): keep the weighted sum, delete everything it was bolted to.`)],-1)]])),[[C]]),a(`div`,b,[r(S,{source:`Vaswani, Shazeer, Parmar, Uszkoreit, Jones, Gomez, Kaiser & Polosukhin, Attention Is All You Need (2017)`,url:`https://arxiv.org/abs/1706.03762`})])]),_:1},16)}}};export{x as default};
|
lecture-06/assets/md-CWLf7t6o.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,V as r,W as i,_ as a,_t as o,b as s,m as c,v as l,vt as u,x as d,xt as f,yt as p,z as m}from"./modules/shiki-BFOesOFp.js";import{n as h,t as g}from"./slidev/context-DrkI8qiD.js";import{t as _}from"./default-sHmABwVd.js";var v={class:`mt-1 flex justify-center`},y={viewBox:`0 0 660 240`,class:`dl-diagram`,style:{"max-height":`12rem`},role:`img`,"aria-label":`The sentence the cat ate the fish because it was hungry. One softmax arc from it lands on cat; dashed lines mark three other questions: nearby noun, previous word, which clause`},b=[`x`],x=[`x`],S={class:`grid grid-cols-2 gap-8 mt-1`},C={class:`dl-callout`},w={__name:`slides.md__slidev_47`,setup(w){let{$slidev:T,$nav:E,$clicksContext:D,$clicks:O,$page:k,$renderContext:A,$frontmatter:j}=h();return D.setup(),(h,w)=>{let T=i(`click`);return m(),l(_,p(t(o(g)(o(j),46))),{default:n(()=>[w[5]||=a(`h1`,null,`One softmax can only point at one thing`,-1),w[6]||=a(`p`,null,[d(`To read `),a(`em`,null,`it`),d(`, the model needs several answers at once.`)],-1),a(`div`,v,[(m(),s(`svg`,y,[w[1]||=a(`path`,{class:`dl-dg-grad is-good`,d:`M476 128 Q296 20 116 128`,style:{"stroke-width":`4`}},null,-1),w[2]||=a(`text`,{class:`dl-dg-small is-good`,x:`296`,y:`54`,"text-anchor":`middle`},`one softmax: most weight on one word`,-1),(m(),s(c,null,r([`the`,`cat`,`ate`,`the`,`fish`,`because`,`it`,`was`,`hungry`],(e,t)=>a(`g`,{key:t},[a(`rect`,{class:u(t===6?`dl-dg-box is-accent`:`dl-dg-box`),x:12+t*72,y:`130`,width:`64`,height:`30`,rx:`4`},null,10,b),a(`text`,{class:`dl-dg-lab is-sm`,x:44+t*72,y:`150`,"text-anchor":`middle`},f(e),9,x)])),64)),e((m(),s(`g`,null,[...w[0]||=[a(`path`,{class:`dl-dg-loss`,d:`M116 162 V188`},null,-1),a(`text`,{class:`dl-dg-small`,x:`116`,y:`202`,"text-anchor":`middle`},`what it refers to`,-1),a(`path`,{class:`dl-dg-loss`,d:`M332 162 V188`},null,-1),a(`text`,{class:`dl-dg-small`,x:`332`,y:`202`,"text-anchor":`middle`},`nearby noun`,-1),a(`path`,{class:`dl-dg-loss`,d:`M404 162 V216`},null,-1),a(`text`,{class:`dl-dg-small`,x:`404`,y:`230`,"text-anchor":`middle`},`previous word`,-1),a(`path`,{class:`dl-dg-loss`,d:`M374 172 H634`},null,-1),a(`text`,{class:`dl-dg-small`,x:`560`,y:`190`,"text-anchor":`middle`},`which clause`,-1)]])),[[T]])]))]),a(`div`,S,[e((m(),s(`div`,C,[...w[3]||=[a(`p`,null,[d(`A softmax puts its weight in `),a(`strong`,null,`one`),d(` place. Four questions need four sets of weights.`)],-1)]])),[[T]]),e((m(),s(`div`,null,[...w[4]||=[a(`p`,null,[d(`So run attention `),a(`strong`,null,`several times side by side`),d(`, each copy with its own `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`msub`,null,[a(`mi`,null,`W`),a(`mi`,null,`Q`)])]),a(`annotation`,{encoding:`application/x-tex`},`W_Q`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.9694em`,"vertical-align":`-0.2861em`}}),a(`span`,{class:`mord`},[a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),a(`span`,{class:`msupsub`},[a(`span`,{class:`vlist-t vlist-t2`},[a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.3283em`}},[a(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[a(`span`,{class:`pstrut`,style:{height:`2.7em`}}),a(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[a(`span`,{class:`mord mathnormal mtight`},`Q`)])])]),a(`span`,{class:`vlist-s`},``)]),a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.2861em`}},[a(`span`)])])])])])])])]),d(`, `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`msub`,null,[a(`mi`,null,`W`),a(`mi`,null,`K`)])]),a(`annotation`,{encoding:`application/x-tex`},`W_K`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),a(`span`,{class:`mord`},[a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),a(`span`,{class:`msupsub`},[a(`span`,{class:`vlist-t vlist-t2`},[a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.3283em`}},[a(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[a(`span`,{class:`pstrut`,style:{height:`2.7em`}}),a(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[a(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0715em`}},`K`)])])]),a(`span`,{class:`vlist-s`},``)]),a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.15em`}},[a(`span`)])])])])])])])]),d(`, `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`msub`,null,[a(`mi`,null,`W`),a(`mi`,null,`V`)])]),a(`annotation`,{encoding:`application/x-tex`},`W_V`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),a(`span`,{class:`mord`},[a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),a(`span`,{class:`msupsub`},[a(`span`,{class:`vlist-t vlist-t2`},[a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.3283em`}},[a(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[a(`span`,{class:`pstrut`,style:{height:`2.7em`}}),a(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[a(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.2222em`}},`V`)])])]),a(`span`,{class:`vlist-s`},``)]),a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.15em`}},[a(`span`)])])])])])])])]),d(`. Each copy is a `),a(`strong`,null,`head`),d(`.`)],-1)]])),[[T]])])]),_:1},16)}}};export{w as default};
|
lecture-06/assets/md-CZ0TTTDs.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,m as l,v as u,x as d,xt as f,yt as p,z as m}from"./modules/shiki-BFOesOFp.js";import{n as h,t as g}from"./slidev/context-DrkI8qiD.js";import{t as _}from"./default-sHmABwVd.js";import{t as v}from"./slidev/VClicks-CGAOkBil.js";var y={class:`grid grid-cols-[17rem_1fr] gap-8 mt-1`},b={viewBox:`0 0 300 340`,class:`dl-diagram`,role:`img`,"aria-label":`The attention layer as a flow of six numbered steps: projections, scaled dot product, softmax, weighted sum, add and normalise, position-wise network`},x=[`cy`],S=[`y`],C={class:`dl-tight`},w={class:`mt-4 dl-callout`},T={__name:`slides.md__slidev_94`,setup(T){let{$slidev:E,$nav:D,$clicksContext:O,$clicks:k,$page:A,$renderContext:j,$frontmatter:M}=h();return O.setup(),(h,T)=>{let E=v,D=a(`click`);return m(),u(_,p(t(s(g)(s(M),93))),{default:n(()=>[T[24]||=o(`h1`,null,`What did not change`,-1),o(`div`,y,[o(`div`,null,[(m(),c(`svg`,b,[T[0]||=o(`defs`,null,[o(`marker`,{id:`l6s4-same`,markerWidth:`6`,markerHeight:`6`,refX:`5`,refY:`3`,orient:`auto`},[o(`path`,{class:`dl-dg-head`,d:`M0 0 L6 3 L0 6 z`})])],-1),T[1]||=o(`rect`,{class:`dl-dg-box`,x:`100`,y:`6`,width:`80`,height:`26`,rx:`4`},null,-1),T[2]||=o(`text`,{class:`dl-dg-lab is-sm`,x:`140`,y:`24`,"text-anchor":`middle`},`token x`,-1),T[3]||=o(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s4-same)`,d:`M140 32 L74 54 M140 32 V54 M140 32 L226 54`},null,-1),T[4]||=o(`rect`,{class:`dl-dg-fill`,x:`44`,y:`56`,width:`56`,height:`24`,rx:`4`},null,-1),T[5]||=o(`rect`,{class:`dl-dg-fill`,x:`122`,y:`56`,width:`56`,height:`24`,rx:`4`},null,-1),T[6]||=o(`rect`,{class:`dl-dg-fill`,x:`202`,y:`56`,width:`56`,height:`24`,rx:`4`},null,-1),T[7]||=o(`text`,{class:`dl-dg-in`,x:`72`,y:`73`,"text-anchor":`middle`},`q`,-1),T[8]||=o(`text`,{class:`dl-dg-in`,x:`150`,y:`73`,"text-anchor":`middle`},`k`,-1),T[9]||=o(`text`,{class:`dl-dg-in`,x:`230`,y:`73`,"text-anchor":`middle`},`v`,-1),T[10]||=o(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s4-same)`,d:`M72 80 L110 102 M150 80 V102 M230 80 V202 M140 130 V152 M140 178 V202 M140 230 V252 M140 280 V302`},null,-1),T[11]||=o(`rect`,{class:`dl-dg-op`,x:`60`,y:`104`,width:`160`,height:`26`,rx:`4`},null,-1),T[12]||=o(`text`,{class:`dl-dg-lab is-sm`,x:`140`,y:`122`,"text-anchor":`middle`},`q · k / √dₖ`,-1),T[13]||=o(`rect`,{class:`dl-dg-op`,x:`80`,y:`154`,width:`120`,height:`24`,rx:`4`},null,-1),T[14]||=o(`text`,{class:`dl-dg-lab is-sm`,x:`140`,y:`171`,"text-anchor":`middle`},`softmax`,-1),T[15]||=o(`rect`,{class:`dl-dg-op`,x:`50`,y:`204`,width:`200`,height:`26`,rx:`4`},null,-1),T[16]||=o(`text`,{class:`dl-dg-lab is-sm`,x:`140`,y:`222`,"text-anchor":`middle`},`Σ weight × v`,-1),T[17]||=o(`rect`,{class:`dl-dg-box`,x:`80`,y:`254`,width:`120`,height:`26`,rx:`4`},null,-1),T[18]||=o(`text`,{class:`dl-dg-lab is-sm`,x:`140`,y:`272`,"text-anchor":`middle`},`add & norm`,-1),T[19]||=o(`path`,{class:`dl-dg-grad is-good`,"marker-end":`url(#l6s4-same)`,d:`M180 19 H286 V267 H202`},null,-1),T[20]||=o(`rect`,{class:`dl-dg-net`,x:`50`,y:`304`,width:`180`,height:`30`,rx:`6`},null,-1),T[21]||=o(`text`,{class:`dl-dg-lab is-sm`,x:`140`,y:`324`,"text-anchor":`middle`},`position-wise network`,-1),(m(),c(l,null,i([[1,68],[2,117],[3,166],[4,217],[5,267],[6,319]],e=>o(`g`,{key:e[0]},[o(`circle`,{class:`dl-dg-fill`,cx:`16`,cy:e[1],r:`10`},null,8,x),o(`text`,{class:`dl-dg-in`,x:`16`,y:e[1]+4.5,"text-anchor":`middle`},f(e[0]),9,S)])),64))]))]),o(`div`,C,[r(E,null,{default:n(()=>[...T[22]||=[o(`ol`,null,[o(`li`,null,[o(`strong`,null,`Query, key, value`),d(`: three projections of every token`)]),o(`li`,null,[d(`A `),o(`strong`,null,`dot product`),d(` score, divided by `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`msqrt`,null,[o(`msub`,null,[o(`mi`,null,`d`),o(`mi`,null,`k`)])])]),o(`annotation`,{encoding:`application/x-tex`},`\\sqrt{d_k}`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`1.04em`,"vertical-align":`-0.1828em`}}),o(`span`,{class:`mord sqrt`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.8572em`}},[o(`span`,{class:`svg-align`,style:{top:`-3em`}},[o(`span`,{class:`pstrut`,style:{height:`3em`}}),o(`span`,{class:`mord`,style:{"padding-left":`0.833em`}},[o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`},`d`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3361em`}},[o(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])])])]),o(`span`,{style:{top:`-2.8172em`}},[o(`span`,{class:`pstrut`,style:{height:`3em`}}),o(`span`,{class:`hide-tail`,style:{"min-width":`0.853em`,height:`1.08em`}},[o(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`1.08em`,viewBox:`0 0 400000 1080`,preserveAspectRatio:`xMinYMin slice`},[o(`path`,{d:`M95,702
|
| 2 |
+
c-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14
|
| 3 |
+
c0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54
|
| 4 |
+
c44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10
|
| 5 |
+
s173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429
|
| 6 |
+
c69,-144,104.5,-217.7,106.5,-221
|
| 7 |
+
l0 -0
|
| 8 |
+
c5.3,-9.3,12,-14,20,-14
|
| 9 |
+
H400000v40H845.2724
|
| 10 |
+
s-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7
|
| 11 |
+
c-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z
|
| 12 |
+
M834 80h400000v40h-400000z`})])])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.1828em`}},[o(`span`)])])])])])])])]),o(`li`,null,[d(`A `),o(`strong`,null,`softmax`),d(` over positions: positive weights that sum to 1`)]),o(`li`,null,[d(`A `),o(`strong`,null,`weighted sum of values`),d(` as the output`)]),o(`li`,null,[o(`strong`,null,`Residual`),d(` and `),o(`strong`,null,`normalise`),d(` around every sub-layer`)]),o(`li`,null,[d(`A `),o(`strong`,null,`position-wise network`),d(` holding most of the parameters`)])],-1)]]),_:1}),e((m(),c(`div`,w,[...T[23]||=[o(`p`,null,`A 2017 attention implementation still runs, unchanged, inside a model trained this year.`,-1)]])),[[D]])])])]),_:1},16)}}};export{T as default};
|
lecture-06/assets/md-Cbf_c7qC.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,x as l,yt as u,z as d}from"./modules/shiki-BFOesOFp.js";import{n as f,t as p}from"./slidev/context-DrkI8qiD.js";import{t as m}from"./default-sHmABwVd.js";import{t as h}from"./slidev/KaTexBlockWrapper-BKHEVd16.js";var g={class:`mt-1`},_={class:`flex justify-center`},v={viewBox:`0 0 520 124`,class:`dl-diagram`,style:{"max-height":`8.6rem`},role:`img`,"aria-label":`Q, T by d_k, times K transpose, d_k by T, gives a T by T grid of scores; divide and softmax gives T by T weights; times V, T by d_v, gives the T by d_v output`},y={class:`mt-1 dl-callout`},b={__name:`slides.md__slidev_42`,setup(b){let{$slidev:x,$nav:S,$clicksContext:C,$clicks:w,$page:T,$renderContext:E,$frontmatter:D}=f();return C.setup(),(f,b)=>{let x=h,S=i(`click`);return d(),c(m,u(t(o(p)(o(D),41))),{default:n(()=>[b[6]||=a(`h1`,null,`The whole layer, in one line`,-1),b[7]||=a(`p`,null,[l(`Stack the queries into `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`mi`,null,`Q`)]),a(`annotation`,{encoding:`application/x-tex`},`Q`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.8778em`,"vertical-align":`-0.1944em`}}),a(`span`,{class:`mord mathnormal`},`Q`)])])]),l(`, the keys into `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`mi`,null,`K`)]),a(`annotation`,{encoding:`application/x-tex`},`K`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0715em`}},`K`)])])]),l(` and the values into `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`mi`,null,`V`)]),a(`annotation`,{encoding:`application/x-tex`},`V`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.2222em`}},`V`)])])]),l(`, one row per token. `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`mi`,null,`T`)]),a(`annotation`,{encoding:`application/x-tex`},`T`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`)])])]),l(` is the number of tokens. The four steps become one line:`)],-1),a(`div`,g,[r(x,{ranges:[]},{default:n(()=>[...b[0]||=[a(`p`,null,[a(`span`,{class:`katex-display`},[a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`,display:`block`},[a(`semantics`,null,[a(`mrow`,null,[a(`mi`,{mathvariant:`normal`},`Attention`),a(`mo`,null,``),a(`mo`,{stretchy:`false`},`(`),a(`mi`,null,`Q`),a(`mo`,{separator:`true`},`,`),a(`mi`,null,`K`),a(`mo`,{separator:`true`},`,`),a(`mi`,null,`V`),a(`mo`,{stretchy:`false`},`)`),a(`mo`,null,`=`),a(`mi`,{mathvariant:`normal`},`softmax`),a(`mo`,null,``),a(`mtext`,null,` `),a(`mrow`,null,[a(`mo`,{fence:`true`},`(`),a(`mfrac`,null,[a(`mrow`,null,[a(`mi`,null,`Q`),a(`msup`,null,[a(`mi`,null,`K`),a(`mi`,{mathvariant:`normal`},`⊤`)])]),a(`msqrt`,null,[a(`msub`,null,[a(`mi`,null,`d`),a(`mi`,null,`k`)])])]),a(`mo`,{fence:`true`},`)`)]),a(`mi`,null,`V`)]),a(`annotation`,{encoding:`application/x-tex`},`\\operatorname{Attention}(Q, K, V) = \\operatorname{softmax}\\!\\left(\\frac{QK^\\top}{\\sqrt{d_k}}\\right)V`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),a(`span`,{class:`mop`},[a(`span`,{class:`mord mathrm`},`Attention`)]),a(`span`,{class:`mopen`},`(`),a(`span`,{class:`mord mathnormal`},`Q`),a(`span`,{class:`mpunct`},`,`),a(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0715em`}},`K`),a(`span`,{class:`mpunct`},`,`),a(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.2222em`}},`V`),a(`span`,{class:`mclose`},`)`),a(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),a(`span`,{class:`mrel`},`=`),a(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`2.4761em`,"vertical-align":`-0.95em`}}),a(`span`,{class:`mop`},[a(`span`,{class:`mord mathrm`},`softmax`)]),a(`span`,{class:`mspace`,style:{"margin-right":`-0.1667em`}}),a(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),a(`span`,{class:`minner`},[a(`span`,{class:`mopen delimcenter`,style:{top:`0em`}},[a(`span`,{class:`delimsizing size3`},`(`)]),a(`span`,{class:`mord`},[a(`span`,{class:`mopen nulldelimiter`}),a(`span`,{class:`mfrac`},[a(`span`,{class:`vlist-t vlist-t2`},[a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`1.5261em`}},[a(`span`,{style:{top:`-2.2528em`}},[a(`span`,{class:`pstrut`,style:{height:`3em`}}),a(`span`,{class:`mord`},[a(`span`,{class:`mord sqrt`},[a(`span`,{class:`vlist-t vlist-t2`},[a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.8572em`}},[a(`span`,{class:`svg-align`,style:{top:`-3em`}},[a(`span`,{class:`pstrut`,style:{height:`3em`}}),a(`span`,{class:`mord`,style:{"padding-left":`0.833em`}},[a(`span`,{class:`mord`},[a(`span`,{class:`mord mathnormal`},`d`),a(`span`,{class:`msupsub`},[a(`span`,{class:`vlist-t vlist-t2`},[a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.3361em`}},[a(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[a(`span`,{class:`pstrut`,style:{height:`2.7em`}}),a(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[a(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),a(`span`,{class:`vlist-s`},``)]),a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.15em`}},[a(`span`)])])])])])])]),a(`span`,{style:{top:`-2.8172em`}},[a(`span`,{class:`pstrut`,style:{height:`3em`}}),a(`span`,{class:`hide-tail`,style:{"min-width":`0.853em`,height:`1.08em`}},[a(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`1.08em`,viewBox:`0 0 400000 1080`,preserveAspectRatio:`xMinYMin slice`},[a(`path`,{d:`M95,702
|
| 2 |
+
c-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14
|
| 3 |
+
c0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54
|
| 4 |
+
c44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10
|
| 5 |
+
s173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429
|
| 6 |
+
c69,-144,104.5,-217.7,106.5,-221
|
| 7 |
+
l0 -0
|
| 8 |
+
c5.3,-9.3,12,-14,20,-14
|
| 9 |
+
H400000v40H845.2724
|
| 10 |
+
s-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7
|
| 11 |
+
c-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z
|
| 12 |
+
M834 80h400000v40h-400000z`})])])])]),a(`span`,{class:`vlist-s`},``)]),a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.1828em`}},[a(`span`)])])])])])]),a(`span`,{style:{top:`-3.23em`}},[a(`span`,{class:`pstrut`,style:{height:`3em`}}),a(`span`,{class:`frac-line`,style:{"border-bottom-width":`0.04em`}})]),a(`span`,{style:{top:`-3.677em`}},[a(`span`,{class:`pstrut`,style:{height:`3em`}}),a(`span`,{class:`mord`},[a(`span`,{class:`mord mathnormal`},`Q`),a(`span`,{class:`mord`},[a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0715em`}},`K`),a(`span`,{class:`msupsub`},[a(`span`,{class:`vlist-t`},[a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.8491em`}},[a(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[a(`span`,{class:`pstrut`,style:{height:`2.7em`}}),a(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[a(`span`,{class:`mord mtight`},`⊤`)])])])])])])])])])]),a(`span`,{class:`vlist-s`},``)]),a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.93em`}},[a(`span`)])])])]),a(`span`,{class:`mclose nulldelimiter`})]),a(`span`,{class:`mclose delimcenter`,style:{top:`0em`}},[a(`span`,{class:`delimsizing size3`},`)`)])]),a(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.2222em`}},`V`)])])])])],-1)]]),_:1})]),a(`div`,_,[(d(),s(`svg`,v,[b[4]||=a(`defs`,null,[a(`marker`,{id:`l6s1-layer`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[a(`path`,{class:`dl-dg-head`,d:`M0 0 L7 3.5 L0 7 z`})])],-1),e((d(),s(`g`,null,[...b[1]||=[a(`rect`,{class:`dl-dg-box`,x:`10`,y:`18`,width:`24`,height:`72`,rx:`2`},null,-1),a(`text`,{class:`dl-dg-lab is-sm`,x:`22`,y:`12`,"text-anchor":`middle`},`Q`,-1),a(`text`,{class:`dl-dg-small`,x:`22`,y:`108`,"text-anchor":`middle`},`T × d_k`,-1),a(`text`,{class:`dl-dg-lab is-sm`,x:`48`,y:`58`,"text-anchor":`middle`},`×`,-1),a(`rect`,{class:`dl-dg-box`,x:`62`,y:`18`,width:`72`,height:`24`,rx:`2`},null,-1),a(`text`,{class:`dl-dg-lab is-sm`,x:`98`,y:`12`,"text-anchor":`middle`},`Kᵀ`,-1),a(`text`,{class:`dl-dg-small`,x:`98`,y:`58`,"text-anchor":`middle`},`d_k × T`,-1),a(`text`,{class:`dl-dg-lab is-sm`,x:`148`,y:`58`,"text-anchor":`middle`},`=`,-1),a(`rect`,{class:`dl-dg-box`,x:`162`,y:`18`,width:`72`,height:`72`,rx:`2`},null,-1),a(`text`,{class:`dl-dg-lab is-sm`,x:`198`,y:`12`,"text-anchor":`middle`},`1 · scores`,-1),a(`text`,{class:`dl-dg-small`,x:`198`,y:`108`,"text-anchor":`middle`},`T × T`,-1)]])),[[S]]),e((d(),s(`g`,null,[...b[2]||=[a(`path`,{class:`dl-dg-arrow`,d:`M236 54 H282`,"marker-end":`url(#l6s1-layer)`},null,-1),a(`text`,{class:`dl-dg-small`,x:`259`,y:`44`,"text-anchor":`middle`},`2 · ÷ √d_k`,-1),a(`text`,{class:`dl-dg-small`,x:`259`,y:`72`,"text-anchor":`middle`},`3 · softmax`,-1),a(`rect`,{class:`dl-dg-box is-accent`,x:`288`,y:`18`,width:`72`,height:`72`,rx:`2`},null,-1),a(`text`,{class:`dl-dg-lab is-sm`,x:`324`,y:`12`,"text-anchor":`middle`},`weights`,-1),a(`text`,{class:`dl-dg-small`,x:`324`,y:`108`,"text-anchor":`middle`},`T × T`,-1),a(`text`,{class:`dl-dg-small`,x:`324`,y:`121`,"text-anchor":`middle`},`rows sum to 1`,-1)]])),[[S]]),e((d(),s(`g`,null,[...b[3]||=[a(`text`,{class:`dl-dg-lab is-sm`,x:`374`,y:`58`,"text-anchor":`middle`},`×`,-1),a(`rect`,{class:`dl-dg-box`,x:`388`,y:`18`,width:`24`,height:`72`,rx:`2`},null,-1),a(`text`,{class:`dl-dg-lab is-sm`,x:`400`,y:`12`,"text-anchor":`middle`},`V`,-1),a(`text`,{class:`dl-dg-small`,x:`400`,y:`108`,"text-anchor":`middle`},`T × d_v`,-1),a(`text`,{class:`dl-dg-lab is-sm`,x:`430`,y:`58`,"text-anchor":`middle`},`=`,-1),a(`rect`,{class:`dl-dg-box is-accent`,x:`452`,y:`18`,width:`24`,height:`72`,rx:`2`},null,-1),a(`text`,{class:`dl-dg-lab is-sm`,x:`464`,y:`12`,"text-anchor":`middle`},`4 · output`,-1),a(`text`,{class:`dl-dg-small`,x:`464`,y:`108`,"text-anchor":`middle`},`T × d_v`,-1)]])),[[S]])]))]),e((d(),s(`div`,y,[...b[5]||=[a(`p`,null,`Two matrix multiplies and a softmax. Every transformer is this line.`,-1)]])),[[S]])]),_:1},16)}}};export{b as default};
|