Release site-v1.0.5 (part 7)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- lecture-06/class/assets/md-BLhYOnc7.js +1 -0
- lecture-06/class/assets/md-BMxY1hjv.js +1 -0
- lecture-06/class/assets/md-BTnUhx7N.css +1 -0
- lecture-06/class/assets/md-BYyZKhcj.js +1 -0
- lecture-06/class/assets/md-BZqsqFo42.js +1 -0
- lecture-06/class/assets/md-Bas86k082.js +3 -0
- lecture-06/class/assets/md-BimdYeK3.js +1 -0
- lecture-06/class/assets/md-BmvwA0Nf.js +1 -0
- lecture-06/class/assets/md-BnAqCEYv2.js +1 -0
- lecture-06/class/assets/md-Bq0k6Ht_.js +1 -0
- lecture-06/class/assets/md-BsO5ZW2F.js +1 -0
- lecture-06/class/assets/md-BtB6vj0a2.js +1 -0
- lecture-06/class/assets/md-BxdQ0RGt.js +1 -0
- lecture-06/class/assets/md-C3JZnPAz.js +1 -0
- lecture-06/class/assets/md-C4a87Mz_.js +1 -0
- lecture-06/class/assets/md-C9_5NLsW.js +0 -0
- lecture-06/class/assets/md-CJukBVkv.js +1 -0
- lecture-06/class/assets/md-CNfnuG9N2.js +19 -0
- lecture-06/class/assets/md-Cdeogdwt.js +1 -0
- lecture-06/class/assets/md-Cuhu_vvG2.js +1 -0
- lecture-06/class/assets/md-CwNbuyVx2.js +1 -0
- lecture-06/class/assets/md-Cy5aODuI.js +2 -0
- lecture-06/class/assets/md-Cyf0zl3n.js +1 -0
- lecture-06/class/assets/md-D-S0zhLV.js +1 -0
- lecture-06/class/assets/md-D0ZW5T8G.js +1 -0
- lecture-06/class/assets/md-D3MJsr7m.js +3 -0
- lecture-06/class/assets/md-D4Ywsi6K.js +1 -0
- lecture-06/class/assets/md-DAJ_ecr1.js +1 -0
- lecture-06/class/assets/md-DCF9sEzr2.js +1 -0
- lecture-06/class/assets/md-DOad-2KZ.js +1 -0
- lecture-06/class/assets/md-DRRRjQ4b.js +1 -0
- lecture-06/class/assets/md-DSyIwSi7.js +1 -0
- lecture-06/class/assets/md-D_ZyssbS2.js +12 -0
- lecture-06/class/assets/md-DgIoPoNt.js +1 -0
- lecture-06/class/assets/md-DhsPSwrk.css +1 -0
- lecture-06/class/assets/md-Dnd6WsWa2.js +9 -0
- lecture-06/class/assets/md-DnwIl7Jl.js +1 -0
- lecture-06/class/assets/md-Dw-XEgB42.js +1 -0
- lecture-06/class/assets/md-K0Oycy6G.js +1 -0
- lecture-06/class/assets/md-KdnLugHx2.js +1 -0
- lecture-06/class/assets/md-kE1uizXw.js +1 -0
- lecture-06/class/assets/md-m6ElLsWQ.js +1 -0
- lecture-06/class/assets/slidev/404-Dkk3G2sr.js +1 -0
- lecture-06/class/assets/slidev/CodeBlockWrapper-gsL5r1Ig.js +1 -0
- lecture-06/class/assets/slidev/DrawingControls-cUpksemn.js +1 -0
- lecture-06/class/assets/slidev/DrawingLayer-BLLWTnj4.js +1 -0
- lecture-06/class/assets/slidev/KaTexBlockWrapper-BIlBldVC.js +1 -0
- lecture-06/class/assets/slidev/NoteDisplay-6LSM8xpP.js +1 -0
- lecture-06/class/assets/slidev/NoteEditable-DTwQcEqa.js +1 -0
- lecture-06/class/assets/slidev/TimerBar-1xtsicrJ.js +1 -0
lecture-06/class/assets/md-BLhYOnc7.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,W as r,_ as i,_t as a,b as o,v as s,x as c,yt as l,z as u}from"./modules/shiki-BFOesOFp.js";import{n as d,t as f}from"./slidev/context-DrkI8qiD.js";import{t as p}from"./default-Cq9fupKf.js";var m={class:`grid grid-cols-[1.3fr_1fr] gap-8 mt-2 items-start`},h={viewBox:`0 0 420 184`,class:`dl-diagram`,style:{"max-height":`14rem`},role:`img`,"aria-label":`The three words of the river bank, each with a bar chart of its four embedding numbers: the is 1 0 0 0, river is 0 2 0 1, bank is 0 0 0 2`},g={class:`dl-tight`},_={class:`mt-3 dl-callout`},v={class:`mt-3`},y={class:`mt-3 dl-secondary`},b={__name:`class.md__slidev_15`,setup(b){let{$slidev:x,$nav:S,$clicksContext:C,$clicks:w,$page:T,$renderContext:E,$frontmatter:D}=d();return C.setup(),(d,b)=>{let x=r(`click`);return u(),s(p,l(t(a(f)(a(D),14))),{default:n(()=>[b[27]||=i(`h1`,null,`The sentence we will use all lecture`,-1),i(`div`,m,[i(`div`,null,[(u(),o(`svg`,h,[b[2]||=i(`rect`,{class:`dl-dg-box`,x:`10`,y:`8`,width:`112`,height:`28`,rx:`4`},null,-1),b[3]||=i(`rect`,{class:`dl-dg-box`,x:`150`,y:`8`,width:`112`,height:`28`,rx:`4`},null,-1),b[4]||=i(`rect`,{class:`dl-dg-box is-accent`,x:`290`,y:`8`,width:`112`,height:`28`,rx:`4`},null,-1),b[5]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`66`,y:`27`,"text-anchor":`middle`,style:{"font-style":`italic`}},`the`,-1),b[6]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`206`,y:`27`,"text-anchor":`middle`,style:{"font-style":`italic`}},`river`,-1),b[7]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`346`,y:`27`,"text-anchor":`middle`,style:{"font-style":`italic`}},`bank`,-1),b[8]||=i(`line`,{class:`dl-dg-split`,x1:`10`,y1:`150`,x2:`122`,y2:`150`},null,-1),b[9]||=i(`line`,{class:`dl-dg-split`,x1:`150`,y1:`150`,x2:`262`,y2:`150`},null,-1),b[10]||=i(`line`,{class:`dl-dg-split`,x1:`290`,y1:`150`,x2:`402`,y2:`150`},null,-1),b[11]||=i(`text`,{class:`dl-dg-small`,x:`24`,y:`166`,"text-anchor":`middle`},`det`,-1),b[12]||=i(`text`,{class:`dl-dg-small`,x:`52`,y:`180`,"text-anchor":`middle`},`water`,-1),b[13]||=i(`text`,{class:`dl-dg-small`,x:`80`,y:`166`,"text-anchor":`middle`},`money`,-1),b[14]||=i(`text`,{class:`dl-dg-small`,x:`108`,y:`180`,"text-anchor":`middle`},`thing`,-1),b[15]||=i(`text`,{class:`dl-dg-small`,x:`164`,y:`166`,"text-anchor":`middle`},`det`,-1),b[16]||=i(`text`,{class:`dl-dg-small`,x:`192`,y:`180`,"text-anchor":`middle`},`water`,-1),b[17]||=i(`text`,{class:`dl-dg-small`,x:`220`,y:`166`,"text-anchor":`middle`},`money`,-1),b[18]||=i(`text`,{class:`dl-dg-small`,x:`248`,y:`180`,"text-anchor":`middle`},`thing`,-1),b[19]||=i(`text`,{class:`dl-dg-small`,x:`304`,y:`166`,"text-anchor":`middle`},`det`,-1),b[20]||=i(`text`,{class:`dl-dg-small`,x:`332`,y:`180`,"text-anchor":`middle`},`water`,-1),b[21]||=i(`text`,{class:`dl-dg-small`,x:`360`,y:`166`,"text-anchor":`middle`},`money`,-1),b[22]||=i(`text`,{class:`dl-dg-small`,x:`388`,y:`180`,"text-anchor":`middle`},`thing`,-1),e((u(),o(`g`,null,[...b[0]||=[i(`rect`,{class:`dl-dg-bar`,x:`14`,y:`110`,width:`20`,height:`40`},null,-1),i(`text`,{class:`dl-dg-small`,x:`24`,y:`104`,"text-anchor":`middle`},`1`,-1),i(`rect`,{class:`dl-dg-bar`,x:`182`,y:`70`,width:`20`,height:`80`},null,-1),i(`text`,{class:`dl-dg-small`,x:`192`,y:`64`,"text-anchor":`middle`},`2`,-1),i(`rect`,{class:`dl-dg-bar`,x:`238`,y:`110`,width:`20`,height:`40`},null,-1),i(`text`,{class:`dl-dg-small`,x:`248`,y:`104`,"text-anchor":`middle`},`1`,-1)]])),[[x]]),e((u(),o(`g`,null,[...b[1]||=[i(`rect`,{class:`dl-dg-bar is-q`,x:`378`,y:`70`,width:`20`,height:`80`},null,-1),i(`text`,{class:`dl-dg-small`,x:`388`,y:`64`,"text-anchor":`middle`},`2`,-1)]])),[[x]])]))]),i(`div`,g,[b[25]||=i(`p`,null,[c(`Four embedding numbers per word. We pretend they mean `),i(`strong`,null,`det`),c(` (a word like `),i(`em`,null,`the`),c(`), `),i(`strong`,null,`water`),c(`, `),i(`strong`,null,`money`),c(` and `),i(`strong`,null,`thing`),c(`.`)],-1),e((u(),o(`div`,_,[...b[23]||=[i(`p`,null,[i(`em`,null,`bank`),c(` is a thing, and `),i(`strong`,null,`nothing else`),c(`. Nothing says riverbank or high-street bank.`)],-1)]])),[[x]]),e((u(),o(`div`,v,[...b[24]||=[i(`p`,null,`So the sense has to come from a neighbour. Attention is how it gets there.`,-1)]])),[[x]])])]),e((u(),o(`div`,y,[...b[26]||=[i(`p`,null,`Hand-picked numbers, so you can check the arithmetic. Real coordinates have no names.`,-1)]])),[[x]])]),_:1},16)}}};export{b as default};
|
lecture-06/class/assets/md-BMxY1hjv.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,x as l,yt as u,z as d}from"./modules/shiki-BFOesOFp.js";import{n as f,t as p}from"./slidev/context-DrkI8qiD.js";import{t as m}from"./slidev/VClicks-Dmx1KjZv.js";import{t as h}from"./interactive-CI8hm4xw.js";import{t as g}from"./PositionalEncoding-CZmL_mXL.js";var _={class:`mt-2 dl-callout`},v={__name:`class.md__slidev_24`,setup(v){let{$slidev:y,$nav:b,$clicksContext:x,$clicks:S,$page:C,$renderContext:w,$frontmatter:T}=f();return x.setup(),(f,v)=>{let y=g,b=m,x=i(`click`);return d(),c(h,u(t(o(p)(o(T),23))),{aside:n(t=>[v[2]||=a(`p`,null,[l(`Press `),a(`strong`,null,`Shuffle the words`),l(`, then read the vectors.`)],-1),r(b,null,{default:n(()=>[...v[0]||=[a(`ul`,null,[a(`li`,null,[l(`Every word’s output is `),a(`strong`,null,`exactly`),l(` what it was`)]),a(`li`,null,[l(`A weighted sum is a sum over a `),a(`strong`,null,`set`),l(`. A set has no order`)]),a(`li`,null,[a(`em`,null,`the river bank`),l(` and `),a(`em`,null,`bank river the`),l(` look the same`)])],-1)]]),_:1}),e((d(),s(`div`,_,[...v[1]||=[a(`p`,null,`A recurrent network knew the order, because it read in order. We have just thrown that away.`,-1)]])),[[x]])]),default:n(()=>[r(y,{mode:`blind`})]),_:1},16)}}};export{v as default};
|
lecture-06/class/assets/md-BTnUhx7N.css
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
.dl-stack[data-v-66f48009]{width:100%;height:100%}.dl-stack__spine[data-v-66f48009]{stroke:var(--dl-border);stroke-width:2px;color:var(--dl-border)}.dl-stack__box[data-v-66f48009]{fill:var(--dl-bg);stroke:var(--dl-border);stroke-width:1.3px;transition:fill .2s,stroke .2s}.dl-stack__box.is-seen[data-v-66f48009]{fill:var(--dl-surface);stroke:var(--dl-body)}.dl-stack__box.is-current[data-v-66f48009]{fill:var(--dl-accent-soft);stroke:var(--dl-accent);stroke-width:2.2px}.dl-stack__label[data-v-66f48009]{fill:var(--dl-muted);font-size:11.5px;font-weight:600;transition:fill .2s}.dl-stack__label.is-seen[data-v-66f48009],.dl-stack__label.is-current[data-v-66f48009]{fill:var(--dl-heading)}.dl-stack__sub[data-v-66f48009]{fill:var(--dl-muted);font-size:8.8px}.dl-stack__sub.is-current[data-v-66f48009]{fill:var(--dl-accent-strong)}.dl-stack__repeat[data-v-66f48009]{fill:none;stroke:var(--dl-border);stroke-width:1.2px;stroke-dasharray:5 4;transition:stroke .2s}.dl-stack__repeat.is-current[data-v-66f48009]{stroke:var(--dl-accent)}.dl-stack__repeatlabel[data-v-66f48009]{fill:var(--dl-muted);font-size:11px;font-weight:700}.dl-stack__repeatlabel.is-current[data-v-66f48009]{fill:var(--dl-accent)}.dl-stack__cross[data-v-66f48009]{fill:none;stroke:var(--dl-border);stroke-width:1.6px;color:var(--dl-border);transition:stroke .2s}.dl-stack__cross.is-seen[data-v-66f48009]{stroke:var(--dl-body);color:var(--dl-body)}.dl-stack__cross.is-current[data-v-66f48009]{stroke:var(--dl-accent);color:var(--dl-accent);stroke-width:2.2px}.dl-stack__crosslabel[data-v-66f48009]{fill:var(--dl-muted);opacity:0;font-size:9px;transition:opacity .2s}.dl-stack__crosslabel.is-seen[data-v-66f48009],.dl-stack__crosslabel.is-current[data-v-66f48009]{opacity:1}.dl-stack__crosslabel.is-current[data-v-66f48009]{fill:var(--dl-accent);font-weight:700}.dl-stack__add[data-v-66f48009]{fill:var(--dl-bg);stroke:var(--dl-border);stroke-width:1.4px}.dl-stack__add.is-seen[data-v-66f48009]{stroke:var(--dl-body)}.dl-stack__add.is-current[data-v-66f48009]{fill:var(--dl-accent-soft);stroke:var(--dl-accent);stroke-width:2.2px}.dl-stack__plus[data-v-66f48009]{fill:var(--dl-heading);font-size:12px;font-weight:700}.dl-stack__pe[data-v-66f48009]{fill:var(--dl-muted);font-size:9px}.dl-stack__pe.is-current[data-v-66f48009]{fill:var(--dl-accent);font-weight:700}.dl-stack__port[data-v-66f48009]{fill:var(--dl-muted);font-size:10px;transition:fill .2s}.dl-stack__port.is-current[data-v-66f48009]{fill:var(--dl-accent);font-weight:700}.dl-stack__side[data-v-66f48009]{fill:var(--dl-muted);letter-spacing:.09em;font-size:9.5px}
|
lecture-06/class/assets/md-BYyZKhcj.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,x as l,yt as u,z as d}from"./modules/shiki-BFOesOFp.js";import{n as f,t as p}from"./slidev/context-DrkI8qiD.js";import{t as m}from"./slidev/VClicks-Dmx1KjZv.js";import{t as h}from"./interactive-CI8hm4xw.js";import{t as g}from"./PositionalEncoding-CZmL_mXL.js";var _={class:`mt-2 dl-callout`},v={__name:`class.md__slidev_25`,setup(v){let{$slidev:y,$nav:b,$clicksContext:x,$clicks:S,$page:C,$renderContext:w,$frontmatter:T}=f();return x.setup(),(f,v)=>{let y=g,b=m,x=i(`click`);return d(),c(h,u(t(o(p)(o(T),24))),{aside:n(t=>[v[2]||=a(`p`,null,[l(`One fixed vector per position, `),a(`strong`,null,`added`),l(` to the embedding before block one.`)],-1),r(b,null,{default:n(()=>[...v[0]||=[a(`ul`,null,[a(`li`,null,`Left dimensions flip fast; right ones barely move`),a(`li`,null,[l(`So one vector carries `),a(`em`,null,`which word`),l(` and `),a(`em`,null,`which end`),l(` at once`)]),a(`li`,null,[l(`Sines and cosines. `),a(`strong`,null,`Nothing learned`),l(`, no maximum length`)])],-1)]]),_:1}),e((d(),s(`div`,_,[...v[1]||=[a(`p`,null,`Added, not concatenated — the model never gets wider.`,-1)]])),[[x]])]),default:n(()=>[r(y,{mode:`sinusoid`,"d-model":16,positions:12})]),_:1},16)}}};export{v as default};
|
lecture-06/class/assets/md-BZqsqFo42.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`class.md__slidev_41`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),40))),{default:t(()=>[p[0]||=r(`h1`,null,`Where that leaves us`,-1),p[1]||=r(`p`,null,`One idea, applied twice per block.`,-1),r(`div`,f,[n(m,{kind:`block`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/class/assets/md-Bas86k082.js
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,g as l,lt as u,m as d,v as f,vt as p,w as m,x as h,xt as g,y as _,yt as v,z as y}from"./modules/shiki-BFOesOFp.js";import{t as ee}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as b,t as x}from"./slidev/context-DrkI8qiD.js";import{t as S}from"./slidev/VClicks-Dmx1KjZv.js";import{n as C,t as w}from"./interactive-CI8hm4xw.js";import{t as T}from"./StepButton-BcP33F-1.js";var E=[`viewBox`,`aria-label`],D=[`x1`,`y1`,`x2`,`y2`],te=[`x`,`y`],ne=[`x`,`y`],re=[`d`],ie={key:0},ae=[`cx`,`cy`],oe=[`x`,`y`],se={key:1},O=[`y`],k=[`x`,`y`],A=[`x`,`y`],j=[`x`,`y`,`transform`],M=440,N=48,P=244,F=132,I=34,L=12,R=26,z=72,B=ee(m({__name:`TransformerBlock`,props:{variant:{default:`post`},switchable:{type:Boolean,default:!0},start:{default:`walk`}},setup(e){let{$slidev:t,$nav:a,$clicksContext:m,$clicks:v,$page:ee,$renderContext:x,$frontmatter:S}=b(),w=e,B=u(w.variant),V=u(0),H=u(w.start===`whole`),U=[{kind:`box`,label:`Multi-head self-attention`,sub:`positions mix here — and only here`,note:`Every position builds a new vector as a weighted sum over every position. This is the only part of the whole architecture where information moves sideways between tokens.`,sublayer:!0},{kind:`add`,label:`add`,note:`The residual connection: the sub-layer’s output is *added* to what came in, not substituted for it. Same trick as a ResNet, and for the same reason — it gives the gradient a path with no matrix on it.`},{kind:`box`,label:`LayerNorm`,sub:`per position, across features`,note:`Re-centre and re-scale each position’s vector using its own mean and variance. Not BatchNorm: nothing is shared across the batch or across positions, which is what makes it safe with variable-length sequences.`},{kind:`box`,label:`Feed-forward network`,sub:`each position alone — 4× wide`,note:`Two linear layers with a non-linearity between them, applied to every position separately with the same weights. No token sees another here. This is where two thirds of a block’s parameters live.`,sublayer:!0},{kind:`add`,label:`add`,note:`The second residual, around the feed-forward network.`},{kind:`box`,label:`LayerNorm`,sub:`per position, across features`,note:`And normalise again. In the 2017 wiring both norms sit *after* the add — which means they sit on the residual path itself.`}],W=[{kind:`box`,label:`LayerNorm`,sub:`RMSNorm in most modern models`,note:`Modern blocks normalise *before* the sub-layer instead of after it. RMSNorm drops the mean-subtraction and the bias and keeps only the rescaling — slightly cheaper, and empirically just as good.`},{kind:`box`,label:`Multi-head self-attention`,sub:`grouped-query, rotary positions`,note:`Still scaled dot-product attention, unchanged since 2017. What changed around it: position now enters by rotating Q and K (RoPE) rather than by adding a vector at the bottom, and several query heads share one key/value head to shrink the cache.`,sublayer:!0},{kind:`add`,label:`add`,note:`The residual — and note what is *not* on it any more. Nothing stands between the input and this add but the sub-layer’s own output.`},{kind:`box`,label:`LayerNorm`,sub:`RMSNorm`,note:`Normalise before the feed-forward network, on a copy of the residual stream rather than on the stream itself.`},{kind:`box`,label:`Feed-forward network`,sub:`SwiGLU — three matrices, not two`,note:`Still position-wise, still wide. The gated variant uses three matrices instead of two, and the hidden width is trimmed (about 8/3 d rather than 4 d) to keep the parameter count level.`,sublayer:!0},{kind:`add`,label:`add`,note:`The second residual. From input to output there is now an unbroken addition-only path through every block in the stack.`}],G=l(()=>B.value===`post`?U:W),ce=l(()=>B.value===`post`?[{from:-1,to:1},{from:2,to:4}]:[{from:-1,to:2},{from:2,to:5}]),K=l(()=>V.value>=G.value.length);function le(){H.value=!1,V.value=K.value?0:V.value+1}function ue(){B.value=B.value===`post`?`pre`:`post`,V.value=0,H.value=w.start===`whole`}function q(e){return H.value||V.value>=e+1}let J=l(()=>V.value===0?null:G.value[V.value-1]);function Y(e){return R+(G.value.length-e)*N}let X=l(()=>Y(-1)+6),Z=R,Q=l(()=>Y(-1)+36);function $(e){return e<0?X.value:Y(e)+(G.value[e].kind===`add`?0:I/2)}return(e,t)=>{let a=T,l=C;return y(),f(l,{"max-width":`30rem`},{controls:n(()=>[r(a,{label:K.value&&!H.value?`Start over`:V.value===0?`Walk the block`:`Next part`,glyph:`▸`,onClick:le},null,8,[`label`]),w.switchable?(y(),f(a,{key:0,label:B.value===`post`?`Rewire: modern (pre-LN)`:`Rewire: 2017 (post-LN)`,variant:`ghost`,glyph:`⇄`,onClick:ue},null,8,[`label`])):_(`v-if`,!0)]),readout:n(()=>[J.value?(y(),c(d,{key:0},[o(`strong`,null,g(J.value.label),1),h(` — `+g(J.value.note),1)],64)):B.value===`post`?(y(),c(d,{key:1},[h(` The 2017 block: attention, add, normalise, feed-forward, add, normalise. Two sub-layers, each wrapped in a residual and a LayerNorm, and the shape at the top is the shape at the bottom — which is what lets you stack six of them, or ninety-six. `)],64)):(y(),c(d,{key:2},[t[1]||=h(` The same two sub-layers, with the normalisation moved `,-1),t[2]||=o(`strong`,null,`before`,-1),t[3]||=h(` each one. Now nothing sits on the residual path: from the bottom of the stack to the top there is a route made only of additions. Post-LN needs a learning-rate warmup to train deep and pre-LN largely does not, which is why essentially every model after GPT-2 is wired this way. `,-1)],64))]),default:n(()=>[(y(),c(`svg`,{class:`dl-blk`,viewBox:`0 0 ${M} ${Q.value}`,preserveAspectRatio:`xMidYMid meet`,role:`img`,"aria-label":`One transformer block, ${B.value===`post`?`with LayerNorm after each sub-layer as in the 2017 paper`:`with normalisation before each sub-layer as in modern models`}`},[t[0]||=o(`defs`,null,[o(`marker`,{id:`dl-blk-arrow`,viewBox:`0 0 8 8`,refX:`7`,refY:`4`,markerWidth:`5`,markerHeight:`5`,orient:`auto-start-reverse`},[o(`path`,{d:`M0,0 L8,4 L0,8 z`,fill:`currentColor`})])],-1),_(` The main path, bottom to top. `),o(`line`,{class:`dl-blk__spine`,x1:254,y1:X.value,x2:254,y2:s(Z)+10,"marker-end":`url(#dl-blk-arrow)`},null,8,D),o(`text`,{class:`dl-blk__port`,x:254,y:X.value+18,"text-anchor":`middle`},` in — (T, d) one vector per token `,8,te),o(`text`,{class:`dl-blk__port`,x:254,y:s(Z)-6,"text-anchor":`middle`},` out — (T, d), same shape, ready for the next block `,8,ne),_(` Residual connections, drawn as a rail down the left. `),(y(!0),c(d,null,i(ce.value,(e,t)=>(y(),c(`path`,{key:`arc${t}`,class:p([`dl-blk__residual`,{"is-lit":H.value||V.value>e.to}]),d:`M 250 ${$(e.from)}
|
| 2 |
+
H ${z} V ${$(e.to)}
|
| 3 |
+
H 238`,"marker-end":`url(#dl-blk-arrow)`},null,10,re))),128)),(y(!0),c(d,null,i(G.value,(e,t)=>(y(),c(d,{key:e.label+t},[_(` Sub-layer output joining its add. `),e.kind===`add`?(y(),c(`g`,ie,[o(`circle`,{class:p([`dl-blk__add`,{"is-lit":q(t),"is-current":V.value===t+1}]),cx:254,cy:Y(t),r:L},null,10,ae),o(`text`,{class:`dl-blk__plus`,x:254,y:Y(t)+5,"text-anchor":`middle`},`+`,8,oe)])):(y(),c(`g`,se,[o(`rect`,{class:p([`dl-blk__box`,{"is-lit":q(t),"is-current":V.value===t+1,"is-sublayer":e.sublayer}]),x:F,y:Y(t),width:P,height:I,rx:`6`},null,10,O),o(`text`,{class:p([`dl-blk__label`,{"is-lit":q(t)}]),x:254,y:Y(t)+(e.sub?15:22),"text-anchor":`middle`},g(e.label),11,k),e.sub?(y(),c(`text`,{key:0,class:p([`dl-blk__sub`,{"is-lit":q(t)}]),x:254,y:Y(t)+27,"text-anchor":`middle`},g(e.sub),11,A)):_(`v-if`,!0)]))],64))),128)),o(`text`,{class:`dl-blk__rail`,x:64,y:Q.value/2,"text-anchor":`middle`,transform:`rotate(-90 64 ${Q.value/2})`},` residual stream `,8,j)],8,E))]),_:1})}}}),[[`__scopeId`,`data-v-c465f205`]]),V={class:`mt-2 dl-callout`},H={__name:`class.md__slidev_28`,setup(i){let{$slidev:l,$nav:u,$clicksContext:d,$clicks:p,$page:m,$renderContext:g,$frontmatter:_}=b();return d.setup(),(i,l)=>{let u=B,d=S,p=a(`click`);return y(),f(w,v(t(s(x)(s(_),27))),{aside:n(t=>[l[2]||=o(`p`,null,[h(`Press `),o(`strong`,null,`Walk the block`),h(`. Six parts, all of them already met.`)],-1),r(d,null,{default:n(()=>[...l[0]||=[o(`ul`,null,[o(`li`,null,[h(`attention — the `),o(`strong`,null,`only`),h(` sideways step`)]),o(`li`,null,`add — the residual`),o(`li`,null,`norm`),o(`li`,null,`feed-forward — each position alone`),o(`li`,null,`add, norm`)],-1)]]),_:1}),e((y(),c(`div`,V,[...l[1]||=[o(`p`,null,`Same shape at the top as at the bottom. Stack it.`,-1)]])),[[p]])]),default:n(()=>[r(u,{variant:`post`,switchable:!1})]),_:1},16)}}};export{H as default};
|
lecture-06/class/assets/md-BimdYeK3.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,V as r,W as i,_ as a,_t as o,b as s,m as c,v as l,vt as u,x as d,xt as f,yt as p,z as m}from"./modules/shiki-BFOesOFp.js";import{n as h,t as g}from"./slidev/context-DrkI8qiD.js";import{t as _}from"./default-Cq9fupKf.js";var v={class:`mt-1 flex justify-center`},y={viewBox:`0 0 660 240`,class:`dl-diagram`,style:{"max-height":`12rem`},role:`img`,"aria-label":`The sentence the cat ate the fish because it was hungry. One softmax arc from it lands on cat; dashed lines mark three other questions: nearby noun, previous word, which clause`},b=[`x`],x=[`x`],S={class:`grid grid-cols-2 gap-8 mt-1`},C={class:`dl-callout`},w={__name:`class.md__slidev_22`,setup(w){let{$slidev:T,$nav:E,$clicksContext:D,$clicks:O,$page:k,$renderContext:A,$frontmatter:j}=h();return D.setup(),(h,w)=>{let T=i(`click`);return m(),l(_,p(t(o(g)(o(j),21))),{default:n(()=>[w[5]||=a(`h1`,null,`One softmax can only point at one thing`,-1),w[6]||=a(`p`,null,[d(`To read `),a(`em`,null,`it`),d(`, the model needs several answers at once.`)],-1),a(`div`,v,[(m(),s(`svg`,y,[w[1]||=a(`path`,{class:`dl-dg-grad is-good`,d:`M476 128 Q296 20 116 128`,style:{"stroke-width":`4`}},null,-1),w[2]||=a(`text`,{class:`dl-dg-small is-good`,x:`296`,y:`54`,"text-anchor":`middle`},`one softmax: most weight on one word`,-1),(m(),s(c,null,r([`the`,`cat`,`ate`,`the`,`fish`,`because`,`it`,`was`,`hungry`],(e,t)=>a(`g`,{key:t},[a(`rect`,{class:u(t===6?`dl-dg-box is-accent`:`dl-dg-box`),x:12+t*72,y:`130`,width:`64`,height:`30`,rx:`4`},null,10,b),a(`text`,{class:`dl-dg-lab is-sm`,x:44+t*72,y:`150`,"text-anchor":`middle`},f(e),9,x)])),64)),e((m(),s(`g`,null,[...w[0]||=[a(`path`,{class:`dl-dg-loss`,d:`M116 162 V188`},null,-1),a(`text`,{class:`dl-dg-small`,x:`116`,y:`202`,"text-anchor":`middle`},`what it refers to`,-1),a(`path`,{class:`dl-dg-loss`,d:`M332 162 V188`},null,-1),a(`text`,{class:`dl-dg-small`,x:`332`,y:`202`,"text-anchor":`middle`},`nearby noun`,-1),a(`path`,{class:`dl-dg-loss`,d:`M404 162 V216`},null,-1),a(`text`,{class:`dl-dg-small`,x:`404`,y:`230`,"text-anchor":`middle`},`previous word`,-1),a(`path`,{class:`dl-dg-loss`,d:`M374 172 H634`},null,-1),a(`text`,{class:`dl-dg-small`,x:`560`,y:`190`,"text-anchor":`middle`},`which clause`,-1)]])),[[T]])]))]),a(`div`,S,[e((m(),s(`div`,C,[...w[3]||=[a(`p`,null,[d(`A softmax puts its weight in `),a(`strong`,null,`one`),d(` place. Four questions need four sets of weights.`)],-1)]])),[[T]]),e((m(),s(`div`,null,[...w[4]||=[a(`p`,null,[d(`So run attention `),a(`strong`,null,`several times side by side`),d(`, each copy with its own `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`msub`,null,[a(`mi`,null,`W`),a(`mi`,null,`Q`)])]),a(`annotation`,{encoding:`application/x-tex`},`W_Q`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.9694em`,"vertical-align":`-0.2861em`}}),a(`span`,{class:`mord`},[a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),a(`span`,{class:`msupsub`},[a(`span`,{class:`vlist-t vlist-t2`},[a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.3283em`}},[a(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[a(`span`,{class:`pstrut`,style:{height:`2.7em`}}),a(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[a(`span`,{class:`mord mathnormal mtight`},`Q`)])])]),a(`span`,{class:`vlist-s`},``)]),a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.2861em`}},[a(`span`)])])])])])])])]),d(`, `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`msub`,null,[a(`mi`,null,`W`),a(`mi`,null,`K`)])]),a(`annotation`,{encoding:`application/x-tex`},`W_K`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),a(`span`,{class:`mord`},[a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),a(`span`,{class:`msupsub`},[a(`span`,{class:`vlist-t vlist-t2`},[a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.3283em`}},[a(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[a(`span`,{class:`pstrut`,style:{height:`2.7em`}}),a(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[a(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0715em`}},`K`)])])]),a(`span`,{class:`vlist-s`},``)]),a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.15em`}},[a(`span`)])])])])])])])]),d(`, `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`msub`,null,[a(`mi`,null,`W`),a(`mi`,null,`V`)])]),a(`annotation`,{encoding:`application/x-tex`},`W_V`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),a(`span`,{class:`mord`},[a(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),a(`span`,{class:`msupsub`},[a(`span`,{class:`vlist-t vlist-t2`},[a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.3283em`}},[a(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[a(`span`,{class:`pstrut`,style:{height:`2.7em`}}),a(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[a(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.2222em`}},`V`)])])]),a(`span`,{class:`vlist-s`},``)]),a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.15em`}},[a(`span`)])])])])])])])]),d(`. Each copy is a `),a(`strong`,null,`head`),d(`.`)],-1)]])),[[T]])])]),_:1},16)}}};export{w as default};
|
lecture-06/class/assets/md-BmvwA0Nf.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,m as l,v as u,x as d,yt as f,z as p}from"./modules/shiki-BFOesOFp.js";import{n as m,t as h}from"./slidev/context-DrkI8qiD.js";import{t as g}from"./slidev/VClicks-Dmx1KjZv.js";import{t as _}from"./default-Cq9fupKf.js";var v={class:`grid grid-cols-2 gap-8 mt-2 items-start`},y={viewBox:`0 0 330 200`,class:`dl-diagram`,role:`img`,"aria-label":`Score grids: 4 words give a 4 by 4 grid of 16 scores; 8 words give an 8 by 8 grid of 64 scores, four times as many`,style:{"max-height":`15rem`}},b=[`x`,`y`],x=[`x`,`y`],S={class:`dl-tight`},C={class:`mt-3 dl-callout`},w={class:`mt-3 dl-secondary`},T={__name:`class.md__slidev_29`,setup(T){let{$slidev:E,$nav:D,$clicksContext:O,$clicks:k,$page:A,$renderContext:j,$frontmatter:M}=m();return O.setup(),(m,T)=>{let E=g,D=a(`click`);return p(),u(_,f(t(s(h)(s(M),28))),{default:n(()=>[T[6]||=o(`h1`,null,`The bill for looking everywhere`,-1),T[7]||=o(`p`,null,[d(`The score matrix is `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`T`),o(`mo`,null,`×`),o(`mi`,null,`T`)]),o(`annotation`,{encoding:`application/x-tex`},`T \\times T`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.7667em`,"vertical-align":`-0.0833em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),o(`span`,{class:`mbin`},`×`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`)])])]),d(`: every position scores every position.`)],-1),o(`div`,v,[o(`div`,null,[(p(),c(`svg`,y,[T[1]||=o(`text`,{class:`dl-dg-small`,x:`10`,y:`16`},`query × key`,-1),(p(),c(l,null,i(16,e=>o(`rect`,{key:`a`+e,class:`dl-dg-box is-accent`,x:10+(e-1)%4*18,y:30+Math.floor((e-1)/4)*18,width:`16`,height:`16`,rx:`2`},null,8,b)),64)),T[2]||=o(`text`,{class:`dl-dg-small`,x:`10`,y:`122`},`T = 4: 16 scores`,-1),e((p(),c(`g`,null,[(p(),c(l,null,i(64,e=>o(`rect`,{key:`b`+e,class:`dl-dg-box`,x:160+(e-1)%8*18,y:30+Math.floor((e-1)/8)*18,width:`16`,height:`16`,rx:`2`},null,8,x)),64)),T[0]||=o(`text`,{class:`dl-dg-small`,x:`160`,y:`190`},`T = 8: 64 scores, 4 ×`,-1)])),[[D]])]))]),o(`div`,S,[r(E,null,{default:n(()=>[...T[3]||=[o(`ul`,null,[o(`li`,null,[d(`512 tokens → 262 144 scores `),o(`strong`,null,`per head, per layer`)])],-1)]]),_:1}),e((p(),c(`div`,C,[...T[4]||=[o(`p`,null,[d(`An RNN took `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`T`)]),o(`annotation`,{encoding:`application/x-tex`},`T`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`)])])]),d(` steps, one after another. Attention takes `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`msup`,null,[o(`mi`,null,`T`),o(`mn`,null,`2`)])]),o(`annotation`,{encoding:`application/x-tex`},`T^2`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8141em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.8141em`}},[o(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`2`)])])])])])])])])])]),d(`, all at once: more work, far less waiting.`)],-1)]])),[[D]]),e((p(),c(`div`,w,[...T[5]||=[o(`p`,null,[d(`Caveat: the feed-forward network grows like `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`T`),o(`msup`,null,[o(`mi`,null,`d`),o(`mn`,null,`2`)])]),o(`annotation`,{encoding:`application/x-tex`},`T d^2`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8141em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`},`d`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.8141em`}},[o(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`2`)])])])])])])])])])]),d(`, attention like `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`msup`,null,[o(`mi`,null,`T`),o(`mn`,null,`2`)]),o(`mi`,null,`d`)]),o(`annotation`,{encoding:`application/x-tex`},`T^2 d`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8141em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.8141em`}},[o(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`2`)])])])])])])]),o(`span`,{class:`mord mathnormal`},`d`)])])]),d(`. At `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`T`),o(`mo`,null,`=`),o(`mi`,null,`d`),o(`mo`,null,`=`),o(`mn`,null,`512`)]),o(`annotation`,{encoding:`application/x-tex`},`T = d = 512`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mrel`},`=`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6944em`}}),o(`span`,{class:`mord mathnormal`},`d`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mrel`},`=`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),o(`span`,{class:`mord`},`512`)])])]),d(`, attention is not yet the expensive part.`)],-1)]])),[[D]])])])]),_:1},16)}}};export{T as default};
|
lecture-06/class/assets/md-BnAqCEYv2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`class.md__slidev_3`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),2))),{default:t(()=>[p[0]||=r(`h1`,null,`Delete the recurrence`,-1),r(`div`,f,[n(m,{kind:`cut`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/class/assets/md-Bq0k6Ht_.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`class.md__slidev_7`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),6))),{default:t(()=>[p[0]||=r(`h1`,null,`The whole architecture first`,-1),p[1]||=r(`p`,null,`One stack reads, one stack writes.`,-1),r(`div`,f,[n(m,{kind:`encdec`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/class/assets/md-BsO5ZW2F.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,H as t,Q as n,S as r,_ as i,_t as a,b as o,v as s,w as c,xt as l,y as u,yt as d,z as f}from"./modules/shiki-BFOesOFp.js";import{t as p}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as m,t as h}from"./slidev/context-DrkI8qiD.js";import{t as g}from"./TransformerGlyph-D9TehqNe.js";var _={class:`slidev-layout dl-end`},v={class:`dl-end__body`},y={class:`dl-end__meta`},b={key:0,class:`dl-end__next`},x=[`href`],S=p(c({__name:`end`,props:{email:{},next:{}},setup(e){let{$slidev:n,$nav:r,$clicksContext:a,$clicks:s,$page:c,$renderContext:d,$frontmatter:p}=m(),h=e;return(e,n)=>(f(),o(`div`,_,[i(`div`,v,[t(e.$slots,`default`,{},void 0,!0)]),n[0]||=i(`div`,{class:`dl-rule`},null,-1),i(`div`,y,[h.next?(f(),o(`span`,b,`Next: `+l(h.next),1)):u(`v-if`,!0),h.email?(f(),o(`a`,{key:1,href:`mailto:${h.email}`},l(h.email),9,x)):u(`v-if`,!0)])]))}}),[[`__scopeId`,`data-v-f8d31aa3`]]),C={class:`mx-auto mt-2`,style:{width:`150px`}},w={__name:`class.md__slidev_44`,setup(t){let{$slidev:o,$nav:c,$clicksContext:l,$clicks:u,$page:p,$renderContext:_,$frontmatter:v}=m();return l.setup(),(t,o)=>{let c=g;return f(),s(S,d(e(a(h)(a(v),43))),{default:n(()=>[o[0]||=i(`h1`,null,`Questions?`,-1),i(`div`,C,[r(c,{kind:`decoder`,size:150})])]),_:1},16)}}};export{w as default};
|
lecture-06/class/assets/md-BtB6vj0a2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`class.md__slidev_38`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),37))),{default:t(()=>[p[0]||=r(`h1`,null,`In PyTorch`,-1),p[1]||=r(`p`,null,`Everything you computed by hand, as code.`,-1),r(`div`,f,[n(m,{kind:`code`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/class/assets/md-BxdQ0RGt.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,x as l,yt as u,z as d}from"./modules/shiki-BFOesOFp.js";import{n as f,t as p}from"./slidev/context-DrkI8qiD.js";import{t as m}from"./slidev/VClicks-Dmx1KjZv.js";import{t as h}from"./interactive-CI8hm4xw.js";import{t as g}from"./AttentionMatrix-D2xrh-OL.js";var _={class:`mt-2 dl-callout`},v={__name:`class.md__slidev_19`,setup(v){let{$slidev:y,$nav:b,$clicksContext:x,$clicks:S,$page:C,$renderContext:w,$frontmatter:T}=f();return x.setup(),(f,v)=>{let y=g,b=m,x=i(`click`);return d(),c(h,u(t(o(p)(o(T),18))),{aside:n(t=>[v[2]||=a(`p`,null,[l(`All three rows. Rows `),a(`strong`,null,`ask`),l(`, columns `),a(`strong`,null,`are read`),l(`.`)],-1),r(b,null,{default:n(()=>[...v[0]||=[a(`ul`,null,[a(`li`,null,[a(`em`,null,`the`),l(` asks nothing, so its row is flat: 0.33 each`)]),a(`li`,null,[a(`em`,null,`bank`),l(`’s row is the one we just derived`)]),a(`li`,null,[l(`Every row sums to `),a(`strong`,null,`1`)])],-1)]]),_:1}),e((d(),s(`div`,_,[...v[1]||=[a(`p`,null,[l(`No cell depends on another, so all nine are computed `),a(`strong`,null,`at the same time`),l(`. The wall comes down.`)],-1)]])),[[x]])]),default:n(()=>[r(y,{maskable:!1,switchable:``})]),_:1},16)}}};export{v as default};
|
lecture-06/class/assets/md-C3JZnPAz.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,W as r,_ as i,_t as a,b as o,v as s,x as c,yt as l,z as u}from"./modules/shiki-BFOesOFp.js";import{n as d,t as f}from"./slidev/context-DrkI8qiD.js";import{t as p}from"./default-Cq9fupKf.js";var m={class:`mt-1 flex justify-center`},h={viewBox:`0 0 620 132`,class:`dl-diagram`,style:{"max-height":`11rem`},role:`img`,"aria-label":`Four boxes in a row — score, scale, softmax, mix — with the bank row after each: 0 2 0, then 0 1.41 0, then 0.16 0.67 0.16, then a new vector for bank`},g={class:`mt-3 dl-callout`},_={__name:`class.md__slidev_17`,setup(_){let{$slidev:v,$nav:y,$clicksContext:b,$clicks:x,$page:S,$renderContext:C,$frontmatter:w}=d();return b.setup(),(d,_)=>{let v=r(`click`);return u(),s(p,l(t(a(f)(a(w),16))),{default:n(()=>[_[7]||=i(`h1`,null,`Four steps, and that is the whole mechanism`,-1),i(`div`,m,[(u(),o(`svg`,h,[_[4]||=i(`defs`,null,[i(`marker`,{id:`l6s1-steps`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[i(`path`,{class:`dl-dg-head`,d:`M0 0 L7 3.5 L0 7 z`})])],-1),_[5]||=i(`text`,{class:`dl-dg-small`,x:`10`,y:`12`},`the bank row of "the river bank"`,-1),e((u(),o(`g`,null,[..._[0]||=[i(`rect`,{class:`dl-dg-box`,x:`10`,y:`22`,width:`130`,height:`40`,rx:`4`},null,-1),i(`text`,{class:`dl-dg-lab is-sm`,x:`75`,y:`47`,"text-anchor":`middle`},`1 · score`,-1),i(`text`,{class:`dl-dg-in`,x:`75`,y:`90`,"text-anchor":`middle`},`0, 2, 0`,-1),i(`text`,{class:`dl-dg-small`,x:`75`,y:`110`,"text-anchor":`middle`},`q · k, every key`,-1)]])),[[v]]),e((u(),o(`g`,null,[..._[1]||=[i(`path`,{class:`dl-dg-arrow`,d:`M140 42 H162`,"marker-end":`url(#l6s1-steps)`},null,-1),i(`rect`,{class:`dl-dg-box`,x:`166`,y:`22`,width:`130`,height:`40`,rx:`4`},null,-1),i(`text`,{class:`dl-dg-lab is-sm`,x:`231`,y:`47`,"text-anchor":`middle`},`2 · scale`,-1),i(`text`,{class:`dl-dg-in`,x:`231`,y:`90`,"text-anchor":`middle`},`0, 1.41, 0`,-1),i(`text`,{class:`dl-dg-small`,x:`231`,y:`110`,"text-anchor":`middle`},`÷ √2`,-1)]])),[[v]]),e((u(),o(`g`,null,[..._[2]||=[i(`path`,{class:`dl-dg-arrow`,d:`M296 42 H318`,"marker-end":`url(#l6s1-steps)`},null,-1),i(`rect`,{class:`dl-dg-box`,x:`322`,y:`22`,width:`130`,height:`40`,rx:`4`},null,-1),i(`text`,{class:`dl-dg-lab is-sm`,x:`387`,y:`47`,"text-anchor":`middle`},`3 · softmax`,-1),i(`text`,{class:`dl-dg-in`,x:`387`,y:`90`,"text-anchor":`middle`},`0.16, 0.67, 0.16`,-1),i(`text`,{class:`dl-dg-small`,x:`387`,y:`110`,"text-anchor":`middle`},`positive, sum to 1`,-1)]])),[[v]]),e((u(),o(`g`,null,[..._[3]||=[i(`path`,{class:`dl-dg-arrow`,d:`M452 42 H474`,"marker-end":`url(#l6s1-steps)`},null,-1),i(`rect`,{class:`dl-dg-box is-accent`,x:`478`,y:`22`,width:`130`,height:`40`,rx:`4`},null,-1),i(`text`,{class:`dl-dg-lab is-sm`,x:`543`,y:`47`,"text-anchor":`middle`},`4 · mix`,-1),i(`text`,{class:`dl-dg-in`,x:`543`,y:`90`,"text-anchor":`middle`},`new vector for bank`,-1),i(`text`,{class:`dl-dg-small`,x:`543`,y:`110`,"text-anchor":`middle`},`weighted sum of values`,-1)]])),[[v]])]))]),_[8]||=i(`div`,{class:`dl-tight mt-2`},[i(`p`,null,[c(`The score `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`msub`,null,[i(`mi`,null,`e`),i(`mrow`,null,[i(`mi`,null,`i`),i(`mi`,null,`j`)])]),i(`mo`,null,`=`),i(`msub`,null,[i(`mi`,{mathvariant:`bold`},`q`),i(`mi`,null,`i`)]),i(`mo`,null,`⋅`),i(`msub`,null,[i(`mi`,{mathvariant:`bold`},`k`),i(`mi`,null,`j`)])]),i(`annotation`,{encoding:`application/x-tex`},`e_{ij} = \\mathbf{q}_i \\cdot \\mathbf{k}_j`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.7167em`,"vertical-align":`-0.2861em`}}),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathnormal`},`e`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t vlist-t2`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.3117em`}},[i(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mtight`},[i(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0572em`}},`ij`)])])])]),i(`span`,{class:`vlist-s`},``)]),i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.2861em`}},[i(`span`)])])])])]),i(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),i(`span`,{class:`mrel`},`=`),i(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.6389em`,"vertical-align":`-0.1944em`}}),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathbf`},`q`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t vlist-t2`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.3117em`}},[i(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mathnormal mtight`},`i`)])])]),i(`span`,{class:`vlist-s`},``)]),i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.15em`}},[i(`span`)])])])])]),i(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),i(`span`,{class:`mbin`},`⋅`),i(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.9805em`,"vertical-align":`-0.2861em`}}),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathbf`},`k`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t vlist-t2`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.3117em`}},[i(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0572em`}},`j`)])])]),i(`span`,{class:`vlist-s`},``)]),i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.2861em`}},[i(`span`)])])])])])])])]),c(` is large when query `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`mi`,null,`i`)]),i(`annotation`,{encoding:`application/x-tex`},`i`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.6595em`}}),i(`span`,{class:`mord mathnormal`},`i`)])])]),c(` and key `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`mi`,null,`j`)]),i(`annotation`,{encoding:`application/x-tex`},`j`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.854em`,"vertical-align":`-0.1944em`}}),i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0572em`}},`j`)])])]),c(` point the same way. `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`msub`,null,[i(`mi`,null,`d`),i(`mi`,null,`k`)])]),i(`annotation`,{encoding:`application/x-tex`},`d_k`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathnormal`},`d`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t vlist-t2`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.3361em`}},[i(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),i(`span`,{class:`vlist-s`},``)]),i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.15em`}},[i(`span`)])])])])])])])]),c(` is the key width.`)])],-1),e((u(),o(`div`,g,[..._[6]||=[i(`p`,null,`Score, scale, softmax, mix. Steps 1, 3 and 4 are the 2014 attention. Step 2 is new, and it is one division.`,-1)]])),[[v]])]),_:1},16)}}};export{_ as default};
|
lecture-06/class/assets/md-C4a87Mz_.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,m as l,v as u,x as d,xt as f,yt as p,z as m}from"./modules/shiki-BFOesOFp.js";import{n as h,t as g}from"./slidev/context-DrkI8qiD.js";import{t as _}from"./slidev/VClicks-Dmx1KjZv.js";import{t as v}from"./default-Cq9fupKf.js";var y={class:`grid grid-cols-2 gap-8 mt-2 dl-tight`},b={viewBox:`0 0 400 190`,class:`dl-diagram`,role:`img`,"aria-label":`Five words of text give four next-word training examples: each word predicts the word after it`},x=[`x`],S=[`x`],C=[`d`],w=[`x`],T=[`x`],E={class:`mt-3 dl-callout`},D={__name:`class.md__slidev_36`,setup(D){let{$slidev:O,$nav:k,$clicksContext:A,$clicks:j,$page:M,$renderContext:N,$frontmatter:P}=h();return A.setup(),(h,D)=>{let O=_,k=a(`click`);return m(),u(v,p(t(s(g)(s(P),35))),{default:n(()=>[D[5]||=o(`h1`,null,`Why decoder-only took over`,-1),D[6]||=o(`p`,null,[d(`Not the most expressive design. The one with the most `),o(`strong`,null,`training data`),d(`.`)],-1),o(`div`,y,[o(`div`,null,[(m(),c(`svg`,b,[D[0]||=o(`defs`,null,[o(`marker`,{id:`l6s4-next`,markerWidth:`6`,markerHeight:`6`,refX:`5`,refY:`3`,orient:`auto`},[o(`path`,{class:`dl-dg-head`,d:`M0 0 L6 3 L0 6 z`})])],-1),D[1]||=o(`text`,{class:`dl-dg-small`,x:`6`,y:`24`},`predicts the next word`,-1),(m(),c(l,null,i([`river`,`bank`,`was`,`flooded`],(e,t)=>o(`g`,{key:`p`+t},[o(`rect`,{class:`dl-dg-fill`,x:6+t*78,y:`36`,width:`70`,height:`30`,rx:`4`},null,8,x),o(`text`,{class:`dl-dg-in`,x:41+t*78,y:`56`,"text-anchor":`middle`},f(e),9,S),o(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s4-next)`,d:`M${41+t*78} 128 V72`},null,8,C)])),64)),(m(),c(l,null,i([`the`,`river`,`bank`,`was`,`flooded`],(e,t)=>o(`g`,{key:`t`+t},[o(`rect`,{class:`dl-dg-box`,x:6+t*78,y:`130`,width:`70`,height:`30`,rx:`4`},null,8,w),o(`text`,{class:`dl-dg-lab is-sm`,x:41+t*78,y:`150`,"text-anchor":`middle`},f(e),9,T)])),64)),D[2]||=o(`text`,{class:`dl-dg-small is-good`,x:`6`,y:`182`},`5 words of text → 4 training examples, no labels`,-1)]))]),o(`div`,null,[r(O,null,{default:n(()=>[...D[3]||=[o(`ul`,null,[o(`li`,null,[d(`Next-token prediction needs `),o(`strong`,null,`no labels`),d(`: any text is training data`)]),o(`li`,null,[o(`strong`,null,`Every position`),d(` is an example, so one document gives thousands`)]),o(`li`,null,`Any task written as text becomes next-token prediction`),o(`li`,null,`One stack instead of two: simpler to scale and to serve`)],-1)]]),_:1})])]),e((m(),c(`div`,E,[...D[4]||=[o(`p`,null,`The design that won was the one whose training data was free.`,-1)]])),[[k]])]),_:1},16)}}};export{D as default};
|
lecture-06/class/assets/md-C9_5NLsW.js
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
lecture-06/class/assets/md-CJukBVkv.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,V as r,W as i,_ as a,_t as o,b as s,m as c,v as l,x as u,yt as d,z as f}from"./modules/shiki-BFOesOFp.js";import{n as p,t as m}from"./slidev/context-DrkI8qiD.js";import{t as h}from"./default-Cq9fupKf.js";var g={class:`mt-1 flex justify-center`},_={viewBox:`0 0 620 240`,class:`dl-diagram is-sm-h`,role:`img`,"aria-label":`An encoder stack reads the source tokens and outputs one vector per token; a decoder stack with three sub-layers reads the encoder output through cross-attention`},v=[`x`],y=[`x`],b=[`x`],x={class:`mt-1 dl-callout`},S={__name:`class.md__slidev_9`,setup(S){let{$slidev:C,$nav:w,$clicksContext:T,$clicks:E,$page:D,$renderContext:O,$frontmatter:k}=p();return T.setup(),(p,S)=>{let C=i(`click`);return f(),l(h,d(t(o(m)(o(k),8))),{default:n(()=>[S[25]||=a(`h1`,null,`Two stacks`,-1),S[26]||=a(`p`,null,[u(`The 2017 paper is a `),a(`strong`,null,`translation`),u(` model. One stack reads the source; the other writes the target.`)],-1),a(`div`,g,[(f(),s(`svg`,_,[S[23]||=a(`defs`,null,[a(`marker`,{id:`l6s3-a1`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[a(`path`,{class:`dl-dg-head`,d:`M0 0 L7 3.5 L0 7 z`})])],-1),a(`g`,null,[S[0]||=a(`text`,{class:`dl-dg-small`,x:`140`,y:`14`,"text-anchor":`middle`},`one vector per source token`,-1),(f(),s(c,null,r(3,e=>a(`rect`,{key:`eo${e}`,class:`dl-dg-box is-accent`,x:88+(e-1)*38,y:`22`,width:`26`,height:`22`,rx:`3`},null,8,v)),64)),S[1]||=a(`rect`,{class:`dl-dg-fill is-muted is-dashed`,x:`40`,y:`70`,width:`200`,height:`110`,rx:`6`},null,-1),S[2]||=a(`text`,{class:`dl-dg-small`,x:`52`,y:`86`},`encoder, 6 blocks`,-1),S[3]||=a(`rect`,{class:`dl-dg-box`,x:`60`,y:`94`,width:`160`,height:`30`,rx:`4`},null,-1),S[4]||=a(`text`,{class:`dl-dg-in`,x:`140`,y:`114`,"text-anchor":`middle`},`feed-forward`,-1),S[5]||=a(`rect`,{class:`dl-dg-box`,x:`60`,y:`134`,width:`160`,height:`30`,rx:`4`},null,-1),S[6]||=a(`text`,{class:`dl-dg-in`,x:`140`,y:`154`,"text-anchor":`middle`},`self-attention, no mask`,-1),(f(),s(c,null,r(3,e=>a(`rect`,{key:`ei${e}`,class:`dl-dg-box`,x:88+(e-1)*38,y:`206`,width:`26`,height:`22`,rx:`3`},null,8,y)),64)),S[7]||=a(`text`,{class:`dl-dg-small`,x:`80`,y:`221`,"text-anchor":`end`},`source`,-1),S[8]||=a(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s3-a1)`,d:`M140 204 V184`},null,-1),S[9]||=a(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s3-a1)`,d:`M140 68 V48`},null,-1)]),e((f(),s(`g`,null,[S[10]||=a(`rect`,{class:`dl-dg-fill is-dashed`,x:`380`,y:`50`,width:`210`,height:`130`,rx:`6`},null,-1),S[11]||=a(`text`,{class:`dl-dg-small`,x:`392`,y:`66`},`decoder, 6 blocks`,-1),S[12]||=a(`rect`,{class:`dl-dg-box`,x:`398`,y:`72`,width:`174`,height:`28`,rx:`4`},null,-1),S[13]||=a(`text`,{class:`dl-dg-in`,x:`485`,y:`91`,"text-anchor":`middle`},`feed-forward`,-1),S[14]||=a(`rect`,{class:`dl-dg-box is-accent`,x:`398`,y:`106`,width:`174`,height:`28`,rx:`4`},null,-1),S[15]||=a(`text`,{class:`dl-dg-in`,x:`485`,y:`125`,"text-anchor":`middle`},`cross-attention`,-1),S[16]||=a(`rect`,{class:`dl-dg-box`,x:`398`,y:`140`,width:`174`,height:`28`,rx:`4`},null,-1),S[17]||=a(`text`,{class:`dl-dg-in`,x:`485`,y:`159`,"text-anchor":`middle`},`masked self-attention`,-1),(f(),s(c,null,r(3,e=>a(`rect`,{key:`di${e}`,class:`dl-dg-box`,x:428+(e-1)*38,y:`206`,width:`26`,height:`22`,rx:`3`},null,8,b)),64)),S[18]||=a(`text`,{class:`dl-dg-small`,x:`420`,y:`221`,"text-anchor":`end`},`target so far`,-1),S[19]||=a(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s3-a1)`,d:`M485 204 V184`},null,-1),S[20]||=a(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s3-a1)`,d:`M485 48 V26`},null,-1),S[21]||=a(`text`,{class:`dl-dg-small`,x:`497`,y:`22`},`next word`,-1)])),[[C]]),e((f(),s(`g`,null,[...S[22]||=[a(`path`,{class:`dl-dg-line`,"marker-end":`url(#l6s3-a1)`,d:`M192 33 H310 V120 H394`},null,-1),a(`text`,{class:`dl-dg-small is-good`,x:`318`,y:`84`},`K and V`,-1)]])),[[C]])]))]),e((f(),s(`div`,x,[...S[24]||=[a(`p`,null,`One vector per source token: the bottleneck is gone. Nothing has to be summarised in advance.`,-1)]])),[[C]])]),_:1},16)}}};export{S as default};
|
lecture-06/class/assets/md-CNfnuG9N2.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,m as l,v as u,x as d,xt as f,yt as p,z as m}from"./modules/shiki-BFOesOFp.js";import{n as h,t as g}from"./slidev/context-DrkI8qiD.js";import{t as _}from"./slidev/VClicks-Dmx1KjZv.js";import{t as v}from"./default-Cq9fupKf.js";import{t as y}from"./slidev/KaTexBlockWrapper-BIlBldVC.js";var b={class:`grid grid-cols-2 gap-8 mt-1 dl-tight`},x={viewBox:`0 0 300 160`,class:`dl-diagram`,style:{"max-height":`9.5rem`},role:`img`,"aria-label":`One decoder query sends lines of different strength to three encoder outputs, which serve as keys and values`},S=[`y`],C=[`y`],w={class:`mt-2 dl-callout`},T={__name:`class.md__slidev_32`,setup(T){let{$slidev:E,$nav:D,$clicksContext:O,$clicks:k,$page:A,$renderContext:j,$frontmatter:M}=h();return O.setup(),(h,T)=>{let E=y,D=_,O=a(`click`);return m(),u(v,p(t(s(g)(s(M),31))),{default:n(()=>[T[7]||=o(`h1`,null,`Cross-attention: the decoder reads the encoder`,-1),T[8]||=o(`p`,null,[d(`Self-attention takes `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`Q`)]),o(`annotation`,{encoding:`application/x-tex`},`Q`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8778em`,"vertical-align":`-0.1944em`}}),o(`span`,{class:`mord mathnormal`},`Q`)])])]),d(`, `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`K`)]),o(`annotation`,{encoding:`application/x-tex`},`K`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0715em`}},`K`)])])]),d(` and `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,null,`V`)]),o(`annotation`,{encoding:`application/x-tex`},`V`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.2222em`}},`V`)])])]),d(` from `),o(`strong`,null,`one`),d(` stack. Cross-attention uses `),o(`strong`,null,`two`),d(`.`)],-1),r(E,{ranges:[]},{default:n(()=>[...T[0]||=[o(`p`,null,[o(`span`,{class:`katex-display`},[o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`,display:`block`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,{mathvariant:`normal`},`Attention`),o(`mo`,null,``),o(`mo`,{stretchy:`false`},`(`),o(`munder`,null,[o(`munder`,null,[o(`mi`,null,`Q`),o(`mo`,{stretchy:`true`},`⏟`)]),o(`mtext`,null,`decoder`)]),o(`mo`,{separator:`true`},`,`),o(`mtext`,null,` `),o(`munder`,null,[o(`munder`,null,[o(`mrow`,null,[o(`mi`,null,`K`),o(`mo`,{separator:`true`},`,`),o(`mtext`,null,` `),o(`mi`,null,`V`)]),o(`mo`,{stretchy:`true`},`⏟`)]),o(`mtext`,null,`encoder`)]),o(`mo`,{stretchy:`false`},`)`)]),o(`annotation`,{encoding:`application/x-tex`},`\\operatorname{Attention}(\\underbrace{Q}_{\\text{decoder}},\\; \\underbrace{K,\\; V}_{\\text{encoder}})`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`2.2785em`,"vertical-align":`-1.5285em`}}),o(`span`,{class:`mop`},[o(`span`,{class:`mord mathrm`},`Attention`)]),o(`span`,{class:`mopen`},`(`),o(`span`,{class:`minner munder`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.6833em`}},[o(`span`,{style:{top:`-1.4715em`}},[o(`span`,{class:`pstrut`,style:{height:`3em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},[o(`span`,{class:`mord text mtight`},[o(`span`,{class:`mord mtight`},`decoder`)])])])]),o(`span`,{style:{top:`-3em`}},[o(`span`,{class:`pstrut`,style:{height:`3em`}}),o(`span`,{class:`minner munder`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.6833em`}},[o(`span`,{class:`svg-align`,style:{top:`-2.1576em`}},[o(`span`,{class:`pstrut`,style:{height:`3em`}}),o(`span`,{class:`katex-stretchy`,style:{height:`0.548em`,"min-width":`1.6em`}},[o(`span`,{class:`brace-left`,style:{height:`0.548em`}},[o(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`0.548em`,viewBox:`0 0 400000 548`,preserveAspectRatio:`xMinYMin slice`},[o(`path`,{d:`M0 6l6-6h17c12.688 0 19.313.3 20 1 4 4 7.313 8.3 10 13
|
| 2 |
+
35.313 51.3 80.813 93.8 136.5 127.5 55.688 33.7 117.188 55.8 184.5 66.5.688
|
| 3 |
+
0 2 .3 4 1 18.688 2.7 76 4.3 172 5h399450v120H429l-6-1c-124.688-8-235-61.7
|
| 4 |
+
-331-161C60.687 138.7 32.312 99.3 7 54L0 41V6z`})])]),o(`span`,{class:`brace-center`,style:{height:`0.548em`}},[o(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`0.548em`,viewBox:`0 0 400000 548`,preserveAspectRatio:`xMidYMin slice`},[o(`path`,{d:`M199572 214
|
| 5 |
+
c100.7 8.3 195.3 44 280 108 55.3 42 101.7 93 139 153l9 14c2.7-4 5.7-8.7 9-14
|
| 6 |
+
53.3-86.7 123.7-153 211-199 66.7-36 137.3-56.3 212-62h199568v120H200432c-178.3
|
| 7 |
+
11.7-311.7 78.3-403 201-6 8-9.7 12-11 12-.7.7-6.7 1-18 1s-17.3-.3-18-1c-1.3 0
|
| 8 |
+
-5-4-11-12-44.7-59.3-101.3-106.3-170-141s-145.3-54.3-229-60H0V214z`})])]),o(`span`,{class:`brace-right`,style:{height:`0.548em`}},[o(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`0.548em`,viewBox:`0 0 400000 548`,preserveAspectRatio:`xMaxYMin slice`},[o(`path`,{d:`M399994 0l6 6v35l-6 11c-56 104-135.3 181.3-238 232-57.3
|
| 9 |
+
28.7-117 45-179 50H-300V214h399897c43.3-7 81-15 113-26 100.7-33 179.7-91 237
|
| 10 |
+
-174 2.7-5 6-9 10-13 .7-1 7.3-1 20-1h17z`})])])])]),o(`span`,{style:{top:`-3em`}},[o(`span`,{class:`pstrut`,style:{height:`3em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`},`Q`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.8424em`}},[o(`span`)])])])])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`1.5285em`}},[o(`span`)])])])]),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mpunct`},`,`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`minner munder`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.6833em`}},[o(`span`,{style:{top:`-1.4715em`}},[o(`span`,{class:`pstrut`,style:{height:`3em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},[o(`span`,{class:`mord text mtight`},[o(`span`,{class:`mord mtight`},`encoder`)])])])]),o(`span`,{style:{top:`-3em`}},[o(`span`,{class:`pstrut`,style:{height:`3em`}}),o(`span`,{class:`minner munder`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.6833em`}},[o(`span`,{class:`svg-align`,style:{top:`-2.1576em`}},[o(`span`,{class:`pstrut`,style:{height:`3em`}}),o(`span`,{class:`katex-stretchy`,style:{height:`0.548em`,"min-width":`1.6em`}},[o(`span`,{class:`brace-left`,style:{height:`0.548em`}},[o(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`0.548em`,viewBox:`0 0 400000 548`,preserveAspectRatio:`xMinYMin slice`},[o(`path`,{d:`M0 6l6-6h17c12.688 0 19.313.3 20 1 4 4 7.313 8.3 10 13
|
| 11 |
+
35.313 51.3 80.813 93.8 136.5 127.5 55.688 33.7 117.188 55.8 184.5 66.5.688
|
| 12 |
+
0 2 .3 4 1 18.688 2.7 76 4.3 172 5h399450v120H429l-6-1c-124.688-8-235-61.7
|
| 13 |
+
-331-161C60.687 138.7 32.312 99.3 7 54L0 41V6z`})])]),o(`span`,{class:`brace-center`,style:{height:`0.548em`}},[o(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`0.548em`,viewBox:`0 0 400000 548`,preserveAspectRatio:`xMidYMin slice`},[o(`path`,{d:`M199572 214
|
| 14 |
+
c100.7 8.3 195.3 44 280 108 55.3 42 101.7 93 139 153l9 14c2.7-4 5.7-8.7 9-14
|
| 15 |
+
53.3-86.7 123.7-153 211-199 66.7-36 137.3-56.3 212-62h199568v120H200432c-178.3
|
| 16 |
+
11.7-311.7 78.3-403 201-6 8-9.7 12-11 12-.7.7-6.7 1-18 1s-17.3-.3-18-1c-1.3 0
|
| 17 |
+
-5-4-11-12-44.7-59.3-101.3-106.3-170-141s-145.3-54.3-229-60H0V214z`})])]),o(`span`,{class:`brace-right`,style:{height:`0.548em`}},[o(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`0.548em`,viewBox:`0 0 400000 548`,preserveAspectRatio:`xMaxYMin slice`},[o(`path`,{d:`M399994 0l6 6v35l-6 11c-56 104-135.3 181.3-238 232-57.3
|
| 18 |
+
28.7-117 45-179 50H-300V214h399897c43.3-7 81-15 113-26 100.7-33 179.7-91 237
|
| 19 |
+
-174 2.7-5 6-9 10-13 .7-1 7.3-1 20-1h17z`})])])])]),o(`span`,{style:{top:`-3em`}},[o(`span`,{class:`pstrut`,style:{height:`3em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0715em`}},`K`),o(`span`,{class:`mpunct`},`,`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.2222em`}},`V`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.8424em`}},[o(`span`)])])])])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`1.5285em`}},[o(`span`)])])])]),o(`span`,{class:`mclose`},`)`)])])])])],-1)]]),_:1}),o(`div`,b,[o(`div`,null,[(m(),c(`svg`,x,[(m(),c(l,null,i(3,e=>o(`rect`,{key:`s${e}`,class:`dl-dg-box`,x:`196`,y:8+(e-1)*48,width:`96`,height:`36`,rx:`4`},null,8,S)),64)),(m(),c(l,null,i(3,e=>o(`text`,{key:`st${e}`,class:`dl-dg-in`,x:`244`,y:31+(e-1)*48,"text-anchor":`middle`},`K, V `+f(e),9,C)),64)),T[2]||=o(`text`,{class:`dl-dg-small`,x:`244`,y:`156`,"text-anchor":`middle`},`encoder output`,-1),e((m(),c(`g`,null,[...T[1]||=[o(`path`,{class:`dl-dg-line is-muted`,d:`M74 75 L194 26`,style:{strokeWidth:1.4}},null,-1),o(`path`,{class:`dl-dg-line`,d:`M74 75 L194 74`,style:{strokeWidth:5}},null,-1),o(`path`,{class:`dl-dg-line is-muted`,d:`M74 75 L194 122`,style:{strokeWidth:2.4}},null,-1)]])),[[O]]),T[3]||=o(`rect`,{class:`dl-dg-box is-accent`,x:`6`,y:`57`,width:`68`,height:`36`,rx:`4`},null,-1),T[4]||=o(`text`,{class:`dl-dg-in`,x:`40`,y:`80`,"text-anchor":`middle`},`Q`,-1)]))]),o(`div`,null,[r(D,null,{default:n(()=>[...T[5]||=[o(`ul`,null,[o(`li`,null,[d(`Decoder query: `),o(`em`,null,`what does the next word need?`)]),o(`li`,null,`Encoder: one key and value per source token`),o(`li`,null,`Score, softmax, weighted sum: Bahdanau’s 2014 attention`)],-1)]]),_:1}),e((m(),c(`div`,w,[...T[6]||=[o(`p`,null,[d(`Compared with 2014: a `),o(`strong`,null,`scaled dot product`),d(` score, and no recurrence on either side.`)],-1)]])),[[O]])])])]),_:1},16)}}};export{T as default};
|
lecture-06/class/assets/md-Cdeogdwt.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,x as l,yt as u,z as d}from"./modules/shiki-BFOesOFp.js";import{n as f,t as p}from"./slidev/context-DrkI8qiD.js";import{t as m}from"./slidev/VClicks-Dmx1KjZv.js";import{t as h}from"./default-Cq9fupKf.js";var g={class:`grid grid-cols-[0.9fr_1.1fr] gap-8 mt-2 items-start`},_={viewBox:`0 0 240 300`,class:`dl-diagram`,role:`img`,"aria-label":`Input x goes up two paths: through the sub-layer, and a copy unchanged. The two are added at a plus sign, then LayerNorm, giving an output of the same shape`,style:{"max-height":`17rem`}},v={class:`dl-tight`},y={class:`mt-3 dl-callout`},b={__name:`class.md__slidev_27`,setup(b){let{$slidev:x,$nav:S,$clicksContext:C,$clicks:w,$page:T,$renderContext:E,$frontmatter:D}=f();return C.setup(),(f,b)=>{let x=m,S=i(`click`);return d(),c(h,u(t(o(p)(o(D),26))),{default:n(()=>[b[6]||=a(`h1`,null,`Two wrappers that make it stackable`,-1),b[7]||=a(`p`,null,`Each sub-layer is wrapped the same way, so blocks can be stacked.`,-1),a(`div`,g,[a(`div`,null,[(d(),s(`svg`,_,[...b[0]||=[a(`defs`,null,[a(`marker`,{id:`l6s2-wrap-arrow`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[a(`path`,{class:`dl-dg-head`,d:`M0 0 L7 3.5 L0 7 z`})])],-1),a(`rect`,{class:`dl-dg-box`,x:`95`,y:`264`,width:`50`,height:`28`,rx:`4`},null,-1),a(`text`,{class:`dl-dg-lab`,x:`120`,y:`284`,"text-anchor":`middle`},`x`,-1),a(`path`,{class:`dl-dg-grad is-good`,"marker-end":`url(#l6s2-wrap-arrow)`,d:`M120 262 V150`,style:{"stroke-width":`2.4`}},null,-1),a(`text`,{class:`dl-dg-small is-good`,x:`112`,y:`214`,"text-anchor":`end`},`x copied,`,-1),a(`text`,{class:`dl-dg-small is-good`,x:`112`,y:`228`,"text-anchor":`end`},`unchanged`,-1),a(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s2-wrap-arrow)`,d:`M120 244 H190 V214`},null,-1),a(`rect`,{class:`dl-dg-box is-accent`,x:`146`,y:`172`,width:`88`,height:`40`,rx:`4`},null,-1),a(`text`,{class:`dl-dg-small`,x:`190`,y:`196`,"text-anchor":`middle`},`sub-layer`,-1),a(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s2-wrap-arrow)`,d:`M190 170 V136 H136`},null,-1),a(`circle`,{class:`dl-dg-op`,cx:`120`,cy:`136`,r:`13`},null,-1),a(`text`,{class:`dl-dg-lab is-sm`,x:`120`,y:`141`,"text-anchor":`middle`},`+`,-1),a(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s2-wrap-arrow)`,d:`M120 122 V104`},null,-1),a(`rect`,{class:`dl-dg-box`,x:`60`,y:`72`,width:`120`,height:`30`,rx:`4`},null,-1),a(`text`,{class:`dl-dg-small`,x:`120`,y:`91`,"text-anchor":`middle`},`LayerNorm`,-1),a(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s2-wrap-arrow)`,d:`M120 70 V40`},null,-1),a(`text`,{class:`dl-dg-small`,x:`120`,y:`30`,"text-anchor":`middle`},`same shape out`,-1)]]))]),a(`div`,v,[b[4]||=a(`h3`,null,`Residual`,-1),r(x,null,{default:n(()=>[...b[1]||=[a(`ul`,null,[a(`li`,null,[a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`mi`,{mathvariant:`bold`},`x`),a(`mo`,null,`+`),a(`mi`,{mathvariant:`normal`},`SubLayer`),a(`mo`,null,``),a(`mo`,{stretchy:`false`},`(`),a(`mi`,{mathvariant:`bold`},`x`),a(`mo`,{stretchy:`false`},`)`)]),a(`annotation`,{encoding:`application/x-tex`},`\\mathbf{x} + \\operatorname{SubLayer}(\\mathbf{x})`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.6667em`,"vertical-align":`-0.0833em`}}),a(`span`,{class:`mord mathbf`},`x`),a(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),a(`span`,{class:`mbin`},`+`),a(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),a(`span`,{class:`mop`},[a(`span`,{class:`mord mathrm`},`SubLayer`)]),a(`span`,{class:`mopen`},`(`),a(`span`,{class:`mord mathbf`},`x`),a(`span`,{class:`mclose`},`)`)])])]),l(`: `),a(`strong`,null,`add`),l(`, never replace`)]),a(`li`,null,`A gradient path with no matrix on it, as in a ResNet`)],-1)]]),_:1}),b[5]||=a(`h3`,null,`LayerNorm`,-1),r(x,null,{default:n(()=>[...b[2]||=[a(`ul`,null,[a(`li`,null,[l(`Re-centre and re-scale `),a(`strong`,null,`each position’s own vector`)]),a(`li`,null,`Not BatchNorm: nothing is shared across the batch or across time`)],-1)]]),_:1}),e((d(),s(`div`,y,[...b[3]||=[a(`p`,null,[l(`Same shape in, `),a(`strong`,null,`same shape out`),l(`. That is why blocks stack.`)],-1)]])),[[S]])])])]),_:1},16)}}};export{b as default};
|
lecture-06/class/assets/md-Cuhu_vvG2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,m as l,v as u,x as d,xt as f,yt as p,z as m}from"./modules/shiki-BFOesOFp.js";import{n as h,t as g}from"./slidev/context-DrkI8qiD.js";import{t as _}from"./slidev/VClicks-Dmx1KjZv.js";import{t as v}from"./default-Cq9fupKf.js";import{t as y}from"./slidev/KaTexBlockWrapper-BIlBldVC.js";var b={class:`mt-1`},x={class:`grid grid-cols-2 gap-8 mt-1 dl-tight`},S={class:`mt-2 dl-secondary`},C={viewBox:`0 0 600 170`,class:`dl-diagram`,role:`img`,"aria-label":`The words the, river and bank each go up through their own copy of the same feed-forward network; there are no arrows between the columns`},w=[`x`],T=[`x`],E=[`d`],D=[`points`],O=[`x`],k=[`d`],A=[`x`],j={class:`mt-2 dl-callout`},M={__name:`class.md__slidev_26`,setup(M){let{$slidev:N,$nav:P,$clicksContext:F,$clicks:I,$page:L,$renderContext:R,$frontmatter:z}=h();return F.setup(),(h,M)=>{let N=y,P=_,F=a(`click`);return m(),u(v,p(t(s(g)(s(z),25))),{default:n(()=>[M[5]||=o(`h1`,null,`The other half of the block`,-1),M[6]||=o(`p`,null,[d(`After attention mixes the positions, each position goes through the `),o(`strong`,null,`same small network, alone`),d(`:`)],-1),o(`div`,b,[r(N,{ranges:[]},{default:n(()=>[...M[0]||=[o(`p`,null,[o(`span`,{class:`katex-display`},[o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`,display:`block`},[o(`semantics`,null,[o(`mrow`,null,[o(`mi`,{mathvariant:`normal`},`FFN`),o(`mo`,null,``),o(`mo`,{stretchy:`false`},`(`),o(`mi`,{mathvariant:`bold`},`x`),o(`mo`,{stretchy:`false`},`)`),o(`mo`,null,`=`),o(`msub`,null,[o(`mi`,null,`W`),o(`mn`,null,`2`)]),o(`mtext`,null,` `),o(`mi`,null,`ϕ`),o(`mtext`,null,` `),o(`mrow`,null,[o(`mo`,{fence:`true`},`(`),o(`msub`,null,[o(`mi`,null,`W`),o(`mn`,null,`1`)]),o(`mi`,{mathvariant:`bold`},`x`),o(`mo`,null,`+`),o(`msub`,null,[o(`mi`,{mathvariant:`bold`},`b`),o(`mn`,null,`1`)]),o(`mo`,{fence:`true`},`)`)]),o(`mo`,null,`+`),o(`msub`,null,[o(`mi`,{mathvariant:`bold`},`b`),o(`mn`,null,`2`)])]),o(`annotation`,{encoding:`application/x-tex`},`\\operatorname{FFN}(\\mathbf{x}) = W_2\\,\\phi\\!\\left(W_1\\mathbf{x} + \\mathbf{b}_1\\right) + \\mathbf{b}_2`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),o(`span`,{class:`mop`},[o(`span`,{class:`mord mathrm`},`FFN`)]),o(`span`,{class:`mopen`},`(`),o(`span`,{class:`mord mathbf`},`x`),o(`span`,{class:`mclose`},`)`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mrel`},`=`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3011em`}},[o(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`2`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])]),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mord mathnormal`},`ϕ`),o(`span`,{class:`mspace`,style:{"margin-right":`-0.1667em`}}),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`minner`},[o(`span`,{class:`mopen delimcenter`,style:{top:`0em`}},`(`),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3011em`}},[o(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`1`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])]),o(`span`,{class:`mord mathbf`},`x`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),o(`span`,{class:`mbin`},`+`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathbf`},`b`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3011em`}},[o(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`1`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])]),o(`span`,{class:`mclose delimcenter`,style:{top:`0em`}},`)`)]),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),o(`span`,{class:`mbin`},`+`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathbf`},`b`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3011em`}},[o(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},`2`)])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])])])])])])],-1)]]),_:1})]),o(`div`,x,[o(`div`,null,[r(P,null,{default:n(()=>[...M[1]||=[o(`ul`,null,[o(`li`,null,`Two linear layers and a non-linearity: an MLP (multilayer perceptron)`),o(`li`,null,[o(`strong`,null,`Position-wise`),d(`: no token sees another`)]),o(`li`,null,[d(`Wide in the middle: `),o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`msub`,null,[o(`mi`,null,`d`),o(`mtext`,null,`ff`)]),o(`mo`,null,`=`),o(`mn`,null,`4`),o(`mi`,null,`d`),o(`mo`,null,`=`),o(`mn`,null,`2048`)]),o(`annotation`,{encoding:`application/x-tex`},`d_{\\text{ff}} = 4d = 2048`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),o(`span`,{class:`mord`},[o(`span`,{class:`mord mathnormal`},`d`),o(`span`,{class:`msupsub`},[o(`span`,{class:`vlist-t vlist-t2`},[o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.3361em`}},[o(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[o(`span`,{class:`pstrut`,style:{height:`2.7em`}}),o(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[o(`span`,{class:`mord mtight`},[o(`span`,{class:`mord text mtight`},[o(`span`,{class:`mord mtight`},`ff`)])])])])]),o(`span`,{class:`vlist-s`},``)]),o(`span`,{class:`vlist-r`},[o(`span`,{class:`vlist`,style:{height:`0.15em`}},[o(`span`)])])])])]),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mrel`},`=`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6944em`}}),o(`span`,{class:`mord`},`4`),o(`span`,{class:`mord mathnormal`},`d`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mrel`},`=`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),o(`span`,{class:`mord`},`2048`)])])])])],-1)]]),_:1}),e((m(),c(`div`,S,[...M[2]||=[o(`p`,null,[o(`span`,{class:`katex`},[o(`span`,{class:`katex-mathml`},[o(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[o(`semantics`,null,[o(`mrow`,null,[o(`mn`,null,`2`),o(`mo`,null,`×`),o(`mn`,null,`512`),o(`mo`,null,`×`),o(`mn`,null,`2048`),o(`mo`,null,`=`),o(`mn`,null,`2`),o(`mtext`,null,` `),o(`mn`,null,`097`),o(`mtext`,null,` `),o(`mn`,null,`152`)]),o(`annotation`,{encoding:`application/x-tex`},`2 \\times 512 \\times 2048 = 2\\,097\\,152`)])])]),o(`span`,{class:`katex-html`,"aria-hidden":`true`},[o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.7278em`,"vertical-align":`-0.0833em`}}),o(`span`,{class:`mord`},`2`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),o(`span`,{class:`mbin`},`×`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.7278em`,"vertical-align":`-0.0833em`}}),o(`span`,{class:`mord`},`512`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),o(`span`,{class:`mbin`},`×`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),o(`span`,{class:`mord`},`2048`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),o(`span`,{class:`mrel`},`=`),o(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),o(`span`,{class:`katex-base`},[o(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),o(`span`,{class:`mord`},`2`),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mord`},`097`),o(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),o(`span`,{class:`mord`},`152`)])])]),d(` weights, against 1.05 M for attention: `),o(`strong`,null,`two thirds of the block.`)],-1)]])),[[F]])]),o(`div`,null,[(m(),c(`svg`,C,[M[3]||=o(`defs`,null,[o(`marker`,{id:`l6s2-ffn-arrow`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[o(`path`,{class:`dl-dg-head`,d:`M0 0 L7 3.5 L0 7 z`})])],-1),(m(),c(l,null,i([`the`,`river`,`bank`],(e,t)=>o(`g`,{key:t},[o(`rect`,{class:`dl-dg-box`,x:50+t*200,y:`138`,width:`100`,height:`28`,rx:`4`},null,8,w),o(`text`,{class:`dl-dg-lab is-sm`,x:100+t*200,y:`157`,"text-anchor":`middle`},f(e),9,T),o(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s2-ffn-arrow)`,d:`M`+(100+t*200)+` 136 V120`},null,8,E),o(`polygon`,{class:`dl-dg-net`,points:80+t*200+`,116 `+(120+t*200)+`,116 `+(160+t*200)+`,80 `+(120+t*200)+`,44 `+(80+t*200)+`,44 `+(40+t*200)+`,80`},null,8,D),o(`text`,{class:`dl-dg-in`,x:100+t*200,y:`85`,"text-anchor":`middle`},`FFN`,8,O),o(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s2-ffn-arrow)`,d:`M`+(100+t*200)+` 42 V24`},null,8,k),o(`rect`,{class:`dl-dg-box is-accent`,x:60+t*200,y:`2`,width:`80`,height:`20`,rx:`4`},null,8,A)])),64))])),e((m(),c(`div`,j,[...M[4]||=[o(`p`,null,[d(`Attention decides `),o(`strong`,null,`what to gather`),d(`. This decides `),o(`strong`,null,`what to do with it`),d(`.`)],-1)]])),[[F]])])])]),_:1},16)}}};export{M as default};
|
lecture-06/class/assets/md-CwNbuyVx2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,W as r,_ as i,_t as a,b as o,v as s,x as c,yt as l,z as u}from"./modules/shiki-BFOesOFp.js";import{n as d,t as f}from"./slidev/context-DrkI8qiD.js";import{t as p}from"./default-Cq9fupKf.js";var m={class:`grid grid-cols-2 gap-6 mt-2`},h={viewBox:`0 0 480 140`,class:`dl-diagram`,style:{"max-height":`8rem`},role:`img`,"aria-label":`Each model drawn as a rectangle with width proportional to d_model and height proportional to the number of blocks; the growth is mostly in width`},g={class:`dl-callout`},_={class:`mt-2 dl-secondary`},v={__name:`class.md__slidev_37`,setup(v){let{$slidev:y,$nav:b,$clicksContext:x,$clicks:S,$page:C,$renderContext:w,$frontmatter:T}=d();return x.setup(),(d,v)=>{let y=r(`click`);return u(),s(p,l(t(a(f)(a(T),36))),{default:n(()=>[v[3]||=i(`h1`,null,`The same block, more of it`,-1),v[4]||=i(`div`,{class:`dl-ledger dl-ledger--scale mt-1`},[i(`table`,null,[i(`thead`,null,[i(`tr`,null,[i(`th`),i(`th`,null,`year`),i(`th`,null,`blocks`),i(`th`,null,[i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`msub`,null,[i(`mi`,null,`d`),i(`mtext`,null,`model`)])]),i(`annotation`,{encoding:`application/x-tex`},`d_{\\text{model}}`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.8444em`,"vertical-align":`-0.15em`}}),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathnormal`},`d`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t vlist-t2`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.3361em`}},[i(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mtight`},[i(`span`,{class:`mord text mtight`},[i(`span`,{class:`mord mtight`},`model`)])])])])]),i(`span`,{class:`vlist-s`},``)]),i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.15em`}},[i(`span`)])])])])])])])])]),i(`th`,null,`heads`),i(`th`,null,`context`),i(`th`,null,`parameters`)])]),i(`tbody`,null,[i(`tr`,null,[i(`td`,null,`Transformer base`),i(`td`,null,`2017`),i(`td`,null,`6`),i(`td`,null,`512`),i(`td`,null,`8`),i(`td`,null,`512`),i(`td`,null,`65 M`)]),i(`tr`,null,[i(`td`,null,`BERT base`),i(`td`,null,`2018`),i(`td`,null,`12`),i(`td`,null,`768`),i(`td`,null,`12`),i(`td`,null,`512`),i(`td`,null,`110 M`)]),i(`tr`,null,[i(`td`,null,`GPT-2 XL`),i(`td`,null,`2019`),i(`td`,null,`48`),i(`td`,null,`1600`),i(`td`,null,`25`),i(`td`,null,`1024`),i(`td`,null,`1.5 B`)]),i(`tr`,null,[i(`td`,null,`GPT-3`),i(`td`,null,`2020`),i(`td`,null,`96`),i(`td`,null,`12288`),i(`td`,null,`96`),i(`td`,null,`2048`),i(`td`,null,`175 B`)]),i(`tr`,null,[i(`td`,null,`Llama 3.1 70B`),i(`td`,null,`2024`),i(`td`,null,`80`),i(`td`,null,`8192`),i(`td`,null,`64`),i(`td`,null,`128 k`),i(`td`,null,`70 B`)])])])],-1),i(`div`,m,[i(`div`,null,[(u(),o(`svg`,h,[...v[0]||=[i(`text`,{class:`dl-dg-small`,x:`4`,y:`12`},`width = d_model · height = blocks`,-1),i(`rect`,{class:`dl-dg-fill is-muted`,x:`10`,y:`94.4`,width:`6.2`,height:`5.6`},null,-1),i(`rect`,{class:`dl-dg-fill is-muted`,x:`75`,y:`88.8`,width:`9.4`,height:`11.2`},null,-1),i(`rect`,{class:`dl-dg-fill is-muted`,x:`135`,y:`55`,width:`19.5`,height:`45`},null,-1),i(`rect`,{class:`dl-dg-fill`,x:`190`,y:`10`,width:`150`,height:`90`},null,-1),i(`rect`,{class:`dl-dg-fill`,x:`360`,y:`25`,width:`100`,height:`75`},null,-1),i(`path`,{class:`dl-dg-split`,d:`M4 100 H476`},null,-1),i(`text`,{class:`dl-dg-small`,x:`13`,y:`118`,"text-anchor":`middle`},`2017`,-1),i(`text`,{class:`dl-dg-small`,x:`80`,y:`118`,"text-anchor":`middle`},`BERT`,-1),i(`text`,{class:`dl-dg-small`,x:`145`,y:`118`,"text-anchor":`middle`},`GPT-2 XL`,-1),i(`text`,{class:`dl-dg-small`,x:`265`,y:`118`,"text-anchor":`middle`},`GPT-3`,-1),i(`text`,{class:`dl-dg-small`,x:`410`,y:`118`,"text-anchor":`middle`},`Llama 3.1`,-1)]]))]),i(`div`,null,[e((u(),o(`div`,g,[...v[1]||=[i(`p`,null,[c(`Deeper, wider, longer — and the `),i(`strong`,null,`same block`),c(`.`)],-1)]])),[[y]]),e((u(),o(`div`,_,[...v[2]||=[i(`p`,null,[c(`Context grew 250-fold, so the `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`msup`,null,[i(`mi`,null,`T`),i(`mn`,null,`2`)])]),i(`annotation`,{encoding:`application/x-tex`},`T^2`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.8141em`}}),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.8141em`}},[i(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mtight`},`2`)])])])])])])])])])]),c(` cost grew too. Faster kernels and caching kept it affordable.`)],-1)]])),[[y]])])])]),_:1},16)}}};export{v as default};
|
lecture-06/class/assets/md-Cy5aODuI.js
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,Y as o,_ as s,_t as c,b as l,bt as u,g as d,lt as f,m as p,v as m,vt as h,w as g,x as _,xt as v,y,yt as b,z as x}from"./modules/shiki-BFOesOFp.js";import{t as S}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as C,t as w}from"./slidev/context-DrkI8qiD.js";import{t as T}from"./slidev/VClicks-Dmx1KjZv.js";import{n as E,t as D}from"./interactive-CI8hm4xw.js";import{t as ee}from"./StepButton-BcP33F-1.js";import{t as O}from"./useConvolution-qO1u8ksh.js";import{i as k,u as A}from"./useAttention-Blk0c2BZ.js";var j={class:`dl-att`},te={class:`dl-att__query`},ne=[`onClick`],re={key:0,class:`dl-att__qvec`},M={class:`dl-att__grid`},N={class:`dl-att__given`},P={class:`dl-att__given`},F={key:0,class:`dl-att__step`},I={key:1,class:`dl-att__step`},L={key:2,class:`dl-att__step is-weights`},R={class:`dl-att__weight`},z={class:`dl-att__bar`},B={key:3,class:`dl-att__step`},V={class:`dl-att__outlabel`},H={class:`dl-att__outvec`},U={key:0,class:`dl-att__note`},W=5,G=S(g({__name:`AttentionTrace`,props:{sentence:{default:`river`},query:{default:2},switchable:{type:Boolean,default:!0},causal:{type:Boolean,default:!1}},setup(e){let{$slidev:t,$nav:a,$clicksContext:g,$clicks:b,$page:S,$renderContext:w,$frontmatter:T}=C(),D=e,G=f(D.sentence),K=f(D.query),q=f(0),J=d(()=>A(k[G.value],D.causal)),Y=d(()=>J.value.words),X=d(()=>J.value.rows[Math.min(K.value,Y.value.length-1)]),Z=d(()=>{let e=X.value.weights,t=e.indexOf(Math.max(...e));return{index:t,word:Y.value[t],weight:e[t]}}),Q=d(()=>q.value>=W);function ie(){q.value=Q.value?0:q.value+1}function ae(e){K.value=e,q.value=0}function oe(){G.value=G.value===`river`?`money`:`river`}o(G,()=>{q.value=0});function $(e){return e.toFixed(2)}let se=[`press Next step — the keys and values are already here, the query is what is new`,`the query: one vector, and the only thing about this step that is new`,`score it against every key — one dot product per position, including its own`,`divide by √d_k, so the softmax does not saturate once d_k gets large`,`softmax: the scores become weights that are positive and sum to exactly 1`,`multiply each value by its weight and add them up — that is the new vector`],ce=d(()=>se[q.value]);return(e,t)=>{let a=ee,o=E;return x(),m(o,{"max-width":`44rem`},{controls:n(()=>[r(a,{label:Q.value?`Start over`:`Next step`,glyph:`▸`,onClick:ie},null,8,[`label`]),D.switchable?(x(),m(a,{key:0,label:G.value===`river`?`Swap river → money`:`Swap money → river`,variant:`ghost`,glyph:`⇄`,onClick:oe},null,8,[`label`])):y(`v-if`,!0)]),readout:n(()=>[q.value<W?(x(),l(p,{key:0},[_(` Line `+v(q.value)+` of `+v(W)+` — `+v(ce.value)+`. `,1)],64)):(x(),l(p,{key:1},[s(`strong`,null,v(X.value.word),1),t[15]||=_(` reads `,-1),s(`strong`,null,v(Z.value.word),1),_(` with `+v($(Z.value.weight))+` of its attention, and comes out as [`+v(X.value.out.map(e=>c(O)(e)).join(`, `))+`]. `,1),X.value.word===`bank`?(x(),l(`span`,U,[t[11]||=_(` Press `,-1),t[12]||=s(`em`,null,`Swap`,-1),t[13]||=_(`. The four matrices do not change and neither does the word — only its neighbour does — and `,-1),t[14]||=s(`em`,null,`bank`,-1),_(`’s output flips to [`+v([...X.value.out].reverse().map(e=>c(O)(e)).join(`, `))+`]. That is what “contextual” means, and it is the entire reason this mechanism exists. `,1)])):y(`v-if`,!0)],64))]),default:n(()=>[s(`div`,j,[y(` Who is asking. `),s(`div`,te,[t[0]||=s(`span`,{class:`dl-att__qlabel`},`query`,-1),(x(!0),l(p,null,i(Y.value,(e,t)=>(x(),l(`button`,{key:e+t,type:`button`,class:h([`dl-att__word`,{"is-current":t===K.value}]),onClick:e=>ae(t)},v(e),11,ne))),128)),q.value>=1?(x(),l(`span`,re,` q = [`+v(X.value.q.map(e=>c(O)(e)).join(`, `))+`] `,1)):y(`v-if`,!0)]),s(`table`,M,[s(`thead`,null,[s(`tr`,null,[t[1]||=s(`th`,{class:`dl-att__rowlabel`},null,-1),(x(!0),l(p,null,i(Y.value,(e,t)=>(x(),l(`th`,{key:e+t,class:h([`dl-att__col`,{"is-focus":q.value>=4&&t===Z.value.index}])},v(e),3))),128))])]),s(`tbody`,null,[y(` Present from the start: these are not recomputed per query. `),s(`tr`,N,[t[2]||=s(`th`,{class:`dl-att__rowlabel`},`key`,-1),(x(!0),l(p,null,i(J.value.k,(e,t)=>(x(),l(`td`,{key:`k${t}`},`[`+v(e.map(e=>c(O)(e)).join(`, `))+`]`,1))),128))]),s(`tr`,P,[t[3]||=s(`th`,{class:`dl-att__rowlabel`},`value`,-1),(x(!0),l(p,null,i(J.value.v,(e,t)=>(x(),l(`td`,{key:`v${t}`},`[`+v(e.map(e=>c(O)(e)).join(`, `))+`]`,1))),128))]),q.value>=2?(x(),l(`tr`,F,[t[4]||=s(`th`,{class:`dl-att__rowlabel`},[s(`span`,{class:`dl-att__n`},`1`),_(` q · k`)],-1),(x(!0),l(p,null,i(X.value.raw,(e,t)=>(x(),l(`td`,{key:`s${t}`},v(c(O)(e)),1))),128))])):y(`v-if`,!0),q.value>=3?(x(),l(`tr`,I,[t[5]||=s(`th`,{class:`dl-att__rowlabel`},[s(`span`,{class:`dl-att__n`},`2`),_(` ÷ √d`),s(`sub`,null,`k`)],-1),(x(!0),l(p,null,i(X.value.scaled,(e,t)=>(x(),l(`td`,{key:`c${t}`},v(e===null?`−∞`:c(O)(e)),1))),128))])):y(`v-if`,!0),q.value>=4?(x(),l(`tr`,L,[t[6]||=s(`th`,{class:`dl-att__rowlabel`},[s(`span`,{class:`dl-att__n`},`3`),_(` softmax`)],-1),(x(!0),l(p,null,i(X.value.weights,(e,t)=>(x(),l(`td`,{key:`w${t}`},[s(`div`,R,v($(e)),1),s(`div`,z,[s(`span`,{style:u({width:`${e*100}%`})},null,4)])]))),128))])):y(`v-if`,!0),q.value>=5?(x(),l(`tr`,B,[t[7]||=s(`th`,{class:`dl-att__rowlabel`},[s(`span`,{class:`dl-att__n`},`4`),_(` w · v`)],-1),(x(!0),l(p,null,i(X.value.terms,(e,t)=>(x(),l(`td`,{key:`t${t}`},`[`+v(e.map(e=>c(O)(e)).join(`, `))+`]`,1))),128))])):y(`v-if`,!0)])]),y(` Height reserved whether or not line 5 is showing, so the widget does
|
| 2 |
+
not walk down the slide as the room clicks through it. `),s(`div`,{class:h([`dl-att__out`,{"is-empty":q.value<W}])},[q.value>=W?(x(),l(p,{key:0},[t[8]||=s(`span`,{class:`dl-att__n`},`5`,-1),t[9]||=s(`span`,{class:`dl-att__sum`},`sum`,-1),t[10]||=s(`span`,{class:`dl-att__arrow`},`→`,-1),s(`span`,V,v(X.value.word)+`′ =`,1),s(`span`,H,[(x(!0),l(p,null,i(X.value.out,(e,t)=>(x(),l(`span`,{key:t,class:`dl-att__outcell`},v(c(O)(e)),1))),128))])],64)):y(`v-if`,!0)],2)])]),_:1})}}}),[[`__scopeId`,`data-v-284316ce`]]),K={class:`mt-2 dl-callout`},q={__name:`class.md__slidev_18`,setup(i){let{$slidev:o,$nav:u,$clicksContext:d,$clicks:f,$page:p,$renderContext:h,$frontmatter:g}=C();return d.setup(),(i,o)=>{let u=G,d=T,f=a(`click`);return x(),m(D,b(t(c(w)(c(g),17))),{aside:n(t=>[o[2]||=s(`p`,null,[_(`Walk to line 5, then press `),s(`strong`,null,`Swap river → money`),_(`.`)],-1),r(d,null,{default:n(()=>[...o[0]||=[s(`ul`,null,[s(`li`,null,[_(`The model did not change — same `),s(`span`,{class:`katex`},[s(`span`,{class:`katex-mathml`},[s(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[s(`semantics`,null,[s(`mrow`,null,[s(`msub`,null,[s(`mi`,null,`W`),s(`mi`,null,`Q`)])]),s(`annotation`,{encoding:`application/x-tex`},`W_Q`)])])]),s(`span`,{class:`katex-html`,"aria-hidden":`true`},[s(`span`,{class:`katex-base`},[s(`span`,{class:`katex-strut`,style:{height:`0.9694em`,"vertical-align":`-0.2861em`}}),s(`span`,{class:`mord`},[s(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),s(`span`,{class:`msupsub`},[s(`span`,{class:`vlist-t vlist-t2`},[s(`span`,{class:`vlist-r`},[s(`span`,{class:`vlist`,style:{height:`0.3283em`}},[s(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[s(`span`,{class:`pstrut`,style:{height:`2.7em`}}),s(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[s(`span`,{class:`mord mathnormal mtight`},`Q`)])])]),s(`span`,{class:`vlist-s`},``)]),s(`span`,{class:`vlist-r`},[s(`span`,{class:`vlist`,style:{height:`0.2861em`}},[s(`span`)])])])])])])])]),_(`, `),s(`span`,{class:`katex`},[s(`span`,{class:`katex-mathml`},[s(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[s(`semantics`,null,[s(`mrow`,null,[s(`msub`,null,[s(`mi`,null,`W`),s(`mi`,null,`K`)])]),s(`annotation`,{encoding:`application/x-tex`},`W_K`)])])]),s(`span`,{class:`katex-html`,"aria-hidden":`true`},[s(`span`,{class:`katex-base`},[s(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),s(`span`,{class:`mord`},[s(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),s(`span`,{class:`msupsub`},[s(`span`,{class:`vlist-t vlist-t2`},[s(`span`,{class:`vlist-r`},[s(`span`,{class:`vlist`,style:{height:`0.3283em`}},[s(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[s(`span`,{class:`pstrut`,style:{height:`2.7em`}}),s(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[s(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0715em`}},`K`)])])]),s(`span`,{class:`vlist-s`},``)]),s(`span`,{class:`vlist-r`},[s(`span`,{class:`vlist`,style:{height:`0.15em`}},[s(`span`)])])])])])])])]),_(`, `),s(`span`,{class:`katex`},[s(`span`,{class:`katex-mathml`},[s(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[s(`semantics`,null,[s(`mrow`,null,[s(`msub`,null,[s(`mi`,null,`W`),s(`mi`,null,`V`)])]),s(`annotation`,{encoding:`application/x-tex`},`W_V`)])])]),s(`span`,{class:`katex-html`,"aria-hidden":`true`},[s(`span`,{class:`katex-base`},[s(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),s(`span`,{class:`mord`},[s(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),s(`span`,{class:`msupsub`},[s(`span`,{class:`vlist-t vlist-t2`},[s(`span`,{class:`vlist-r`},[s(`span`,{class:`vlist`,style:{height:`0.3283em`}},[s(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[s(`span`,{class:`pstrut`,style:{height:`2.7em`}}),s(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[s(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.2222em`}},`V`)])])]),s(`span`,{class:`vlist-s`},``)]),s(`span`,{class:`vlist-r`},[s(`span`,{class:`vlist`,style:{height:`0.15em`}},[s(`span`)])])])])])])])])]),s(`li`,null,[s(`em`,null,`bank`),_(` is the same word, with the same embedding`)]),s(`li`,null,[_(`Its output goes `),s(`span`,{class:`katex`},[s(`span`,{class:`katex-mathml`},[s(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[s(`semantics`,null,[s(`mrow`,null,[s(`mo`,{stretchy:`false`},`[`),s(`mn`,null,`1.68`),s(`mo`,{separator:`true`},`,`),s(`mtext`,null,` `),s(`mn`,null,`0.33`),s(`mo`,{stretchy:`false`},`]`),s(`mo`,null,`→`),s(`mo`,{stretchy:`false`},`[`),s(`mn`,null,`0.33`),s(`mo`,{separator:`true`},`,`),s(`mtext`,null,` `),s(`mn`,null,`1.68`),s(`mo`,{stretchy:`false`},`]`)]),s(`annotation`,{encoding:`application/x-tex`},`[1.68,\\, 0.33] \\to [0.33,\\, 1.68]`)])])]),s(`span`,{class:`katex-html`,"aria-hidden":`true`},[s(`span`,{class:`katex-base`},[s(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),s(`span`,{class:`mopen`},`[`),s(`span`,{class:`mord`},`1.68`),s(`span`,{class:`mpunct`},`,`),s(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),s(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),s(`span`,{class:`mord`},`0.33`),s(`span`,{class:`mclose`},`]`),s(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),s(`span`,{class:`mrel`},`→`),s(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),s(`span`,{class:`katex-base`},[s(`span`,{class:`katex-strut`,style:{height:`1em`,"vertical-align":`-0.25em`}}),s(`span`,{class:`mopen`},`[`),s(`span`,{class:`mord`},`0.33`),s(`span`,{class:`mpunct`},`,`),s(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),s(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),s(`span`,{class:`mord`},`1.68`),s(`span`,{class:`mclose`},`]`)])])])])],-1)]]),_:1}),e((x(),l(`div`,K,[...o[1]||=[s(`p`,null,`Sense is not in the word. It is assembled, per sentence, by the layer.`,-1)]])),[[f]])]),default:n(()=>[r(u,{query:2})]),_:1},16)}}};export{q as default};
|
lecture-06/class/assets/md-Cyf0zl3n.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`class.md__slidev_30`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),29))),{default:t(()=>[p[0]||=r(`h1`,null,`Inside the decoder`,-1),p[1]||=r(`p`,null,`Two extras: a mask, and a wire into the encoder.`,-1),r(`div`,f,[n(m,{kind:`decoder`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/class/assets/md-D-S0zhLV.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,W as r,_ as i,_t as a,b as o,v as s,yt as c,z as l}from"./modules/shiki-BFOesOFp.js";import{n as u,t as d}from"./slidev/context-DrkI8qiD.js";import{t as f}from"./default-Cq9fupKf.js";var p={class:`mt-1 flex justify-center`},m={viewBox:`0 0 680 250`,class:`dl-diagram`,style:{"max-width":`36rem`},role:`img`,"aria-label":`One transformer block on the river bank: attention mixes the three positions sideways, then the same small network processes each position alone, and the block repeats N times`},h={class:`mt-1 dl-callout`},g={__name:`class.md__slidev_8`,setup(g){let{$slidev:_,$nav:v,$clicksContext:y,$clicks:b,$page:x,$renderContext:S,$frontmatter:C}=u();return y.setup(),(u,g)=>{let _=r(`click`);return l(),s(f,c(t(a(d)(a(C),7))),{default:n(()=>[g[11]||=i(`h1`,null,`What a transformer is`,-1),g[12]||=i(`p`,null,`A stack of identical blocks. Each block does exactly two things.`,-1),i(`div`,p,[(l(),o(`svg`,m,[g[3]||=i(`defs`,null,[i(`marker`,{id:`l6s0-what-head`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[i(`path`,{class:`dl-dg-head`,d:`M0 0 L7 3.5 L0 7 z`})])],-1),g[4]||=i(`rect`,{class:`dl-dg-box is-accent`,x:`120`,y:`204`,width:`90`,height:`30`,rx:`4`},null,-1),g[5]||=i(`rect`,{class:`dl-dg-box is-accent`,x:`280`,y:`204`,width:`90`,height:`30`,rx:`4`},null,-1),g[6]||=i(`rect`,{class:`dl-dg-box is-accent`,x:`440`,y:`204`,width:`90`,height:`30`,rx:`4`},null,-1),g[7]||=i(`text`,{class:`dl-dg-in`,x:`165`,y:`224`,"text-anchor":`middle`},`the`,-1),g[8]||=i(`text`,{class:`dl-dg-in`,x:`325`,y:`224`,"text-anchor":`middle`},`river`,-1),g[9]||=i(`text`,{class:`dl-dg-in`,x:`485`,y:`224`,"text-anchor":`middle`},`bank`,-1),e((l(),o(`g`,null,[...g[0]||=[i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-what-head)`,d:`M165 202 V174`},null,-1),i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-what-head)`,d:`M325 202 V174`},null,-1),i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-what-head)`,d:`M485 202 V174`},null,-1),i(`rect`,{class:`dl-dg-fill`,x:`110`,y:`130`,width:`430`,height:`42`,rx:`6`},null,-1),i(`path`,{class:`dl-dg-line`,d:`M165 168 L325 134 M325 168 L165 134 M325 168 L485 134 M485 168 L325 134 M165 168 L485 134 M485 168 L165 134`},null,-1),i(`text`,{class:`dl-dg-in`,x:`8`,y:`148`},`sideways`,-1),i(`text`,{class:`dl-dg-small`,x:`8`,y:`164`},`mix positions`,-1)]])),[[_]]),e((l(),o(`g`,null,[...g[1]||=[i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-what-head)`,d:`M165 128 V104`},null,-1),i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-what-head)`,d:`M325 128 V104`},null,-1),i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-what-head)`,d:`M485 128 V104`},null,-1),i(`polygon`,{class:`dl-dg-net`,points:`125,102 205,102 193,58 137,58`},null,-1),i(`polygon`,{class:`dl-dg-net`,points:`285,102 365,102 353,58 297,58`},null,-1),i(`polygon`,{class:`dl-dg-net`,points:`445,102 525,102 513,58 457,58`},null,-1),i(`text`,{class:`dl-dg-small`,x:`165`,y:`85`,"text-anchor":`middle`},`same net`,-1),i(`text`,{class:`dl-dg-small`,x:`325`,y:`85`,"text-anchor":`middle`},`same net`,-1),i(`text`,{class:`dl-dg-small`,x:`485`,y:`85`,"text-anchor":`middle`},`same net`,-1),i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-what-head)`,d:`M165 56 V26`},null,-1),i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-what-head)`,d:`M325 56 V26`},null,-1),i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-what-head)`,d:`M485 56 V26`},null,-1),i(`text`,{class:`dl-dg-in`,x:`8`,y:`76`},`upward`,-1),i(`text`,{class:`dl-dg-small`,x:`8`,y:`92`},`each position alone`,-1)]])),[[_]]),e((l(),o(`g`,null,[...g[2]||=[i(`path`,{class:`dl-dg-line is-muted`,d:`M572 56 H584 V172 H572`},null,-1),i(`text`,{class:`dl-dg-lab`,x:`594`,y:`114`},`× N`,-1),i(`text`,{class:`dl-dg-small`,x:`594`,y:`134`},`6 … 96 blocks`,-1)]])),[[_]])]))]),e((l(),o(`div`,h,[...g[10]||=[i(`p`,null,`Mix across positions, then process each position on its own. Repeat. The rest of the lecture is detail.`,-1)]])),[[_]])]),_:1},16)}}};export{g as default};
|
lecture-06/class/assets/md-D0ZW5T8G.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,H as n,Q as r,S as i,W as a,_ as o,_t as s,b as c,m as l,v as u,w as d,x as f,xt as p,yt as m,z as h}from"./modules/shiki-BFOesOFp.js";import{t as g}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as _,t as v}from"./slidev/context-DrkI8qiD.js";import{t as y}from"./default-Cq9fupKf.js";var b={class:`dl-citation`},x=[`href`],S=g(d({__name:`Citation`,props:{source:{},url:{}},setup(e){let{$slidev:t,$nav:r,$clicksContext:i,$clicks:a,$page:o,$renderContext:s,$frontmatter:u}=_(),d=e;return(e,t)=>(h(),c(`span`,b,[d.url?(h(),c(`a`,{key:0,href:d.url,target:`_blank`,rel:`noopener`},p(d.source),9,x)):(h(),c(l,{key:1},[f(p(d.source),1)],64)),n(e.$slots,`default`,{},void 0,!0)]))}}),[[`__scopeId`,`data-v-11a8cabc`]]),C={class:`mt-1 flex justify-center`},w={viewBox:`0 0 680 200`,class:`dl-diagram`,style:{"max-width":`34rem`},role:`img`,"aria-label":`Left: attention sits on top of a recurrent chain. Right: the chain is gone and the words the, river, bank connect directly through attention`},T={class:`dl-tight`},E={class:`mt-2 dl-callout`},D={class:`mt-1`},O={__name:`class.md__slidev_5`,setup(n){let{$slidev:l,$nav:d,$clicksContext:p,$clicks:g,$page:b,$renderContext:x,$frontmatter:O}=_();return p.setup(),(n,l)=>{let d=S,p=a(`click`);return h(),u(y,m(t(s(v)(s(O),4))),{default:r(()=>[l[21]||=o(`h1`,null,`The move`,-1),l[22]||=o(`p`,null,[f(`Attention began as a `),o(`strong`,null,`passenger`),f(` on a recurrent model that did the real work.`)],-1),o(`div`,C,[(h(),c(`svg`,w,[l[1]||=o(`defs`,null,[o(`marker`,{id:`l6s0-move-head`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[o(`path`,{class:`dl-dg-head`,d:`M0 0 L7 3.5 L0 7 z`})])],-1),l[2]||=o(`text`,{class:`dl-dg-small`,x:`8`,y:`16`},`before — attention on top`,-1),l[3]||=o(`circle`,{class:`dl-dg-op`,cx:`95`,cy:`48`,r:`16`},null,-1),l[4]||=o(`text`,{class:`dl-dg-lab is-sm`,x:`95`,y:`53`,"text-anchor":`middle`},`Σ`,-1),l[5]||=o(`text`,{class:`dl-dg-small`,x:`120`,y:`52`},`weighted sum`,-1),l[6]||=o(`path`,{class:`dl-dg-line`,d:`M85 61 L31 128`},null,-1),l[7]||=o(`path`,{class:`dl-dg-line`,d:`M95 64 L99 128`},null,-1),l[8]||=o(`path`,{class:`dl-dg-line`,d:`M105 61 L167 128`},null,-1),l[9]||=o(`rect`,{class:`dl-dg-box`,x:`8`,y:`130`,width:`46`,height:`30`,rx:`4`},null,-1),l[10]||=o(`rect`,{class:`dl-dg-box`,x:`76`,y:`130`,width:`46`,height:`30`,rx:`4`},null,-1),l[11]||=o(`rect`,{class:`dl-dg-box`,x:`144`,y:`130`,width:`46`,height:`30`,rx:`4`},null,-1),l[12]||=o(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-move-head)`,d:`M54 145 H74`},null,-1),l[13]||=o(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-move-head)`,d:`M122 145 H142`},null,-1),l[14]||=o(`text`,{class:`dl-dg-small`,x:`31`,y:`184`,"text-anchor":`middle`},`the`,-1),l[15]||=o(`text`,{class:`dl-dg-small`,x:`99`,y:`184`,"text-anchor":`middle`},`river`,-1),l[16]||=o(`text`,{class:`dl-dg-small`,x:`167`,y:`184`,"text-anchor":`middle`},`bank`,-1),l[17]||=o(`text`,{class:`dl-dg-small`,x:`200`,y:`150`},`the chain does the work`,-1),l[18]||=o(`line`,{class:`dl-dg-split`,x1:`340`,y1:`8`,x2:`340`,y2:`192`},null,-1),e((h(),c(`g`,null,[...l[0]||=[o(`text`,{class:`dl-dg-small is-good`,x:`360`,y:`16`},`2017 — keep the weighted sum, delete the chain`,-1),o(`path`,{class:`dl-dg-line`,d:`M415 128 Q460 92 505 128`},null,-1),o(`path`,{class:`dl-dg-line`,d:`M505 128 Q550 92 595 128`},null,-1),o(`path`,{class:`dl-dg-line`,d:`M415 128 Q505 46 595 128`},null,-1),o(`rect`,{class:`dl-dg-box is-accent`,x:`380`,y:`130`,width:`70`,height:`30`,rx:`4`},null,-1),o(`rect`,{class:`dl-dg-box is-accent`,x:`470`,y:`130`,width:`70`,height:`30`,rx:`4`},null,-1),o(`rect`,{class:`dl-dg-box is-accent`,x:`560`,y:`130`,width:`70`,height:`30`,rx:`4`},null,-1),o(`text`,{class:`dl-dg-in`,x:`415`,y:`150`,"text-anchor":`middle`},`the`,-1),o(`text`,{class:`dl-dg-in`,x:`505`,y:`150`,"text-anchor":`middle`},`river`,-1),o(`text`,{class:`dl-dg-in`,x:`595`,y:`150`,"text-anchor":`middle`},`bank`,-1),o(`text`,{class:`dl-dg-small`,x:`505`,y:`184`,"text-anchor":`middle`},`all positions at once — nothing waits`,-1)]])),[[p]])]))]),e((h(),c(`div`,T,[...l[19]||=[o(`ul`,null,[o(`li`,null,[f(`They removed the RNN (recurrent neural network) as a test. Translation quality went `),o(`strong`,null,`up`),f(`.`)])],-1)]])),[[p]]),e((h(),c(`div`,E,[...l[20]||=[o(`p`,null,[o(`strong`,null,`Attention Is All You Need`),f(` (Vaswani et al., 2017): keep the weighted sum, delete everything it was bolted to.`)],-1)]])),[[p]]),o(`div`,D,[i(d,{source:`Vaswani, Shazeer, Parmar, Uszkoreit, Jones, Gomez, Kaiser & Polosukhin, Attention Is All You Need (2017)`,url:`https://arxiv.org/abs/1706.03762`})])]),_:1},16)}}};export{O as default};
|
lecture-06/class/assets/md-D3MJsr7m.js
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,x as l,yt as u,z as d}from"./modules/shiki-BFOesOFp.js";import{n as f,t as p}from"./slidev/context-DrkI8qiD.js";import{t as m}from"./slidev/VClicks-Dmx1KjZv.js";import{t as h}from"./default-Cq9fupKf.js";import{t as g}from"./slidev/CodeBlockWrapper-gsL5r1Ig.js";var _={class:`grid grid-cols-[1fr_1.05fr] gap-8 mt-1 items-start`},v={class:`dl-tight`},y={viewBox:`0 0 320 232`,class:`dl-diagram`,style:{"max-height":`16rem`},role:`img`,"aria-label":`Top: a dictionary query river matches one key exactly and returns one value. Bottom: an attention query is compared with every key, and every value is blended by how well its key matched`},b={class:`mt-2 dl-callout`},x={__name:`class.md__slidev_13`,setup(x){let{$slidev:S,$nav:C,$clicksContext:w,$clicks:T,$page:E,$renderContext:D,$frontmatter:O}=f();return w.setup(),(f,x)=>{let S=g,C=m,w=i(`click`);return d(),c(h,u(t(o(p)(o(O),12))),{default:n(()=>[x[18]||=a(`h1`,null,`Start from a lookup`,-1),a(`div`,_,[a(`div`,v,[x[2]||=a(`p`,null,[l(`A Python dictionary finds the `),a(`strong`,null,`exact`),l(` key and hands back its value.`)],-1),r(S,{title:``,ranges:[]},{default:n(()=>[...x[0]||=[a(`pre`,{class:`shiki shiki-themes vitesse-dark vitesse-light slidev-code`,style:{"--shiki-dark":`#dbd7caee`,"--shiki-light":`#393a34`,"--shiki-dark-bg":`#121212`,"--shiki-light-bg":`#ffffff`}},[a(`code`,{class:`language-python`},[a(`span`,{class:`line`},[a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`store `),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},` {`),a(`span`,{style:{"--shiki-dark":`#C98A7D77`,"--shiki-light":`#B5695977`}},`"`),a(`span`,{style:{"--shiki-dark":`#C98A7D`,"--shiki-light":`#B56959`}},`river`),a(`span`,{style:{"--shiki-dark":`#C98A7D77`,"--shiki-light":`#B5695977`}},`"`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`:`),a(`span`,{style:{"--shiki-dark":`#C98A7D77`,"--shiki-light":`#B5695977`}},` "`),a(`span`,{style:{"--shiki-dark":`#C98A7D`,"--shiki-light":`#B56959`}},`waterside`),a(`span`,{style:{"--shiki-dark":`#C98A7D77`,"--shiki-light":`#B5695977`}},`"`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),a(`span`,{style:{"--shiki-dark":`#C98A7D77`,"--shiki-light":`#B5695977`}},` "`),a(`span`,{style:{"--shiki-dark":`#C98A7D`,"--shiki-light":`#B56959`}},`money`),a(`span`,{style:{"--shiki-dark":`#C98A7D77`,"--shiki-light":`#B5695977`}},`"`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`:`),a(`span`,{style:{"--shiki-dark":`#C98A7D77`,"--shiki-light":`#B5695977`}},` "`),a(`span`,{style:{"--shiki-dark":`#C98A7D`,"--shiki-light":`#B56959`}},`financial`),a(`span`,{style:{"--shiki-dark":`#C98A7D77`,"--shiki-light":`#B5695977`}},`"`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`}`)]),l(`
|
| 2 |
+
`),a(`span`,{class:`line`},[a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`store`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`[`),a(`span`,{style:{"--shiki-dark":`#C98A7D77`,"--shiki-light":`#B5695977`}},`"`),a(`span`,{style:{"--shiki-dark":`#C98A7D`,"--shiki-light":`#B56959`}},`river`),a(`span`,{style:{"--shiki-dark":`#C98A7D77`,"--shiki-light":`#B5695977`}},`"`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`]`),a(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # → 'waterside' one winner`)]),l(`
|
| 3 |
+
`),a(`span`,{class:`line`},[a(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`store`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`[`),a(`span`,{style:{"--shiki-dark":`#C98A7D77`,"--shiki-light":`#B5695977`}},`"`),a(`span`,{style:{"--shiki-dark":`#C98A7D`,"--shiki-light":`#B56959`}},`stream`),a(`span`,{style:{"--shiki-dark":`#C98A7D77`,"--shiki-light":`#B5695977`}},`"`),a(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`]`),a(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # → KeyError nothing at all`)])])],-1)]]),_:1}),r(C,null,{default:n(()=>[...x[1]||=[a(`ul`,null,[a(`li`,null,[l(`What you bring: the `),a(`strong`,null,`query`)]),a(`li`,null,[l(`What each entry advertises: its `),a(`strong`,null,`key`)]),a(`li`,null,[l(`What you get back: its `),a(`strong`,null,`value`)])],-1)]]),_:1})]),a(`div`,null,[(d(),s(`svg`,y,[x[4]||=a(`defs`,null,[a(`marker`,{id:`l6s1-lookup`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[a(`path`,{class:`dl-dg-head`,d:`M0 0 L7 3.5 L0 7 z`})])],-1),x[5]||=a(`text`,{class:`dl-dg-small`,x:`4`,y:`14`},`dictionary`,-1),x[6]||=a(`rect`,{class:`dl-dg-box`,x:`4`,y:`34`,width:`64`,height:`26`,rx:`4`},null,-1),x[7]||=a(`text`,{class:`dl-dg-lab is-sm`,x:`36`,y:`52`,"text-anchor":`middle`,style:{"font-style":`italic`}},`river`,-1),x[8]||=a(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s1-lookup)`,d:`M70 40 L110 35`},null,-1),x[9]||=a(`rect`,{class:`dl-dg-box is-accent`,x:`114`,y:`24`,width:`64`,height:`22`,rx:`3`},null,-1),x[10]||=a(`text`,{class:`dl-dg-small`,x:`146`,y:`39`,"text-anchor":`middle`},`river`,-1),x[11]||=a(`rect`,{class:`dl-dg-box`,x:`114`,y:`52`,width:`64`,height:`22`,rx:`3`},null,-1),x[12]||=a(`text`,{class:`dl-dg-small`,x:`146`,y:`67`,"text-anchor":`middle`},`money`,-1),x[13]||=a(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s1-lookup)`,d:`M180 35 H218`},null,-1),x[14]||=a(`rect`,{class:`dl-dg-box is-accent`,x:`222`,y:`24`,width:`90`,height:`22`,rx:`3`},null,-1),x[15]||=a(`text`,{class:`dl-dg-small`,x:`267`,y:`39`,"text-anchor":`middle`},`waterside`,-1),x[16]||=a(`text`,{class:`dl-dg-small`,x:`114`,y:`92`},`exact match: one winner`,-1),e((d(),s(`g`,null,[...x[3]||=[a(`line`,{class:`dl-dg-split`,x1:`4`,y1:`104`,x2:`316`,y2:`104`},null,-1),a(`text`,{class:`dl-dg-small is-good`,x:`4`,y:`122`},`attention`,-1),a(`path`,{class:`dl-dg-line is-muted`,d:`M40 176 L116 145`,style:{strokeWidth:1.5}},null,-1),a(`path`,{class:`dl-dg-line`,d:`M40 176 L116 173`,style:{strokeWidth:6}},null,-1),a(`path`,{class:`dl-dg-line is-muted`,d:`M40 176 L116 201`,style:{strokeWidth:2.5}},null,-1),a(`circle`,{class:`dl-dg-dot is-accent`,cx:`30`,cy:`176`,r:`12`},null,-1),a(`text`,{x:`30`,y:`181`,"text-anchor":`middle`,style:{fill:`var(--dl-bg)`,"font-size":`13px`,"font-weight":`700`}},`q`,-1),a(`rect`,{class:`dl-dg-box`,x:`116`,y:`134`,width:`50`,height:`22`,rx:`3`},null,-1),a(`rect`,{class:`dl-dg-box is-accent`,x:`116`,y:`162`,width:`50`,height:`22`,rx:`3`},null,-1),a(`rect`,{class:`dl-dg-box`,x:`116`,y:`190`,width:`50`,height:`22`,rx:`3`},null,-1),a(`text`,{class:`dl-dg-small`,x:`141`,y:`149`,"text-anchor":`middle`},`key 1`,-1),a(`text`,{class:`dl-dg-small`,x:`141`,y:`177`,"text-anchor":`middle`},`key 2`,-1),a(`text`,{class:`dl-dg-small`,x:`141`,y:`205`,"text-anchor":`middle`},`key 3`,-1),a(`rect`,{class:`dl-dg-box`,x:`176`,y:`134`,width:`54`,height:`22`,rx:`3`},null,-1),a(`rect`,{class:`dl-dg-box is-accent`,x:`176`,y:`162`,width:`54`,height:`22`,rx:`3`},null,-1),a(`rect`,{class:`dl-dg-box`,x:`176`,y:`190`,width:`54`,height:`22`,rx:`3`},null,-1),a(`text`,{class:`dl-dg-small`,x:`203`,y:`149`,"text-anchor":`middle`},`value 1`,-1),a(`text`,{class:`dl-dg-small`,x:`203`,y:`177`,"text-anchor":`middle`},`value 2`,-1),a(`text`,{class:`dl-dg-small`,x:`203`,y:`205`,"text-anchor":`middle`},`value 3`,-1),a(`path`,{class:`dl-dg-line is-muted`,d:`M230 145 L262 170`,style:{strokeWidth:1.5}},null,-1),a(`path`,{class:`dl-dg-line`,d:`M230 173 L262 173`,style:{strokeWidth:6}},null,-1),a(`path`,{class:`dl-dg-line is-muted`,d:`M230 201 L262 176`,style:{strokeWidth:2.5}},null,-1),a(`rect`,{class:`dl-dg-box is-accent`,x:`262`,y:`160`,width:`52`,height:`26`,rx:`4`},null,-1),a(`text`,{class:`dl-dg-small`,x:`288`,y:`177`,"text-anchor":`middle`},`blend`,-1),a(`text`,{class:`dl-dg-small`,x:`116`,y:`228`},`similarity: every value, weighted`,-1)]])),[[w]])]))])]),e((d(),s(`div`,b,[...x[17]||=[a(`p`,null,[l(`Attention is this lookup made `),a(`strong`,null,`soft`),l(`: it matches by `),a(`em`,null,`similarity`),l(`, and returns a blend of `),a(`strong`,null,`every`),l(` value.`)],-1)]])),[[w]])]),_:1},16)}}};export{x as default};
|
lecture-06/class/assets/md-D4Ywsi6K.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,x as l,yt as u,z as d}from"./modules/shiki-BFOesOFp.js";import{n as f,t as p}from"./slidev/context-DrkI8qiD.js";import{t as m}from"./slidev/VClicks-Dmx1KjZv.js";import{t as h}from"./interactive-CI8hm4xw.js";import{t as g}from"./AttentionMatrix-D2xrh-OL.js";var _={class:`mt-2 dl-callout`},v={__name:`class.md__slidev_31`,setup(v){let{$slidev:y,$nav:b,$clicksContext:x,$clicks:S,$page:C,$renderContext:w,$frontmatter:T}=f();return x.setup(),(f,v)=>{let y=g,b=m,x=i(`click`);return d(),c(h,u(t(o(p)(o(T),30))),{aside:n(t=>[v[2]||=a(`p`,null,[l(`Position `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`mi`,null,`t`)]),a(`annotation`,{encoding:`application/x-tex`},`t`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.6151em`}}),a(`span`,{class:`mord mathnormal`},`t`)])])]),l(` may read positions `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`mn`,null,`1`),a(`mo`,null,`…`),a(`mi`,null,`t`)]),a(`annotation`,{encoding:`application/x-tex`},`1 \\ldots t`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.6444em`}}),a(`span`,{class:`mord`},`1`),a(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),a(`span`,{class:`minner`},`…`),a(`span`,{class:`mspace`,style:{"margin-right":`0.1667em`}}),a(`span`,{class:`mord mathnormal`},`t`)])])]),l(` only.`)],-1),r(b,null,{default:n(()=>[...v[0]||=[a(`ul`,null,[a(`li`,null,[l(`Row `),a(`em`,null,`the`),l(` becomes `),a(`strong`,null,`1.00`),l(`: it can only see itself`)]),a(`li`,null,[l(`Rows still sum to 1: the softmax comes `),a(`strong`,null,`after`),l(` the `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`mo`,null,`−`),a(`mi`,{mathvariant:`normal`},`∞`)]),a(`annotation`,{encoding:`application/x-tex`},`-\\infty`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`0.6667em`,"vertical-align":`-0.0833em`}}),a(`span`,{class:`mord`},`−`),a(`span`,{class:`mord`},`∞`)])])])]),a(`li`,null,`The button takes the mask off and puts it back`)],-1)]]),_:1}),e((d(),s(`div`,_,[...v[1]||=[a(`p`,null,`Without the mask, the model reads the answer off its own input.`,-1)]])),[[x]])]),default:n(()=>[r(y,{causal:!0})]),_:1},16)}}};export{v as default};
|
lecture-06/class/assets/md-DAJ_ecr1.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,H as t,Q as n,S as r,_ as i,_t as a,b as o,v as s,w as c,x as l,xt as u,y as d,yt as f,z as p}from"./modules/shiki-BFOesOFp.js";import{t as m}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as h,t as g}from"./slidev/context-DrkI8qiD.js";import{t as _}from"./TransformerGlyph-D9TehqNe.js";var v={class:`slidev-layout dl-title`},y={class:`dl-title__eyebrow`},b={key:0,class:`dl-title__code`},x={key:1,class:`dl-title__lecture`},S={class:`dl-title__body`},C={class:`dl-title__meta`},w={key:0,class:`dl-title__author`},T=[`href`],E={key:2,class:`dl-title__date`},D=m(c({__name:`title`,props:{courseCode:{},lecture:{},author:{},email:{},date:{}},setup(e){let{$slidev:n,$nav:r,$clicksContext:a,$clicks:s,$page:c,$renderContext:l,$frontmatter:f}=h(),m=e;return(e,n)=>(p(),o(`div`,v,[i(`div`,y,[m.courseCode?(p(),o(`span`,b,u(m.courseCode),1)):d(`v-if`,!0),m.lecture?(p(),o(`span`,x,`Lecture `+u(m.lecture),1)):d(`v-if`,!0)]),i(`div`,S,[t(e.$slots,`default`,{},void 0,!0)]),n[0]||=i(`div`,{class:`dl-rule dl-title__rule`},null,-1),i(`div`,C,[m.author?(p(),o(`div`,w,u(m.author),1)):d(`v-if`,!0),m.email?(p(),o(`a`,{key:1,href:`mailto:${m.email}`,class:`dl-title__email`},u(m.email),9,T)):d(`v-if`,!0),m.date?(p(),o(`div`,E,u(m.date),1)):d(`v-if`,!0)])]))}}),[[`__scopeId`,`data-v-dc59a472`]]),O={class:`mt-6 flex items-center gap-6`},k={__name:`class.md__slidev_1`,setup(t){let{$slidev:o,$nav:c,$clicksContext:u,$clicks:d,$page:m,$renderContext:v,$frontmatter:y}=h();return u.setup(),(t,o)=>{let c=_;return p(),s(D,f(e(a(g)(a(y),0))),{default:n(()=>[o[3]||=i(`h1`,null,`Transformers`,-1),o[4]||=i(`p`,null,`A network that reads a sentence one word at a time is slow to train and forgets the start. Let every word look at every other word at once instead. Every large language model since 2017 is built on that idea.`,-1),i(`div`,O,[r(c,{kind:`cut`,size:110},{default:n(()=>[...o[0]||=[l(`read in order`,-1)]]),_:1}),o[2]||=i(`span`,{class:`text-3xl`,style:{color:`var(--dl-accent)`}},`→`,-1),r(c,{kind:`block`,size:110},{default:n(()=>[...o[1]||=[l(`look everywhere`,-1)]]),_:1})])]),_:1},16)}}};export{k as default};
|
lecture-06/class/assets/md-DCF9sEzr2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,W as r,_ as i,_t as a,b as o,v as s,x as c,yt as l,z as u}from"./modules/shiki-BFOesOFp.js";import{n as d,t as f}from"./slidev/context-DrkI8qiD.js";import{t as p}from"./default-Cq9fupKf.js";var m={class:`mt-2 dl-callout`},h={__name:`class.md__slidev_40`,setup(h){let{$slidev:g,$nav:_,$clicksContext:v,$clicks:y,$page:b,$renderContext:x,$frontmatter:S}=d();return v.setup(),(d,h)=>{let g=r(`click`);return u(),s(p,l(t(a(f)(a(S),39))),{default:n(()=>[h[1]||=i(`h1`,null,`Shapes, end to end`,-1),h[2]||=i(`div`,{class:`grid grid-cols-[1fr_12rem] gap-6 items-center`},[i(`div`,{class:`dl-ledger`},[i(`table`,null,[i(`thead`,null,[i(`tr`,null,[i(`th`,null,`step`),i(`th`,null,`shape`),i(`th`,null,`what it is`)])]),i(`tbody`,null,[i(`tr`,null,[i(`td`,null,[i(`code`,null,`ids`)]),i(`td`,null,[i(`code`,null,`(32, 128)`)]),i(`td`,null,`token ids, padded`)]),i(`tr`,null,[i(`td`,null,[i(`code`,null,`embed(ids)`),c(` + position`)]),i(`td`,null,[i(`code`,null,`(32, 128, 512)`)]),i(`td`,null,`added: same width`)]),i(`tr`,null,[i(`td`,null,[i(`code`,null,`qkv(x)`)]),i(`td`,null,[i(`code`,null,`(32, 128, 1536)`)]),i(`td`,null,`Q, K, V in one tensor`)]),i(`tr`,null,[i(`td`,null,`reshape`),i(`td`,null,[i(`code`,null,`(32, 8, 128, 64)`)]),i(`td`,null,`heads as batch`)]),i(`tr`,null,[i(`td`,null,[i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`mi`,null,`Q`),i(`msup`,null,[i(`mi`,null,`K`),i(`mi`,{mathvariant:`normal`},`⊤`)])]),i(`annotation`,{encoding:`application/x-tex`},`QK^\\top`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`1.0435em`,"vertical-align":`-0.1944em`}}),i(`span`,{class:`mord mathnormal`},`Q`),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.0715em`}},`K`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.8491em`}},[i(`span`,{style:{top:`-3.063em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mtight`},`⊤`)])])])])])])])])])])]),i(`td`,null,[i(`code`,null,`(32, 8, 128, 128)`)]),i(`td`,null,[i(`strong`,null,`the scores`)])]),i(`tr`,null,[i(`td`,null,`weights @ V`),i(`td`,null,[i(`code`,null,`(32, 8, 128, 64)`)]),i(`td`,null,`output per head`)]),i(`tr`,null,[i(`td`,null,[c(`merged + `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`msub`,null,[i(`mi`,null,`W`),i(`mi`,null,`O`)])]),i(`annotation`,{encoding:`application/x-tex`},`W_O`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t vlist-t2`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.3283em`}},[i(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0278em`}},`O`)])])]),i(`span`,{class:`vlist-s`},``)]),i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.15em`}},[i(`span`)])])])])])])])])]),i(`td`,null,[i(`code`,null,`(32, 128, 512)`)]),i(`td`,null,`residual stream`)]),i(`tr`,null,[i(`td`,null,[i(`code`,null,`head(x)`)]),i(`td`,null,[i(`code`,null,`(32, 128, 50257)`)]),i(`td`,null,`logit per vocabulary entry`)])])])]),i(`div`,null,[i(`svg`,{class:`dl-diagram`,viewBox:`0 0 200 224`,role:`img`,"aria-label":`The scores tensor drawn to scale by area. At 128 tokens it is a tiny square of 4.2 million numbers in the corner. At 2048 tokens it is the large square, 1.07 billion numbers, 256 times the area.`},[i(`rect`,{x:`20`,y:`20`,width:`160`,height:`160`,class:`dl-dg-fill is-muted is-dashed`}),i(`rect`,{x:`20`,y:`20`,width:`10`,height:`10`,class:`dl-dg-fill`}),i(`text`,{x:`20`,y:`13`,class:`dl-dg-small is-good`},`T = 128: 4.2 M (corner)`),i(`text`,{x:`100`,y:`198`,class:`dl-dg-small`,"text-anchor":`middle`},`T = 2048: 1.07 billion`)])])],-1),e((u(),o(`div`,m,[...h[0]||=[i(`p`,null,[c(`Only the scores grow with `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`mi`,null,`T`)]),i(`annotation`,{encoding:`application/x-tex`},`T`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`)])])]),c(),i(`strong`,null,`twice`),c(`. That row is the long-context problem.`)],-1)]])),[[g]])]),_:1},16)}}};export{h as default};
|
lecture-06/class/assets/md-DOad-2KZ.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,bt as l,g as u,lt as d,m as f,v as p,vt as m,w as h,x as g,xt as _,y as v,yt as y,z as b}from"./modules/shiki-BFOesOFp.js";import{t as x}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as S,t as C}from"./slidev/context-DrkI8qiD.js";import{t as w}from"./slidev/VClicks-Dmx1KjZv.js";import{n as T,t as E}from"./interactive-CI8hm4xw.js";import{t as D}from"./StepButton-BcP33F-1.js";import{t as O}from"./useConvolution-qO1u8ksh.js";import{n as k,r as A}from"./useAttention-Blk0c2BZ.js";var j={class:`dl-mh`},M={class:`dl-mh__sentence`},N=[`onClick`],P={class:`dl-mh__rows`},F=[`onClick`],I={class:`dl-mh__headlabel`},L={class:`dl-mh__headreads`},R={class:`dl-mh__bars`},z=[`title`],B={class:`dl-mh__reads`},V={class:`dl-mh__concat`},H={class:`dl-mh__whole`},U={class:`dl-mh__whole is-out`},W=x(h({__name:`MultiHeadLab`,props:{query:{default:6},dModel:{default:512}},setup(e){let{$slidev:t,$nav:a,$clicksContext:h,$clicks:y,$page:x,$renderContext:C,$frontmatter:w}=S(),E=e,W=d(E.query),G=d(null),K=A,q=u(()=>k.map((e,t)=>({...e,index:t,weights:e.weights[W.value],top:e.weights[W.value].indexOf(Math.max(...e.weights[W.value]))}))),J=u(()=>G.value===null?q.value:[q.value[G.value]]),Y=u(()=>Math.round(E.dModel/k.length));function X(){W.value=(W.value+1)%K.length}function Z(e){return e<=W.value}return(e,t)=>{let a=D,u=T;return b(),p(u,{"max-width":`45rem`},{controls:n(()=>[r(a,{label:`Next query word`,glyph:`▸`,onClick:X}),G.value===null?v(`v-if`,!0):(b(),p(a,{key:0,label:`All heads`,variant:`ghost`,glyph:`▦`,onClick:t[0]||=e=>G.value=null}))]),readout:n(()=>[W.value===6?(b(),c(f,{key:0},[t[5]||=o(`strong`,null,`it`,-1),t[6]||=g(` asks four questions at once: head 1 takes the word before it, head 2 the nearby nouns, head 3 goes back five words to `,-1),t[7]||=o(`strong`,null,`cat`,-1),t[8]||=g(` — what `,-1),t[9]||=o(`em`,null,`it`,-1),t[10]||=g(` refers to — and head 4 parks on the first token. One softmax puts its mass in one place, so one head could answer only one of those. `,-1),t[11]||=o(`span`,{class:`dl-mh__note`},[g(` Head 4 is not a drawing error: heads that sit almost entirely on the first token turn up in every trained model. Nothing after `),o(`em`,null,`it`),g(` is drawn because this is a decoder, which may not look forward — section 03. `)],-1)],64)):(b(),c(f,{key:1},[o(`strong`,null,_(s(K)[W.value]),1),t[12]||=g(` asking, four heads answering differently: `,-1),(b(!0),c(f,null,i(q.value,(e,t)=>(b(),c(f,{key:e.name},[t>0?(b(),c(f,{key:0},[g(`, `)],64)):v(`v-if`,!0),g(` `+_(e.name)+` → `,1),o(`strong`,null,_(s(K)[e.top]),1)],64))),128)),t[13]||=g(`. Each head is its own W`,-1),t[14]||=o(`sub`,null,`Q`,-1),t[15]||=g(`, W`,-1),t[16]||=o(`sub`,null,`K`,-1),t[17]||=g(`, W`,-1),t[18]||=o(`sub`,null,`V`,-1),g(` at width `+_(Y.value)+` = `+_(E.dModel)+`/`+_(s(k).length)+`, so all `+_(s(k).length)+` together cost exactly what one head of width `+_(E.dModel)+` would. `,1)],64))]),default:n(()=>[o(`div`,j,[v(` The sentence, and which word is asking. `),o(`div`,M,[t[1]||=o(`span`,{class:`dl-mh__label`},`query`,-1),(b(!0),c(f,null,i(s(K),(e,t)=>(b(),c(`button`,{key:e+t,type:`button`,class:m([`dl-mh__word`,{"is-current":t===W.value}]),onClick:e=>W.value=t},_(e),11,N))),128))]),v(` One row per head: the same question, four different answers. `),o(`div`,P,[(b(!0),c(f,null,i(J.value,e=>(b(),c(`div`,{key:e.name,class:`dl-mh__row`},[o(`button`,{type:`button`,class:m([`dl-mh__headname`,{"is-solo":G.value===e.index}]),onClick:t=>G.value=G.value===e.index?null:e.index},[o(`span`,I,_(e.name),1),o(`span`,L,_(e.reads),1)],10,F),o(`div`,R,[(b(!0),c(f,null,i(e.weights,(t,n)=>(b(),c(`div`,{key:n,class:m([`dl-mh__bar`,{"is-top":n===e.top&&Z(n),"is-dead":!Z(n)}]),title:`${s(K)[n]}: ${s(O)(t)}`},[o(`span`,{class:`dl-mh__fill`,style:l({height:`${Z(n)?t*100:0}%`})},null,4)],10,z))),128))]),o(`div`,B,[e.weights[e.top]>.4?(b(),c(f,{key:0},[t[2]||=g(`→ `,-1),o(`strong`,null,_(s(K)[e.top]),1)],64)):(b(),c(f,{key:1},[g(`→ spread`)],64))])]))),128))]),v(` What happens to the four outputs. `),o(`div`,V,[(b(!0),c(f,null,i(q.value,e=>(b(),c(`span`,{key:`c${e.index}`,class:`dl-mh__slice`},_(Y.value),1))),128)),t[3]||=o(`span`,{class:`dl-mh__op`},`concat →`,-1),o(`span`,H,_(E.dModel),1),t[4]||=o(`span`,{class:`dl-mh__op`},[g(`× W`),o(`sub`,null,`O`),g(` →`)],-1),o(`span`,U,_(E.dModel),1)])])]),_:1})}}}),[[`__scopeId`,`data-v-d4d1022e`]]),G={class:`mt-2 dl-callout`},K={__name:`class.md__slidev_23`,setup(i){let{$slidev:l,$nav:u,$clicksContext:d,$clicks:f,$page:m,$renderContext:h,$frontmatter:_}=S();return d.setup(),(i,l)=>{let u=W,d=w,f=a(`click`);return b(),p(E,y(t(s(C)(s(_),22))),{aside:n(t=>[l[2]||=o(`p`,null,[o(`em`,null,`it`),g(` is asking. Four heads, four different answers.`)],-1),r(d,null,{default:n(()=>[...l[0]||=[o(`ul`,null,[o(`li`,null,[o(`strong`,null,`head 1`),g(` — the previous word`)]),o(`li`,null,[o(`strong`,null,`head 3`),g(` — five words back, to `),o(`em`,null,`cat`)]),o(`li`,null,[o(`strong`,null,`head 4`),g(` — parks on token 1 and does nothing`)])],-1)]]),_:1}),e((b(),c(`div`,G,[...l[1]||=[o(`p`,null,`Head 4 is not a drawing error. Nearly-dead heads are in every model anyone has looked inside.`,-1)]])),[[f]])]),default:n(()=>[r(u)]),_:1},16)}}};export{K as default};
|
lecture-06/class/assets/md-DRRRjQ4b.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,W as r,_ as i,_t as a,b as o,v as s,x as c,yt as l,z as u}from"./modules/shiki-BFOesOFp.js";import{n as d,t as f}from"./slidev/context-DrkI8qiD.js";import{t as p}from"./default-Cq9fupKf.js";var m={class:`mt-2 flex justify-center`},h={viewBox:`0 0 520 196`,class:`dl-diagram`,style:{"max-height":`14rem`},role:`img`,"aria-label":`One token vector x passes through three learned matrices W_Q, W_K and W_V and becomes a query, a key and a value`},g={class:`mt-2 dl-callout`},_={__name:`class.md__slidev_14`,setup(_){let{$slidev:v,$nav:y,$clicksContext:b,$clicks:x,$page:S,$renderContext:C,$frontmatter:w}=d();return b.setup(),(d,_)=>{let v=r(`click`);return u(),s(p,l(t(a(f)(a(w),13))),{default:n(()=>[_[2]||=i(`h1`,null,`Three roles, one vector`,-1),_[3]||=i(`p`,null,[c(`Each token makes all three from `),i(`strong`,null,`the same`),c(` vector `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`mi`,{mathvariant:`bold`},`x`)]),i(`annotation`,{encoding:`application/x-tex`},`\\mathbf{x}`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.4444em`}}),i(`span`,{class:`mord mathbf`},`x`)])])]),c(`, through three `),i(`strong`,null,`learned`),c(` matrices: `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`mi`,{mathvariant:`bold`},`q`),i(`mo`,null,`=`),i(`mi`,{mathvariant:`bold`},`x`),i(`msub`,null,[i(`mi`,null,`W`),i(`mi`,null,`Q`)])]),i(`annotation`,{encoding:`application/x-tex`},`\\mathbf{q} = \\mathbf{x}W_Q`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.6389em`,"vertical-align":`-0.1944em`}}),i(`span`,{class:`mord mathbf`},`q`),i(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),i(`span`,{class:`mrel`},`=`),i(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.9694em`,"vertical-align":`-0.2861em`}}),i(`span`,{class:`mord mathbf`},`x`),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t vlist-t2`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.3283em`}},[i(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mathnormal mtight`},`Q`)])])]),i(`span`,{class:`vlist-s`},``)]),i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.2861em`}},[i(`span`)])])])])])])])]),c(`, `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`mi`,{mathvariant:`bold`},`k`),i(`mo`,null,`=`),i(`mi`,{mathvariant:`bold`},`x`),i(`msub`,null,[i(`mi`,null,`W`),i(`mi`,null,`K`)])]),i(`annotation`,{encoding:`application/x-tex`},`\\mathbf{k} = \\mathbf{x}W_K`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.6944em`}}),i(`span`,{class:`mord mathbf`},`k`),i(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),i(`span`,{class:`mrel`},`=`),i(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),i(`span`,{class:`mord mathbf`},`x`),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t vlist-t2`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.3283em`}},[i(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0715em`}},`K`)])])]),i(`span`,{class:`vlist-s`},``)]),i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.15em`}},[i(`span`)])])])])])])])]),c(`, `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`mi`,{mathvariant:`bold`},`v`),i(`mo`,null,`=`),i(`mi`,{mathvariant:`bold`},`x`),i(`msub`,null,[i(`mi`,null,`W`),i(`mi`,null,`V`)])]),i(`annotation`,{encoding:`application/x-tex`},`\\mathbf{v} = \\mathbf{x}W_V`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.4444em`}}),i(`span`,{class:`mord mathbf`,style:{"margin-right":`0.016em`}},`v`),i(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}}),i(`span`,{class:`mrel`},`=`),i(`span`,{class:`mspace`,style:{"margin-right":`0.2778em`}})]),i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.8333em`,"vertical-align":`-0.15em`}}),i(`span`,{class:`mord mathbf`},`x`),i(`span`,{class:`mord`},[i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`W`),i(`span`,{class:`msupsub`},[i(`span`,{class:`vlist-t vlist-t2`},[i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.3283em`}},[i(`span`,{style:{top:`-2.55em`,"margin-left":`-0.1389em`,"margin-right":`0.05em`}},[i(`span`,{class:`pstrut`,style:{height:`2.7em`}}),i(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[i(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.2222em`}},`V`)])])]),i(`span`,{class:`vlist-s`},``)]),i(`span`,{class:`vlist-r`},[i(`span`,{class:`vlist`,style:{height:`0.15em`}},[i(`span`)])])])])])])])]),c(`.`)],-1),i(`div`,m,[(u(),o(`svg`,h,[..._[0]||=[i(`defs`,null,[i(`marker`,{id:`l6s1-roles`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[i(`path`,{class:`dl-dg-head`,d:`M0 0 L7 3.5 L0 7 z`})])],-1),i(`rect`,{class:`dl-dg-box is-accent`,x:`10`,y:`74`,width:`70`,height:`46`,rx:`4`},null,-1),i(`text`,{class:`dl-dg-lab`,x:`45`,y:`103`,"text-anchor":`middle`},`x`,-1),i(`text`,{class:`dl-dg-small`,x:`45`,y:`138`,"text-anchor":`middle`},`one token`,-1),i(`path`,{class:`dl-dg-arrow`,d:`M80 97 C 110 97, 110 34, 140 34`,"marker-end":`url(#l6s1-roles)`},null,-1),i(`path`,{class:`dl-dg-arrow`,d:`M80 97 H140`,"marker-end":`url(#l6s1-roles)`},null,-1),i(`path`,{class:`dl-dg-arrow`,d:`M80 97 C 110 97, 110 160, 140 160`,"marker-end":`url(#l6s1-roles)`},null,-1),i(`polygon`,{class:`dl-dg-net`,points:`146,16 206,24 206,44 146,52`},null,-1),i(`polygon`,{class:`dl-dg-net`,points:`146,79 206,87 206,107 146,115`},null,-1),i(`polygon`,{class:`dl-dg-net`,points:`146,142 206,150 206,170 146,178`},null,-1),i(`text`,{class:`dl-dg-in`,x:`174`,y:`39`,"text-anchor":`middle`},[c(`W`),i(`tspan`,{"baseline-shift":`sub`,style:{"font-size":`9px`}},`Q`)],-1),i(`text`,{class:`dl-dg-in`,x:`174`,y:`102`,"text-anchor":`middle`},[c(`W`),i(`tspan`,{"baseline-shift":`sub`,style:{"font-size":`9px`}},`K`)],-1),i(`text`,{class:`dl-dg-in`,x:`174`,y:`165`,"text-anchor":`middle`},[c(`W`),i(`tspan`,{"baseline-shift":`sub`,style:{"font-size":`9px`}},`V`)],-1),i(`path`,{class:`dl-dg-arrow`,d:`M206 34 H238`,"marker-end":`url(#l6s1-roles)`},null,-1),i(`path`,{class:`dl-dg-arrow`,d:`M206 97 H238`,"marker-end":`url(#l6s1-roles)`},null,-1),i(`path`,{class:`dl-dg-arrow`,d:`M206 160 H238`,"marker-end":`url(#l6s1-roles)`},null,-1),i(`rect`,{class:`dl-dg-box`,x:`244`,y:`20`,width:`44`,height:`28`,rx:`3`},null,-1),i(`rect`,{class:`dl-dg-box`,x:`244`,y:`83`,width:`44`,height:`28`,rx:`3`},null,-1),i(`rect`,{class:`dl-dg-box`,x:`244`,y:`146`,width:`44`,height:`28`,rx:`3`},null,-1),i(`text`,{class:`dl-dg-lab`,x:`266`,y:`40`,"text-anchor":`middle`},`q`,-1),i(`text`,{class:`dl-dg-lab`,x:`266`,y:`103`,"text-anchor":`middle`},`k`,-1),i(`text`,{class:`dl-dg-lab`,x:`266`,y:`166`,"text-anchor":`middle`},`v`,-1),i(`text`,{class:`dl-dg-lab is-sm`,x:`302`,y:`39`},`query: what am I looking for?`,-1),i(`text`,{class:`dl-dg-lab is-sm`,x:`302`,y:`102`},`key: what do I advertise?`,-1),i(`text`,{class:`dl-dg-lab is-sm`,x:`302`,y:`165`},`value: what do I hand over?`,-1)]]))]),e((u(),o(`div`,g,[..._[1]||=[i(`p`,null,[c(`Three `),i(`strong`,null,`projections`),c(` of one vector, not three stored things. Every token is a query, a key and a value at once.`)],-1)]])),[[v]])]),_:1},16)}}};export{_ as default};
|
lecture-06/class/assets/md-DSyIwSi7.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,g as l,lt as u,m as d,v as f,vt as p,w as m,x as h,xt as g,y as _,yt as v,z as y}from"./modules/shiki-BFOesOFp.js";import{t as b}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as x,t as S}from"./slidev/context-DrkI8qiD.js";import{t as C}from"./slidev/VClicks-Dmx1KjZv.js";import{n as w,t as T}from"./interactive-CI8hm4xw.js";import{t as E}from"./StepButton-BcP33F-1.js";var D=[`viewBox`],O=[`x`,`width`],k=[`x`],A=[`x`,`width`],j=[`x`],M=[`d`],N=[`cx`],P=[`x`],F=[`x`],I=[`x`,`y`],L=[`x`,`y`],R=[`x`,`y`],z=640,B=372,V=218,H=56,U=366,W=30,G=b(m({__name:`TransformerStack`,props:{layers:{default:6}},setup(e){let{$slidev:t,$nav:a,$clicksContext:s,$clicks:m,$page:v,$renderContext:b,$frontmatter:S}=x(),C=e,T=[{id:`enc-emb`,x:H,y:300,label:`Input embedding`,sub:`one vector per source token`},{id:`enc-sa`,x:H,y:218,label:`Self-attention`,sub:`every token sees every token`},{id:`enc-ff`,x:H,y:180,label:`Feed-forward`,sub:`each position on its own`},{id:`dec-emb`,x:U,y:300,label:`Output embedding`,sub:`what has been written so far`},{id:`dec-sa`,x:U,y:218,label:`Masked self-attention`,sub:`no peeking at the future`},{id:`dec-ca`,x:U,y:180,label:`Cross-attention`,sub:`Q from here, K and V from the encoder`},{id:`dec-ff`,x:U,y:142,label:`Feed-forward`},{id:`dec-lin`,x:U,y:96,label:`Linear`,sub:`d → vocabulary size`},{id:`dec-sm`,x:U,y:58,label:`Softmax`,sub:`a probability per word in the vocabulary`}],G=[{title:`Embed the source`,lit:[`enc-in`,`enc-emb`],note:"The source sentence is split into tokens and each one is looked up in an embedding table (`nn.Embedding`), which turns a token id into a learned vector. Nothing here knows anything about order yet."},{title:`Add position`,lit:[`enc-in`,`enc-emb`,`enc-pe`],note:`A fixed positional vector is added to each embedding. Without it the encoder would treat the sentence as a bag of words, because a weighted sum has no order in it.`},{title:`Encoder self-attention`,lit:[`enc-sa`],note:`Every source token attends to every other, in both directions — there is no mask here, because the whole source is available at once. This is the step that makes each vector *contextual*.`},{title:`Encoder block, ×N`,lit:[`enc-sa`,`enc-ff`,`enc-stack`],note:`Attention and feed-forward, each wrapped in a residual and a LayerNorm, repeated N times. The output is one vector per source token — not one vector for the whole sentence, which is precisely the bottleneck this architecture removes.`},{title:`The decoder starts`,lit:[`dec-in`,`dec-emb`,`dec-pe`],note:`The decoder is fed what has been produced so far, shifted right by one and starting from a begin-of-sequence token. During training that is the true target sentence, so every position trains at once. This is teacher forcing: training on the true previous words instead of the model’s own guesses.`},{title:`Masked self-attention`,lit:[`dec-sa`],note:`The decoder attends to its own output, with position t forbidden from seeing anything after t. Without the mask the model could read the answer off its own input and the loss would collapse to nothing.`},{title:`Cross-attention`,lit:[`dec-ca`,`cross`],note:`The only place the two stacks meet. Queries come from the decoder — “what do I need next?” — and keys and values from the encoder’s output. This is Bahdanau’s 2014 attention for translation, with the recurrence removed from both sides.`},{title:`Decoder block, ×N`,lit:[`dec-sa`,`dec-ca`,`dec-ff`,`dec-stack`],note:`Three sub-layers now rather than two, again each with a residual and a norm, again repeated N times.`},{title:`Predict the next token`,lit:[`dec-lin`,`dec-sm`,`dec-out`],note:`A linear layer to vocabulary size and a softmax: a probability for every word. Take one, append it, and run the decoder again. The encoder runs once; the decoder runs once per output token — which is where “transformers are parallel” gets its asterisk.`}],K=u(0),q=l(()=>K.value>=G.length),J=l(()=>K.value===0?null:G[K.value-1]);function Y(){K.value=q.value?0:K.value+1}function X(e){return K.value===0?`idle`:J.value?.lit.includes(e)?`current`:G.slice(0,K.value).some(t=>t.lit.includes(e))?`seen`:`idle`}return(e,t)=>{let a=E,s=w;return y(),f(s,{"max-width":`43rem`},{controls:n(()=>[r(a,{label:q.value?`Start over`:K.value===0?`Walk the architecture`:`Next step`,glyph:`▸`,onClick:Y},null,8,[`label`])]),readout:n(()=>[J.value?(y(),c(d,{key:0},[o(`strong`,null,g(K.value)+`/`+g(G.length)+` · `+g(J.value.title),1),h(` — `+g(J.value.note),1)],64)):(y(),c(d,{key:1},[t[1]||=h(` Six boxes you have already met, arranged into two stacks. Press `,-1),t[2]||=o(`em`,null,`Walk the architecture`,-1),t[3]||=h(` and follow the data: up the encoder once, then up the decoder once for every word it writes. `,-1)],64))]),default:n(()=>[(y(),c(`svg`,{class:`dl-stack`,viewBox:`0 0 ${z} ${B}`,preserveAspectRatio:`xMidYMid meet`,role:`img`,"aria-label":`The 2017 encoder-decoder transformer: an encoder stack reading the source sentence and a decoder stack that attends to its own output and to the encoder's`},[t[0]||=o(`defs`,null,[o(`marker`,{id:`dl-stack-arrow`,viewBox:`0 0 8 8`,refX:`7`,refY:`4`,markerWidth:`5`,markerHeight:`5`,orient:`auto-start-reverse`},[o(`path`,{d:`M0,0 L8,4 L0,8 z`,fill:`currentColor`})])],-1),_(` Column spines, drawn first so the boxes sit on top of them. `),o(`line`,{class:`dl-stack__spine`,x1:165,y1:336,x2:165,y2:176}),o(`line`,{class:`dl-stack__spine`,x1:475,y1:336,x2:475,y2:54,"marker-end":`url(#dl-stack-arrow)`}),_(` Repeated-block brackets. `),o(`rect`,{class:p([`dl-stack__repeat`,`is-${X(`enc-stack`)}`]),x:44,y:`172`,width:242,height:`84`,rx:`8`},null,10,O),o(`text`,{class:p([`dl-stack__repeatlabel`,`is-${X(`enc-stack`)}`]),x:38,y:`214`,"text-anchor":`end`},`×`+g(C.layers),11,k),o(`rect`,{class:p([`dl-stack__repeat`,`is-${X(`dec-stack`)}`]),x:354,y:`134`,width:242,height:`122`,rx:`8`},null,10,A),o(`text`,{class:p([`dl-stack__repeatlabel`,`is-${X(`dec-stack`)}`]),x:602,y:`198`,"text-anchor":`start`},`×`+g(C.layers),11,j),_(` The one place the two stacks touch. `),o(`path`,{class:p([`dl-stack__cross`,`is-${X(`cross`)}`]),d:`M 165 176 V 158 H 320 V 195 H 362`,"marker-end":`url(#dl-stack-arrow)`},null,10,M),o(`text`,{class:p([`dl-stack__crosslabel`,`is-${X(`cross`)}`]),x:`318`,y:`152`,"text-anchor":`middle`},` K, V from the encoder `,2),_(` Positional-encoding adders. `),(y(!0),c(d,null,i([{id:`enc-pe`,x:165},{id:`dec-pe`,x:475}],e=>(y(),c(`g`,{key:e.id},[o(`circle`,{class:p([`dl-stack__add`,`is-${X(e.id)}`]),cx:e.x,cy:`272`,r:`11`},null,10,N),o(`text`,{class:`dl-stack__plus`,x:e.x,y:`277`,"text-anchor":`middle`},`+`,8,P),o(`text`,{class:p([`dl-stack__pe`,`is-${X(e.id)}`]),x:e.x-18,y:`276`,"text-anchor":`end`},`position`,10,F)]))),128)),_(` The boxes. `),(y(),c(d,null,i(T,e=>o(`g`,{key:e.id},[o(`rect`,{class:p([`dl-stack__box`,`is-${X(e.id)}`]),x:e.x,y:e.y,width:V,height:W,rx:`5`},null,10,I),o(`text`,{class:p([`dl-stack__label`,`is-${X(e.id)}`]),x:e.x+V/2,y:e.y+(e.sub?14:20),"text-anchor":`middle`},g(e.label),11,L),e.sub?(y(),c(`text`,{key:0,class:p([`dl-stack__sub`,`is-${X(e.id)}`]),x:e.x+V/2,y:e.y+25,"text-anchor":`middle`},g(e.sub),11,R)):_(`v-if`,!0)])),64)),_(` Ports. `),o(`text`,{class:p([`dl-stack__port`,`is-${X(`enc-in`)}`]),x:165,y:`350`,"text-anchor":`middle`},` source sentence `,2),o(`text`,{class:p([`dl-stack__port`,`is-${X(`dec-in`)}`]),x:475,y:`350`,"text-anchor":`middle`},` output so far, shifted right `,2),o(`text`,{class:p([`dl-stack__port`,`is-${X(`dec-out`)}`]),x:475,y:`44`,"text-anchor":`middle`},` probability of every next word `,2),o(`text`,{class:`dl-stack__side`,x:H,y:`18`},`ENCODER — reads once`),o(`text`,{class:`dl-stack__side`,x:U,y:`18`},`DECODER — runs once per output token`)],8,D))]),_:1})}}}),[[`__scopeId`,`data-v-66f48009`]]),K={class:`mt-2 dl-callout`},q={__name:`class.md__slidev_10`,setup(i){let{$slidev:l,$nav:u,$clicksContext:d,$clicks:p,$page:m,$renderContext:g,$frontmatter:_}=x();return d.setup(),(i,l)=>{let u=G,d=C,p=a(`click`);return y(),f(T,v(t(s(S)(s(_),9))),{aside:n(t=>[l[2]||=o(`p`,null,[h(`Press `),o(`strong`,null,`Walk the architecture`),h(` and follow the data: nine steps, no new boxes.`)],-1),r(d,null,{default:n(()=>[...l[0]||=[o(`ul`,null,[o(`li`,null,[h(`up the encoder `),o(`strong`,null,`once`)]),o(`li`,null,[h(`then up the decoder `),o(`strong`,null,`once per word it writes`)]),o(`li`,null,`cross-attention is the only wire between them`)],-1)]]),_:1}),e((y(),c(`div`,K,[...l[1]||=[o(`p`,null,`The famous figure, in the order the data actually moves.`,-1)]])),[[p]])]),default:n(()=>[r(u,{layers:6})]),_:1},16)}}};export{q as default};
|
lecture-06/class/assets/md-D_ZyssbS2.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,w as l,x as u,xt as d,y as f,yt as p,z as m}from"./modules/shiki-BFOesOFp.js";import{t as h}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as g,t as _}from"./slidev/context-DrkI8qiD.js";import{t as v}from"./TransformerGlyph-D9TehqNe.js";import{t as y}from"./default-Cq9fupKf.js";var b=[`href`],x={key:0,class:`dl-linkcard__icon`},S={class:`dl-linkcard__text`},C={class:`dl-linkcard__title`},w={key:0,class:`dl-linkcard__blurb`},T={class:`dl-linkcard__host`},E=h(l({__name:`LinkCard`,props:{href:{},title:{},blurb:{},icon:{}},setup(e){let{$slidev:t,$nav:n,$clicksContext:r,$clicks:i,$page:c,$renderContext:l,$frontmatter:u}=g(),p=e,h=p.href.replace(/^https?:\/\//,``).replace(/\/.*$/,``);return(e,t)=>(m(),s(`a`,{class:`dl-linkcard`,href:p.href,target:`_blank`,rel:`noopener`},[p.icon?(m(),s(`div`,x,d(p.icon),1)):f(`v-if`,!0),a(`div`,S,[a(`div`,C,d(p.title),1),p.blurb?(m(),s(`div`,w,d(p.blurb),1)):f(`v-if`,!0),a(`div`,T,d(o(h)),1)])],8,b))}}),[[`__scopeId`,`data-v-77ab5d10`]]),D={class:`grid grid-cols-3 gap-4 mt-4`},O={class:`mt-4 flex items-center gap-4 dl-tight`},k={class:`mt-2 dl-secondary`},A={__name:`class.md__slidev_43`,setup(l){let{$slidev:d,$nav:f,$clicksContext:h,$clicks:b,$page:x,$renderContext:S,$frontmatter:C}=g();return h.setup(),(l,d)=>{let f=E,h=v,g=i(`click`);return m(),c(y,p(t(o(_)(o(C),42))),{default:n(()=>[d[2]||=a(`h1`,null,`Reading, and the lab`,-1),a(`div`,D,[e((m(),s(`div`,null,[r(f,{href:`https://jalammar.github.io/illustrated-transformer/`,title:`The Illustrated Transformer`,blurb:`Alammar, 2018. The pictures everyone has seen, and still the best first read. Do this one before the lab.`,icon:`🖼️`})])),[[g]]),e((m(),s(`div`,null,[r(f,{href:`https://sebastianraschka.com/blog/2023/self-attention-from-scratch.html`,title:`Self-attention from scratch`,blurb:`Raschka, 2023. The same worked example as today's widgets, in NumPy and PyTorch, at full size.`,icon:`✍️`})])),[[g]]),e((m(),s(`div`,null,[r(f,{href:`https://github.com/karpathy/nanoGPT`,title:`nanoGPT`,blurb:`Karpathy. A complete, readable, trainable decoder-only transformer in about 300 lines. Its attention is our “Self-attention, written out”.`,icon:`💻`})])),[[g]])]),e((m(),s(`div`,O,[r(h,{kind:`decoder`,size:64}),d[0]||=a(`div`,null,[a(`p`,null,[a(`strong`,null,`In the lab.`),u(` Build the block from “A block, and a stack”. Train a small decoder-only model on character-level text. Then drop `),a(`code`,null,`is_causal`),u(`, the `),a(`span`,{class:`katex`},[a(`span`,{class:`katex-mathml`},[a(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[a(`semantics`,null,[a(`mrow`,null,[a(`msqrt`,null,[a(`msub`,null,[a(`mi`,null,`d`),a(`mi`,null,`k`)])])]),a(`annotation`,{encoding:`application/x-tex`},`\\sqrt{d_k}`)])])]),a(`span`,{class:`katex-html`,"aria-hidden":`true`},[a(`span`,{class:`katex-base`},[a(`span`,{class:`katex-strut`,style:{height:`1.04em`,"vertical-align":`-0.1828em`}}),a(`span`,{class:`mord sqrt`},[a(`span`,{class:`vlist-t vlist-t2`},[a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.8572em`}},[a(`span`,{class:`svg-align`,style:{top:`-3em`}},[a(`span`,{class:`pstrut`,style:{height:`3em`}}),a(`span`,{class:`mord`,style:{"padding-left":`0.833em`}},[a(`span`,{class:`mord`},[a(`span`,{class:`mord mathnormal`},`d`),a(`span`,{class:`msupsub`},[a(`span`,{class:`vlist-t vlist-t2`},[a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.3361em`}},[a(`span`,{style:{top:`-2.55em`,"margin-left":`0em`,"margin-right":`0.05em`}},[a(`span`,{class:`pstrut`,style:{height:`2.7em`}}),a(`span`,{class:`katex-sizing reset-size6 size3 mtight`},[a(`span`,{class:`mord mathnormal mtight`,style:{"margin-right":`0.0315em`}},`k`)])])]),a(`span`,{class:`vlist-s`},``)]),a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.15em`}},[a(`span`)])])])])])])]),a(`span`,{style:{top:`-2.8172em`}},[a(`span`,{class:`pstrut`,style:{height:`3em`}}),a(`span`,{class:`hide-tail`,style:{"min-width":`0.853em`,height:`1.08em`}},[a(`svg`,{xmlns:`http://www.w3.org/2000/svg`,width:`400em`,height:`1.08em`,viewBox:`0 0 400000 1080`,preserveAspectRatio:`xMinYMin slice`},[a(`path`,{d:`M95,702
|
| 2 |
+
c-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14
|
| 3 |
+
c0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54
|
| 4 |
+
c44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10
|
| 5 |
+
s173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429
|
| 6 |
+
c69,-144,104.5,-217.7,106.5,-221
|
| 7 |
+
l0 -0
|
| 8 |
+
c5.3,-9.3,12,-14,20,-14
|
| 9 |
+
H400000v40H845.2724
|
| 10 |
+
s-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7
|
| 11 |
+
c-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z
|
| 12 |
+
M834 80h400000v40h-400000z`})])])])]),a(`span`,{class:`vlist-s`},``)]),a(`span`,{class:`vlist-r`},[a(`span`,{class:`vlist`,style:{height:`0.1828em`}},[a(`span`)])])])])])])]),u(` and the positional encoding, one at a time. Report what each one costs.`)])],-1)])),[[g]]),e((m(),s(`div`,k,[...d[1]||=[a(`p`,null,`Dropping the positional encoding is the interesting one. Predict first.`,-1)]])),[[g]])]),_:1},16)}}};export{A as default};
|
lecture-06/class/assets/md-DgIoPoNt.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,W as r,_ as i,_t as a,b as o,v as s,yt as c,z as l}from"./modules/shiki-BFOesOFp.js";import{n as u,t as d}from"./slidev/context-DrkI8qiD.js";import{t as f}from"./default-Cq9fupKf.js";var p={class:`mt-1 flex justify-center`},m={viewBox:`0 0 690 250`,class:`dl-diagram`,style:{"max-width":`40rem`},role:`img`,"aria-label":`Three columns. Encoder-only: a whole review in, one label out, as in sentiment, search embeddings and named entities. Decoder-only: a prompt in, text written word by word, as in chat, code and question answering. Encoder-decoder: a full source in, a different full sequence out, as in translation, summarisation and speech to text.`},h={class:`dl-callout`},g={__name:`class.md__slidev_35`,setup(g){let{$slidev:_,$nav:v,$clicksContext:y,$clicks:b,$page:x,$renderContext:S,$frontmatter:C}=u();return y.setup(),(u,g)=>{let _=r(`click`);return l(),s(f,c(t(a(d)(a(C),34))),{default:n(()=>[g[14]||=i(`h1`,null,`Which kind for which job`,-1),g[15]||=i(`p`,null,`Pick by the shape of the task: what goes in, what comes out.`,-1),i(`div`,p,[(l(),o(`svg`,m,[g[2]||=i(`defs`,null,[i(`marker`,{id:`l6s5-jobs-head`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[i(`path`,{d:`M0 0 L7 3.5 L0 7 z`,class:`dl-dg-head`})])],-1),g[3]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`110`,y:`18`,"text-anchor":`middle`},`encoder-only`,-1),g[4]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`345`,y:`18`,"text-anchor":`middle`},`decoder-only`,-1),g[5]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`580`,y:`18`,"text-anchor":`middle`},`encoder–decoder`,-1),g[6]||=i(`rect`,{x:`50`,y:`34`,width:`120`,height:`70`,rx:`6`,class:`dl-dg-box`},null,-1),g[7]||=i(`text`,{class:`dl-dg-in`,x:`110`,y:`74`,"text-anchor":`middle`},`reads all`,-1),g[8]||=i(`rect`,{x:`285`,y:`34`,width:`120`,height:`70`,rx:`6`,class:`dl-dg-box is-accent`},null,-1),g[9]||=i(`text`,{class:`dl-dg-in`,x:`345`,y:`74`,"text-anchor":`middle`},`writes left → right`,-1),g[10]||=i(`rect`,{x:`510`,y:`34`,width:`62`,height:`70`,rx:`6`,class:`dl-dg-box`},null,-1),g[11]||=i(`rect`,{x:`588`,y:`34`,width:`62`,height:`70`,rx:`6`,class:`dl-dg-box is-accent`},null,-1),g[12]||=i(`path`,{d:`M572 69 H586`,class:`dl-dg-line`,"marker-end":`url(#l6s5-jobs-head)`},null,-1),e((l(),o(`g`,null,[...g[0]||=[i(`text`,{class:`dl-dg-small`,x:`110`,y:`128`,"text-anchor":`middle`},`in: a whole review`,-1),i(`text`,{class:`dl-dg-small`,x:`110`,y:`146`,"text-anchor":`middle`},`out: one label, "positive"`,-1),i(`text`,{class:`dl-dg-small`,x:`345`,y:`128`,"text-anchor":`middle`},`in: "Write a haiku about"`,-1),i(`text`,{class:`dl-dg-small`,x:`345`,y:`146`,"text-anchor":`middle`},`out: next word, again and again`,-1),i(`text`,{class:`dl-dg-small`,x:`580`,y:`128`,"text-anchor":`middle`},`in: "Ich habe es gelesen"`,-1),i(`text`,{class:`dl-dg-small`,x:`580`,y:`146`,"text-anchor":`middle`},`out: "I have read it"`,-1)]])),[[_]]),e((l(),o(`g`,null,[...g[1]||=[i(`text`,{class:`dl-dg-small is-good`,x:`110`,y:`180`,"text-anchor":`middle`},`sentiment, spam, topic`,-1),i(`text`,{class:`dl-dg-small is-good`,x:`110`,y:`198`,"text-anchor":`middle`},`search embeddings`,-1),i(`text`,{class:`dl-dg-small is-good`,x:`110`,y:`216`,"text-anchor":`middle`},`named entities`,-1),i(`text`,{class:`dl-dg-small is-good`,x:`345`,y:`180`,"text-anchor":`middle`},`chat assistants`,-1),i(`text`,{class:`dl-dg-small is-good`,x:`345`,y:`198`,"text-anchor":`middle`},`code completion`,-1),i(`text`,{class:`dl-dg-small is-good`,x:`345`,y:`216`,"text-anchor":`middle`},`question answering`,-1),i(`text`,{class:`dl-dg-small is-good`,x:`580`,y:`180`,"text-anchor":`middle`},`translation`,-1),i(`text`,{class:`dl-dg-small is-good`,x:`580`,y:`198`,"text-anchor":`middle`},`summarisation`,-1),i(`text`,{class:`dl-dg-small is-good`,x:`580`,y:`216`,"text-anchor":`middle`},`speech to text`,-1),i(`text`,{class:`dl-dg-small`,x:`110`,y:`242`,"text-anchor":`middle`},`BERT`,-1),i(`text`,{class:`dl-dg-small`,x:`345`,y:`242`,"text-anchor":`middle`},`GPT, Llama`,-1),i(`text`,{class:`dl-dg-small`,x:`580`,y:`242`,"text-anchor":`middle`},`T5, Whisper`,-1)]])),[[_]])]))]),e((l(),o(`div`,h,[...g[13]||=[i(`p`,null,`Label it → encoder. Write it → decoder. Turn it into another sequence → both.`,-1)]])),[[_]])]),_:1},16)}}};export{g as default};
|
lecture-06/class/assets/md-DhsPSwrk.css
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
.dl-blk[data-v-c465f205]{width:100%;height:100%}.dl-blk__spine[data-v-c465f205]{stroke:var(--dl-border);stroke-width:2px;color:var(--dl-border)}.dl-blk__residual[data-v-c465f205]{fill:none;stroke:var(--dl-border);stroke-width:1.6px;stroke-dasharray:4 3;color:var(--dl-border);transition:stroke .2s}.dl-blk__residual.is-lit[data-v-c465f205]{stroke:var(--dl-accent);color:var(--dl-accent)}.dl-blk__box[data-v-c465f205]{fill:var(--dl-bg);stroke:var(--dl-border);stroke-width:1.3px;transition:fill .2s,stroke .2s}.dl-blk__box.is-sublayer[data-v-c465f205]{stroke-dasharray:none;stroke-width:1.8px}.dl-blk__box.is-lit[data-v-c465f205]{fill:var(--dl-surface);stroke:var(--dl-body)}.dl-blk__box.is-current[data-v-c465f205]{fill:var(--dl-accent-soft);stroke:var(--dl-accent);stroke-width:2.2px}.dl-blk__add[data-v-c465f205]{fill:var(--dl-bg);stroke:var(--dl-border);stroke-width:1.5px;transition:fill .2s,stroke .2s}.dl-blk__add.is-lit[data-v-c465f205]{stroke:var(--dl-body)}.dl-blk__add.is-current[data-v-c465f205]{fill:var(--dl-accent-soft);stroke:var(--dl-accent);stroke-width:2.2px}.dl-blk__plus[data-v-c465f205]{fill:var(--dl-heading);font-size:14px;font-weight:700}.dl-blk__label[data-v-c465f205]{fill:var(--dl-muted);font-size:12.5px;font-weight:600;transition:fill .2s}.dl-blk__label.is-lit[data-v-c465f205]{fill:var(--dl-heading)}.dl-blk__sub[data-v-c465f205]{fill:var(--dl-muted);font-size:9.5px}.dl-blk__sub.is-lit[data-v-c465f205]{fill:var(--dl-body)}.dl-blk__port[data-v-c465f205]{fill:var(--dl-muted);font-size:10px}.dl-blk__rail[data-v-c465f205]{fill:var(--dl-muted);letter-spacing:.08em;font-size:9.5px}
|
lecture-06/class/assets/md-Dnd6WsWa2.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import{E as e,M as t,Q as n,S as r,_ as i,_t as a,v as o,x as s,yt as c,z as l}from"./modules/shiki-BFOesOFp.js";import{n as u,t as d}from"./slidev/context-DrkI8qiD.js";import{t as f}from"./slidev/VClicks-Dmx1KjZv.js";import{t as p}from"./default-Cq9fupKf.js";import{t as m}from"./slidev/CodeBlockWrapper-gsL5r1Ig.js";var h={class:`grid grid-cols-[1fr_1fr] gap-6 mt-2 items-center`},g={class:`dl-tight`},_={__name:`class.md__slidev_39`,setup(_){let{$slidev:v,$nav:y,$clicksContext:b,$clicks:x,$page:S,$renderContext:C,$frontmatter:w}=u();return b.setup(),(u,_)=>{let v=m,y=f;return l(),o(p,c(e(a(d)(a(w),38))),{default:n(()=>[_[3]||=i(`h1`,null,`The mechanism is one function call`,-1),r(v,t({lines:!0},{title:``,ranges:[`all`,`1-4`,`6`,`8-9`,`all`]}),{default:n(()=>[..._[0]||=[i(`pre`,{class:`shiki shiki-themes vitesse-dark vitesse-light slidev-code`,style:{"--shiki-dark":`#dbd7caee`,"--shiki-light":`#393a34`,"--shiki-dark-bg":`#121212`,"--shiki-light-bg":`#ffffff`}},[i(`code`,{class:`language-python`},[i(`span`,{class:`line`},[i(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`import`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` torch`)]),s(`
|
| 2 |
+
`),i(`span`,{class:`line`},[i(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`import`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` torch`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`nn`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`functional `),i(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`as`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` F`)]),s(`
|
| 3 |
+
`),i(`span`,{class:`line`}),s(`
|
| 4 |
+
`),i(`span`,{class:`line`},[i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`q`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` k`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` v `),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},` (`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`torch`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`randn`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),i(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},`32`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),i(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},` 8`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),i(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},` 128`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),i(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},` 64`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`),i(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},` for`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` _ `),i(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`in`),i(`span`,{style:{"--shiki-dark":`#B8A965`,"--shiki-light":`#998418`}},` range`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),i(`span`,{style:{"--shiki-dark":`#4C9A91`,"--shiki-light":`#2F798A`}},`3`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`))`),i(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # (B, heads, T, d_k)`)]),s(`
|
| 5 |
+
`),i(`span`,{class:`line`}),s(`
|
| 6 |
+
`),i(`span`,{class:`line`},[i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`out `),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` F`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`scaled_dot_product_attention`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`q`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` k`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},` v`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`,`),i(`span`,{style:{"--shiki-dark":`#BD976A`,"--shiki-light":`#B07D48`}},` is_causal`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`=`),i(`span`,{style:{"--shiki-dark":`#4D9375`,"--shiki-light":`#1E754F`}},`True`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`)]),s(`
|
| 7 |
+
`),i(`span`,{class:`line`}),s(`
|
| 8 |
+
`),i(`span`,{class:`line`},[i(`span`,{style:{"--shiki-dark":`#B8A965`,"--shiki-light":`#998418`}},`print`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`(`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`out`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`.`),i(`span`,{style:{"--shiki-dark":`#DBD7CAEE`,"--shiki-light":`#393A34`}},`shape`),i(`span`,{style:{"--shiki-dark":`#666666`,"--shiki-light":`#999999`}},`)`),i(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},` # torch.Size([32, 8, 128, 64])`)]),s(`
|
| 9 |
+
`),i(`span`,{class:`line`},[i(`span`,{style:{"--shiki-dark":`#758575DD`,"--shiki-light":`#A0ADA0`}},`# scores (32, 8, 128, 128) are never built — that is what FlashAttention buys`)])])],-1)]]),_:1},16),i(`div`,h,[_[2]||=i(`div`,null,[i(`svg`,{class:`dl-diagram`,viewBox:`0 0 440 130`,role:`img`,"aria-label":`q, k and v, each of shape 32 by 8 by 128 by 64, go into one attention call that scales, masks, applies softmax and multiplies by v. The output has the same shape. The scores tensor, 32 by 8 by 128 by 128, is drawn dashed in red: it is never stored.`},[i(`defs`,null,[i(`marker`,{id:`l6s5-a1`,markerWidth:`8`,markerHeight:`8`,refX:`7`,refY:`4`,orient:`auto`},[i(`path`,{d:`M0 0 L8 4 L0 8 z`,class:`dl-dg-head`})])]),i(`rect`,{x:`4`,y:`8`,width:`70`,height:`24`,rx:`3`,class:`dl-dg-box`}),i(`text`,{x:`39`,y:`25`,class:`dl-dg-lab is-sm`,"text-anchor":`middle`},`q`),i(`rect`,{x:`4`,y:`38`,width:`70`,height:`24`,rx:`3`,class:`dl-dg-box`}),i(`text`,{x:`39`,y:`55`,class:`dl-dg-lab is-sm`,"text-anchor":`middle`},`k`),i(`rect`,{x:`4`,y:`68`,width:`70`,height:`24`,rx:`3`,class:`dl-dg-box`}),i(`text`,{x:`39`,y:`85`,class:`dl-dg-lab is-sm`,"text-anchor":`middle`},`v`),i(`text`,{x:`44`,y:`112`,class:`dl-dg-small`,"text-anchor":`middle`},`(32, 8, 128, 64)`),i(`path`,{d:`M78 50 H122`,class:`dl-dg-arrow`,"marker-end":`url(#l6s5-a1)`}),i(`rect`,{x:`126`,y:`26`,width:`176`,height:`48`,rx:`6`,class:`dl-dg-op`}),i(`text`,{x:`214`,y:`46`,class:`dl-dg-lab is-sm`,"text-anchor":`middle`},`attention`),i(`text`,{x:`214`,y:`64`,class:`dl-dg-small`,"text-anchor":`middle`},`scale, mask, softmax, @ v`),i(`rect`,{x:`144`,y:`88`,width:`140`,height:`38`,rx:`4`,class:`dl-dg-box is-bad`}),i(`text`,{x:`214`,y:`103`,class:`dl-dg-small is-bad`,"text-anchor":`middle`},`scores (32, 8, 128, 128)`),i(`text`,{x:`214`,y:`119`,class:`dl-dg-small is-bad`,"text-anchor":`middle`},`never stored`),i(`path`,{d:`M306 50 H340`,class:`dl-dg-arrow`,"marker-end":`url(#l6s5-a1)`}),i(`rect`,{x:`344`,y:`38`,width:`70`,height:`24`,rx:`3`,class:`dl-dg-box is-accent`}),i(`text`,{x:`379`,y:`55`,class:`dl-dg-lab is-sm`,"text-anchor":`middle`},`out`),i(`text`,{x:`384`,y:`80`,class:`dl-dg-small`,"text-anchor":`middle`},`(32, 8, 128, 64)`)])],-1),i(`div`,g,[r(y,null,{default:n(()=>[..._[1]||=[i(`ul`,null,[i(`li`,null,[s(`Scaling, the mask and the softmax all happen inside; `),i(`code`,null,`is_causal=True`),s(` is the triangle`)]),i(`li`,null,[s(`A fused kernel never stores the `),i(`span`,{class:`katex`},[i(`span`,{class:`katex-mathml`},[i(`math`,{xmlns:`http://www.w3.org/1998/Math/MathML`},[i(`semantics`,null,[i(`mrow`,null,[i(`mi`,null,`T`),i(`mo`,null,`×`),i(`mi`,null,`T`)]),i(`annotation`,{encoding:`application/x-tex`},`T \\times T`)])])]),i(`span`,{class:`katex-html`,"aria-hidden":`true`},[i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.7667em`,"vertical-align":`-0.0833em`}}),i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`),i(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}}),i(`span`,{class:`mbin`},`×`),i(`span`,{class:`mspace`,style:{"margin-right":`0.2222em`}})]),i(`span`,{class:`katex-base`},[i(`span`,{class:`katex-strut`,style:{height:`0.6833em`}}),i(`span`,{class:`mord mathnormal`,style:{"margin-right":`0.1389em`}},`T`)])])]),s(` scores`)]),i(`li`,null,[s(`Heads are a `),i(`strong`,null,`batch dimension`),s(`: `),i(`code`,null,`(B, heads, T, d_k)`)])],-1)]]),_:1})])])]),_:1},16)}}};export{_ as default};
|
lecture-06/class/assets/md-DnwIl7Jl.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,W as r,_ as i,_t as a,b as o,v as s,x as c,yt as l,z as u}from"./modules/shiki-BFOesOFp.js";import{n as d,t as f}from"./slidev/context-DrkI8qiD.js";import{t as p}from"./default-Cq9fupKf.js";var m={class:`mt-1 flex justify-center`},h={viewBox:`0 0 640 250`,class:`dl-diagram`,style:{"max-width":`38rem`},role:`img`,"aria-label":`The two-stack transformer with five numbered parts: 1 attention, 2 several heads and position, 3 the feed-forward network, 4 residual and norm wrappers, 5 the decoder's mask and cross-attention`},g={class:`dl-tight`},_={__name:`class.md__slidev_11`,setup(_){let{$slidev:v,$nav:y,$clicksContext:b,$clicks:x,$page:S,$renderContext:C,$frontmatter:w}=d();return b.setup(),(d,_)=>{let v=r(`click`);return u(),s(p,l(t(a(f)(a(w),10))),{default:n(()=>[_[19]||=i(`h1`,null,`The parts, one at a time`,-1),_[20]||=i(`p`,null,`Every box is one of five parts. The next three sections open them in order.`,-1),i(`div`,m,[(u(),o(`svg`,h,[_[1]||=i(`defs`,null,[i(`marker`,{id:`l6s1-parts-head`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[i(`path`,{d:`M0 0 L7 3.5 L0 7 z`,class:`dl-dg-head`})])],-1),_[2]||=i(`rect`,{x:`40`,y:`70`,width:`200`,height:`130`,rx:`8`,class:`dl-dg-box`},null,-1),_[3]||=i(`text`,{class:`dl-dg-small`,x:`50`,y:`62`},`encoder`,-1),_[4]||=i(`rect`,{x:`58`,y:`88`,width:`164`,height:`30`,rx:`5`,class:`dl-dg-box`},null,-1),_[5]||=i(`text`,{class:`dl-dg-in`,x:`140`,y:`108`,"text-anchor":`middle`},`feed-forward`,-1),_[6]||=i(`rect`,{x:`58`,y:`150`,width:`164`,height:`30`,rx:`5`,class:`dl-dg-box is-accent`},null,-1),_[7]||=i(`text`,{class:`dl-dg-in`,x:`140`,y:`170`,"text-anchor":`middle`},`self-attention`,-1),_[8]||=i(`path`,{d:`M48 196 V80`,class:`dl-dg-line`,style:{"stroke-dasharray":`4 3`}},null,-1),_[9]||=i(`rect`,{x:`380`,y:`40`,width:`220`,height:`160`,rx:`8`,class:`dl-dg-box`},null,-1),_[10]||=i(`text`,{class:`dl-dg-small`,x:`390`,y:`32`},`decoder`,-1),_[11]||=i(`rect`,{x:`398`,y:`56`,width:`184`,height:`28`,rx:`5`,class:`dl-dg-box`},null,-1),_[12]||=i(`text`,{class:`dl-dg-in`,x:`490`,y:`75`,"text-anchor":`middle`},`feed-forward`,-1),_[13]||=i(`rect`,{x:`398`,y:`100`,width:`184`,height:`28`,rx:`5`,class:`dl-dg-box is-accent`},null,-1),_[14]||=i(`text`,{class:`dl-dg-in`,x:`490`,y:`119`,"text-anchor":`middle`},`cross-attention`,-1),_[15]||=i(`rect`,{x:`398`,y:`146`,width:`184`,height:`28`,rx:`5`,class:`dl-dg-box is-accent`},null,-1),_[16]||=i(`text`,{class:`dl-dg-in`,x:`490`,y:`165`,"text-anchor":`middle`},`masked self-attention`,-1),_[17]||=i(`path`,{d:`M240 104 C 310 104, 320 114, 396 114`,class:`dl-dg-line`,"marker-end":`url(#l6s1-parts-head)`},null,-1),e((u(),o(`g`,null,[..._[0]||=[i(`circle`,{cx:`20`,cy:`165`,r:`12`,class:`dl-dg-op`},null,-1),i(`text`,{class:`dl-dg-lab is-sm`,x:`20`,y:`170`,"text-anchor":`middle`},`1`,-1),i(`circle`,{cx:`140`,cy:`226`,r:`12`,class:`dl-dg-op`},null,-1),i(`text`,{class:`dl-dg-lab is-sm`,x:`140`,y:`231`,"text-anchor":`middle`},`2`,-1),i(`text`,{class:`dl-dg-small`,x:`158`,y:`231`},`heads + position`,-1),i(`circle`,{cx:`20`,cy:`103`,r:`12`,class:`dl-dg-op`},null,-1),i(`text`,{class:`dl-dg-lab is-sm`,x:`20`,y:`108`,"text-anchor":`middle`},`3`,-1),i(`circle`,{cx:`48`,cy:`226`,r:`12`,class:`dl-dg-op`},null,-1),i(`text`,{class:`dl-dg-lab is-sm`,x:`48`,y:`231`,"text-anchor":`middle`},`4`,-1),i(`text`,{class:`dl-dg-small`,x:`64`,y:`231`},`residual`,-1),i(`circle`,{cx:`620`,cy:`140`,r:`12`,class:`dl-dg-op`},null,-1),i(`text`,{class:`dl-dg-lab is-sm`,x:`620`,y:`145`,"text-anchor":`middle`},`5`,-1)]])),[[v]])]))]),e((u(),o(`div`,g,[..._[18]||=[i(`p`,null,[c(`Parts `),i(`strong`,null,`1`),c(` and `),i(`strong`,null,`2`),c(` → sections 02 and 03. Parts `),i(`strong`,null,`3`),c(` and `),i(`strong`,null,`4`),c(` → section 03. Part `),i(`strong`,null,`5`),c(` → section 04.`)],-1)]])),[[v]])]),_:1},16)}}};export{_ as default};
|
lecture-06/class/assets/md-Dw-XEgB42.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`class.md__slidev_33`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),32))),{default:t(()=>[p[0]||=r(`h1`,null,`Three kinds of transformer`,-1),p[1]||=r(`p`,null,`The same parts, wired three ways, for different jobs.`,-1),r(`div`,f,[n(m,{kind:`fork`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/class/assets/md-K0Oycy6G.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{E as e,Q as t,S as n,_ as r,_t as i,v as a,yt as o,z as s}from"./modules/shiki-BFOesOFp.js";import{n as c,t as l}from"./slidev/context-DrkI8qiD.js";import{t as u}from"./TransformerGlyph-D9TehqNe.js";import{t as d}from"./section-C2g2VRmZ.js";var f={class:`absolute right-18 top-1/2 -translate-y-1/2`},p={__name:`class.md__slidev_12`,setup(p){let{$slidev:m,$nav:h,$clicksContext:g,$clicks:_,$page:v,$renderContext:y,$frontmatter:b}=c();return g.setup(),(c,p)=>{let m=u;return s(),a(d,o(e(i(l)(i(b),11))),{default:t(()=>[p[0]||=r(`h1`,null,`Query, key, value`,-1),p[1]||=r(`p`,null,`A dictionary lookup, made soft.`,-1),r(`div`,f,[n(m,{kind:`qkv`,size:190})])]),_:1},16)}}};export{p as default};
|
lecture-06/class/assets/md-KdnLugHx2.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,V as i,W as a,_ as o,_t as s,b as c,g as l,lt as u,m as d,v as f,vt as p,w as m,x as h,xt as g,y as _,yt as v,z as y}from"./modules/shiki-BFOesOFp.js";import{t as b}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as x,t as S}from"./slidev/context-DrkI8qiD.js";import{t as C}from"./slidev/VClicks-Dmx1KjZv.js";import{n as w,t as T}from"./interactive-CI8hm4xw.js";var E={class:`dl-fam`},D={class:`dl-fam__tabs`,role:`tablist`},O=[`aria-selected`,`onClick`],k={class:`dl-fam__tabname`},A={class:`dl-fam__tabshape`},j={class:`dl-fam__panel`},M={class:`dl-fam__masks`},N={key:0,class:`dl-fam__mask`},P={class:`dl-fam__maskcap`},F={class:`dl-fam__grid`},I={key:1,class:`dl-fam__mask`},L={class:`dl-fam__maskcap`},R={class:`dl-fam__grid`},z={key:2,class:`dl-fam__mask`},B={class:`dl-fam__grid`},V={class:`dl-fam__facts`},H={class:`dl-fam__examples`},U={class:`dl-fam__year`},W=5,G=b(m({__name:`ModelFamilies`,setup(e){let{$slidev:t,$nav:r,$clicksContext:a,$clicks:s,$page:m,$renderContext:v,$frontmatter:b}=x(),S=[{key:`enc`,name:`Encoder-only`,shape:`the encoder stack, on its own`,mask:`full`,trained:`Mask out 15% of the tokens and predict them from both sides — “masked language modelling”.`,good:`Understanding a text you already have in full: classification, sentiment, named entities, retrieval embeddings, reranking.`,bad:`Cannot generate. There is no notion of “next”, because every position was trained while seeing the future.`,examples:[{name:`BERT`,year:2018},{name:`RoBERTa`,year:2019},{name:`DeBERTa`,year:2020},{name:`ModernBERT`,year:2024}]},{key:`dec`,name:`Decoder-only`,shape:`the decoder stack, with the cross-attention removed`,mask:`causal`,trained:`Predict the next token, over and over, on everything ever written. Every position is a training example, and no labels are needed.`,good:`Generation — and, as it turned out, nearly everything else, because any task phrased as text becomes next-token prediction.`,bad:`Each position only ever sees its left context, so a pure embedding of a whole sentence is weaker than an encoder’s.`,examples:[{name:`GPT-2`,year:2019},{name:`GPT-3`,year:2020},{name:`Llama`,year:2023},{name:`everything since`,year:2024}]},{key:`encdec`,name:`Encoder–decoder`,shape:`the 2017 original, both stacks`,mask:`both`,trained:`Map one sequence to another: translation, summarisation, or a span-corruption objective that rewrites the whole task as text-to-text.`,good:`Problems with a clear source and a clear target, where the source should be read bidirectionally and the target written left to right.`,bad:`Two stacks to train and serve, and a strict source/target split that a general-purpose chat model does not want.`,examples:[{name:`the 2017 paper`,year:2017},{name:`T5`,year:2019},{name:`BART`,year:2019},{name:`Whisper`,year:2022}]}],C=u(1),T=l(()=>S[C.value]);function G(e,t,n){return n===`full`||t<=e}return(e,t)=>{let r=w;return y(),f(r,{"max-width":`44rem`},{readout:n(()=>[T.value.key===`dec`?(y(),c(d,{key:0},[t[7]||=h(` One stack, one mask, one objective — and it ate the field. Not because it is the most expressive of the three, but because “predict the next token” needs `,-1),t[8]||=o(`strong`,null,`no labelled data at all`,-1),t[9]||=h(`, so it is the only one of the three whose training set is the whole internet. `,-1)],64)):T.value.key===`enc`?(y(),c(d,{key:1},[h(` Still the right answer when you have the whole text and want a vector for it. A sentence-embedding or reranking model in production today is far more likely to be encoder-only than to be a chat model. `)],64)):(y(),c(d,{key:2},[h(` The original, and the one all three of these descend from. It is still what you want when the source and the target are genuinely different things — which is why speech recognition and translation kept it long after chat models stopped using it. `)],64))]),default:n(()=>[o(`div`,E,[o(`div`,D,[(y(),c(d,null,i(S,(e,t)=>o(`button`,{key:e.key,type:`button`,role:`tab`,class:p([`dl-fam__tab`,{"is-current":t===C.value}]),"aria-selected":t===C.value,onClick:e=>C.value=t},[o(`span`,k,g(e.name),1),o(`span`,A,g(e.shape),1)],10,O)),64))]),o(`div`,j,[_(` The mask is the difference, so it leads. `),o(`div`,M,[T.value.mask===`causal`?_(`v-if`,!0):(y(),c(`div`,N,[o(`div`,P,g(T.value.mask===`both`?`source`:`all positions`),1),o(`div`,F,[(y(!0),c(d,null,i(25,e=>(y(),c(`span`,{key:e,class:p([`dl-fam__cell`,{"is-on":G(Math.floor((e-1)/W),(e-1)%W,`full`)}])},null,2))),128))]),t[0]||=o(`div`,{class:`dl-fam__maskrole`},`sees both ways`,-1)])),T.value.mask===`full`?_(`v-if`,!0):(y(),c(`div`,I,[o(`div`,L,g(T.value.mask===`both`?`target`:`all positions`),1),o(`div`,R,[(y(!0),c(d,null,i(25,e=>(y(),c(`span`,{key:e,class:p([`dl-fam__cell`,{"is-on":G(Math.floor((e-1)/W),(e-1)%W,`causal`)}])},null,2))),128))]),t[1]||=o(`div`,{class:`dl-fam__maskrole`},`left context only`,-1)])),T.value.mask===`both`?(y(),c(`div`,z,[t[2]||=o(`div`,{class:`dl-fam__maskcap`},`cross`,-1),o(`div`,B,[(y(!0),c(d,null,i(25,e=>(y(),c(`span`,{key:e,class:`dl-fam__cell is-on is-cross`}))),128))]),t[3]||=o(`div`,{class:`dl-fam__maskrole`},`target reads all of source`,-1)])):_(`v-if`,!0)]),o(`dl`,V,[o(`div`,null,[t[4]||=o(`dt`,null,`trained by`,-1),o(`dd`,null,g(T.value.trained),1)]),o(`div`,null,[t[5]||=o(`dt`,null,`use it for`,-1),o(`dd`,null,g(T.value.good),1)]),o(`div`,null,[t[6]||=o(`dt`,null,`cannot`,-1),o(`dd`,null,g(T.value.bad),1)])])]),o(`div`,H,[(y(!0),c(d,null,i(T.value.examples,e=>(y(),c(`span`,{key:e.name,class:`dl-fam__ex`},[h(g(e.name),1),o(`span`,U,g(e.year),1)]))),128))])])]),_:1})}}}),[[`__scopeId`,`data-v-be69a577`]]),K={class:`mt-2 dl-callout`},q={__name:`class.md__slidev_34`,setup(i){let{$slidev:l,$nav:u,$clicksContext:d,$clicks:p,$page:m,$renderContext:g,$frontmatter:_}=x();return d.setup(),(i,l)=>{let u=G,d=C,p=a(`click`);return y(),f(T,v(t(s(S)(s(_),33))),{aside:n(t=>[l[2]||=o(`p`,null,[h(`One paper, three descendants. The difference is `),o(`strong`,null,`the mask`),h(`, not the layer.`)],-1),r(d,null,{default:n(()=>[...l[0]||=[o(`ul`,null,[o(`li`,null,[o(`strong`,null,`encoder-only`),h(` — sees both ways, cannot generate`)]),o(`li`,null,[o(`strong`,null,`decoder-only`),h(` — left context only, generates`)]),o(`li`,null,[o(`strong`,null,`encoder–decoder`),h(` — the 2017 original, both`)])],-1)]]),_:1}),e((y(),c(`div`,K,[...l[1]||=[o(`p`,null,`Same blocks, same attention. Click through all three.`,-1)]])),[[p]])]),default:n(()=>[r(u)]),_:1},16)}}};export{q as default};
|
lecture-06/class/assets/md-kE1uizXw.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,S as r,W as i,_ as a,_t as o,b as s,v as c,yt as l,z as u}from"./modules/shiki-BFOesOFp.js";import{n as d,t as f}from"./slidev/context-DrkI8qiD.js";import{t as p}from"./TransformerGlyph-D9TehqNe.js";import{t as m}from"./default-Cq9fupKf.js";var h={class:`grid grid-cols-4 gap-x-4 gap-y-2 mt-3`},g={class:`flex flex-col items-center text-center gap-1`},_={class:`flex flex-col items-center text-center gap-1`},v={class:`flex flex-col items-center text-center gap-1`},y={class:`flex flex-col items-center text-center gap-1`},b={class:`flex flex-col items-center text-center gap-1`},x={class:`flex flex-col items-center text-center gap-1`},S={class:`flex flex-col items-center text-center gap-1`},C={class:`flex flex-col items-center text-center gap-1`},w={class:`mt-3 dl-callout`},T={__name:`class.md__slidev_2`,setup(T){let{$slidev:E,$nav:D,$clicksContext:O,$clicks:k,$page:A,$renderContext:j,$frontmatter:M}=d();return O.setup(),(d,T)=>{let E=p,D=i(`click`);return u(),c(m,l(t(o(f)(o(M),1))),{default:n(()=>[T[9]||=a(`h1`,null,`Today, in eight parts`,-1),T[10]||=a(`p`,null,`The whole machine first, then each part, then the kinds of transformer in use.`,-1),a(`div`,h,[a(`div`,g,[r(E,{kind:`cut`,size:64}),T[0]||=a(`span`,{class:`dl-secondary`},`00 Why a new design`,-1)]),a(`div`,_,[r(E,{kind:`encdec`,size:64}),T[1]||=a(`span`,{class:`dl-secondary`},`01 The whole architecture`,-1)]),a(`div`,v,[r(E,{kind:`qkv`,size:64}),T[2]||=a(`span`,{class:`dl-secondary`},`02 Attention: query, key, value`,-1)]),a(`div`,y,[r(E,{kind:`block`,size:64}),T[3]||=a(`span`,{class:`dl-secondary`},`03 One block`,-1)]),a(`div`,b,[r(E,{kind:`decoder`,size:64}),T[4]||=a(`span`,{class:`dl-secondary`},`04 Inside the decoder`,-1)]),a(`div`,x,[r(E,{kind:`fork`,size:64}),T[5]||=a(`span`,{class:`dl-secondary`},`05 Three kinds, and their uses`,-1)]),a(`div`,S,[r(E,{kind:`code`,size:64}),T[6]||=a(`span`,{class:`dl-secondary`},`06 In PyTorch`,-1)]),a(`div`,C,[r(E,{kind:`block`,size:64}),T[7]||=a(`span`,{class:`dl-secondary`},`07 Where that leaves us`,-1)])]),e((u(),s(`div`,w,[...T[8]||=[a(`p`,null,`Section 01 shows the full picture. Sections 02 to 04 open its boxes one by one.`,-1)]])),[[D]])]),_:1},16)}}};export{T as default};
|
lecture-06/class/assets/md-m6ElLsWQ.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,E as t,Q as n,W as r,_ as i,_t as a,b as o,v as s,yt as c,z as l}from"./modules/shiki-BFOesOFp.js";import{n as u,t as d}from"./slidev/context-DrkI8qiD.js";import{t as f}from"./default-Cq9fupKf.js";var p={class:`mt-2 flex justify-center`},m={viewBox:`0 0 680 220`,class:`dl-diagram`,role:`img`,"aria-label":`Wall one: the encoder squeezes the source into one vector c, and attention lines fix it. Wall two: a chain of 200 states, each waiting for the one before`},h={class:`mt-2 dl-callout`},g={__name:`class.md__slidev_4`,setup(g){let{$slidev:_,$nav:v,$clicksContext:y,$clicks:b,$page:x,$renderContext:S,$frontmatter:C}=u();return y.setup(),(u,g)=>{let _=r(`click`);return l(),s(f,c(t(a(d)(a(C),3))),{default:n(()=>[g[40]||=i(`h1`,null,`Two walls in reading one word at a time`,-1),g[41]||=i(`p`,null,`A recurrent encoder–decoder reads one word at a time. It hits two walls (hard limits). Attention removed only one.`,-1),i(`div`,p,[(l(),o(`svg`,m,[g[2]||=i(`defs`,null,[i(`marker`,{id:`l6s0-walls-head`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[i(`path`,{class:`dl-dg-head`,d:`M0 0 L7 3.5 L0 7 z`})])],-1),g[3]||=i(`text`,{class:`dl-dg-small`,x:`8`,y:`16`},`wall one — the bottleneck`,-1),g[4]||=i(`rect`,{class:`dl-dg-box`,x:`8`,y:`120`,width:`46`,height:`34`,rx:`4`},null,-1),g[5]||=i(`rect`,{class:`dl-dg-box`,x:`72`,y:`120`,width:`46`,height:`34`,rx:`4`},null,-1),g[6]||=i(`rect`,{class:`dl-dg-box`,x:`136`,y:`120`,width:`46`,height:`34`,rx:`4`},null,-1),g[7]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`31`,y:`142`,"text-anchor":`middle`},`h₁`,-1),g[8]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`95`,y:`142`,"text-anchor":`middle`},`h₂`,-1),g[9]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`159`,y:`142`,"text-anchor":`middle`},`h₃`,-1),g[10]||=i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-walls-head)`,d:`M54 137 H70`},null,-1),g[11]||=i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-walls-head)`,d:`M118 137 H134`},null,-1),g[12]||=i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-walls-head)`,d:`M182 137 H202`},null,-1),g[13]||=i(`rect`,{class:`dl-dg-box`,x:`204`,y:`124`,width:`30`,height:`26`,rx:`3`},null,-1),g[14]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`219`,y:`142`,"text-anchor":`middle`},`c`,-1),g[15]||=i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-walls-head)`,d:`M234 137 H254`},null,-1),g[16]||=i(`rect`,{class:`dl-dg-box is-accent`,x:`256`,y:`120`,width:`46`,height:`34`,rx:`4`},null,-1),g[17]||=i(`rect`,{class:`dl-dg-box is-accent`,x:`316`,y:`120`,width:`30`,height:`34`,rx:`4`},null,-1),g[18]||=i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-walls-head)`,d:`M302 137 H314`},null,-1),g[19]||=i(`text`,{class:`dl-dg-small`,x:`95`,y:`174`,"text-anchor":`middle`},`encoder`,-1),g[20]||=i(`text`,{class:`dl-dg-small`,x:`219`,y:`174`,"text-anchor":`middle`},`one vector`,-1),g[21]||=i(`text`,{class:`dl-dg-small`,x:`301`,y:`174`,"text-anchor":`middle`},`decoder`,-1),e((l(),o(`g`,null,[...g[0]||=[i(`path`,{class:`dl-dg-line`,d:`M331 118 C 290 40, 70 40, 31 118`},null,-1),i(`path`,{class:`dl-dg-line`,d:`M331 118 C 300 60, 125 60, 95 118`},null,-1),i(`path`,{class:`dl-dg-line`,d:`M331 118 C 310 80, 180 80, 159 118`},null,-1),i(`text`,{class:`dl-dg-small is-good`,x:`8`,y:`204`},`attention fixed it: a fresh weighted sum at every step`,-1)]])),[[_]]),g[22]||=i(`line`,{class:`dl-dg-split`,x1:`360`,y1:`8`,x2:`360`,y2:`212`},null,-1),g[23]||=i(`text`,{class:`dl-dg-small`,x:`378`,y:`16`},`wall two — no parallelism`,-1),g[24]||=i(`rect`,{class:`dl-dg-box`,x:`378`,y:`120`,width:`42`,height:`34`,rx:`4`},null,-1),g[25]||=i(`rect`,{class:`dl-dg-box`,x:`436`,y:`120`,width:`42`,height:`34`,rx:`4`},null,-1),g[26]||=i(`rect`,{class:`dl-dg-box`,x:`494`,y:`120`,width:`42`,height:`34`,rx:`4`},null,-1),g[27]||=i(`rect`,{class:`dl-dg-box`,x:`552`,y:`120`,width:`42`,height:`34`,rx:`4`},null,-1),g[28]||=i(`rect`,{class:`dl-dg-box`,x:`610`,y:`120`,width:`54`,height:`34`,rx:`4`},null,-1),g[29]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`399`,y:`142`,"text-anchor":`middle`},`h₁`,-1),g[30]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`457`,y:`142`,"text-anchor":`middle`},`h₂`,-1),g[31]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`515`,y:`142`,"text-anchor":`middle`},`h₃`,-1),g[32]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`573`,y:`142`,"text-anchor":`middle`},`…`,-1),g[33]||=i(`text`,{class:`dl-dg-lab is-sm`,x:`637`,y:`142`,"text-anchor":`middle`},`h₂₀₀`,-1),g[34]||=i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-walls-head)`,d:`M420 137 H434`},null,-1),g[35]||=i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-walls-head)`,d:`M478 137 H492`},null,-1),g[36]||=i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-walls-head)`,d:`M536 137 H550`},null,-1),g[37]||=i(`path`,{class:`dl-dg-arrow`,"marker-end":`url(#l6s0-walls-head)`,d:`M594 137 H608`},null,-1),g[38]||=i(`text`,{class:`dl-dg-small`,x:`378`,y:`182`},`200 tokens = 200 steps in a row`,-1),e((l(),o(`g`,null,[...g[1]||=[i(`rect`,{class:`dl-dg-box is-bad`,x:`370`,y:`110`,width:`302`,height:`54`,rx:`6`},null,-1),i(`text`,{class:`dl-dg-small is-bad`,x:`378`,y:`208`},`attention on top leaves this chain in place`,-1)]])),[[_]])]))]),e((l(),o(`div`,h,[...g[39]||=[i(`p`,null,`Two walls, one fix. This lecture removes the chain itself.`,-1)]])),[[_]])]),_:1},16)}}};export{g as default};
|
lecture-06/class/assets/slidev/404-Dkk3G2sr.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{Q as e,U as t,_ as n,_t as r,b as i,g as a,v as o,w as s,x as c,xt as l,y as u,z as d}from"../modules/shiki-BFOesOFp.js";import{t as f}from"../useNav-D8SbgLr9.js";import{t as p}from"../_plugin-vue_export-helper-BDNMzG2s.js";import{V as m}from"../modules/vue-Ymwlffw_.js";var h={class:`grid justify-center text-center pt-15% gap-5`},g={class:`text-2xl`},_={class:`op-60`},v={class:`mt-3 flex flex-col gap-2 max-w-xs mx-auto w-full`},y=p(s({__name:`404`,setup(s){let p=/\d+/,{currentRoute:y}=m(),{total:b}=f(),x=a(()=>{let e=y.value.path.match(p);if(e){let t=+e[0];if(t>0&&t<=b.value)return t}return null});return(a,s)=>{let f=t(`RouterLink`);return d(),i(`div`,h,[n(`div`,null,[s[2]||=n(`h1`,{class:`text-9xl font-light`},` 404 `,-1),n(`p`,g,[s[0]||=c(` Page `,-1),n(`code`,_,l(r(y).path),1),s[1]||=c(` not found `,-1)])]),n(`div`,v,[x.value===1?u(`v-if`,!0):(d(),o(f,{key:0,to:`/`,class:`page-link`},{default:e(()=>[...s[3]||=[c(` Go Home `,-1)]]),_:1})),x.value?(d(),o(f,{key:1,to:`/${x.value}`,class:`page-link`},{default:e(()=>[c(` Go to Slide `+l(x.value),1)]),_:1},8,[`to`])):u(`v-if`,!0)])])}}}),[[`__scopeId`,`data-v-b584852f`]]);export{y as default};
|
lecture-06/class/assets/slidev/CodeBlockWrapper-gsL5r1Ig.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{H as e,L as t,R as n,S as r,X as i,_ as a,_t as o,b as s,bt as c,g as l,k as u,lt as d,v as f,vt as p,w as m,xt as h,y as g,z as _}from"../modules/shiki-BFOesOFp.js";import{tt as v}from"../useNav-D8SbgLr9.js";import{u as y}from"../modules/vue-Ymwlffw_.js";import{a as b,l as x,n as S}from"./context-DrkI8qiD.js";import{d as C,u as w}from"../index-DvHcJSI0.js";import{a as T,i as E}from"../modules/unplugin-icons-BIrNarL0.js";var D=m({__name:`TitleIcon`,props:{title:{}},setup(e){let t={pnpm:`i-vscode-icons:file-type-light-pnpm`,npm:`i-vscode-icons:file-type-npm`,yarn:`i-vscode-icons:file-type-yarn`,bun:`i-vscode-icons:file-type-bun`,deno:`i-vscode-icons:file-type-deno`,vue:`i-vscode-icons:file-type-vue`,svelte:`i-vscode-icons:file-type-svelte`,angular:`i-vscode-icons:file-type-angular`,react:`i-vscode-icons:file-type-reactjs`,next:`i-vscode-icons:file-type-light-next`,nuxt:`i-vscode-icons:file-type-nuxt`,solid:`logos:solidjs-icon`,astro:`i-vscode-icons:file-type-light-astro`,rollup:`i-vscode-icons:file-type-rollup`,webpack:`i-vscode-icons:file-type-webpack`,vite:`i-vscode-icons:file-type-vite`,esbuild:`i-vscode-icons:file-type-esbuild`,"package.json":`i-vscode-icons:file-type-node`,"tsconfig.json":`i-vscode-icons:file-type-tsconfig`,".npmrc":`i-vscode-icons:file-type-npm`,".editorconfig":`i-vscode-icons:file-type-editorconfig`,".eslintrc":`i-vscode-icons:file-type-eslint`,".eslintignore":`i-vscode-icons:file-type-eslint`,"eslint.config":`i-vscode-icons:file-type-eslint`,".gitignore":`i-vscode-icons:file-type-git`,".gitattributes":`i-vscode-icons:file-type-git`,".env":`i-vscode-icons:file-type-dotenv`,".env.example":`i-vscode-icons:file-type-dotenv`,".vscode":`i-vscode-icons:file-type-vscode`,"tailwind.config":`vscode-icons:file-type-tailwind`,"uno.config":`i-vscode-icons:file-type-unocss`,"unocss.config":`i-vscode-icons:file-type-unocss`,".oxlintrc":`i-vscode-icons:file-type-oxlint`,"vue.config":`i-vscode-icons:file-type-vueconfig`,".mts":`i-vscode-icons:file-type-typescript`,".cts":`i-vscode-icons:file-type-typescript`,".ts":`i-vscode-icons:file-type-typescript`,".tsx":`i-vscode-icons:file-type-typescript`,".mjs":`i-vscode-icons:file-type-js`,".cjs":`i-vscode-icons:file-type-js`,".json":`i-vscode-icons:file-type-json`,".js":`i-vscode-icons:file-type-js`,".jsx":`i-vscode-icons:file-type-js`,".md":`i-vscode-icons:file-type-markdown`,".py":`i-vscode-icons:file-type-python`,".ico":`i-vscode-icons:file-type-favicon`,".html":`i-vscode-icons:file-type-html`,".css":`i-vscode-icons:file-type-css`,".scss":`i-vscode-icons:file-type-scss`,".yml":`i-vscode-icons:file-type-light-yaml`,".yaml":`i-vscode-icons:file-type-light-yaml`,".php":`i-vscode-icons:file-type-php`,".svg":`i-vscode-icons:file-type-svg`};function n(e){let n=e.match(/~([^~]+)~/g);if(n&&n.length>0)return n[0].slice(1,-1);let r=Object.keys(t).sort((e,t)=>t.length-e.length);for(let n of r)if(e.toLowerCase().includes(n.toLowerCase()))return t[n];return``}return(t,r)=>n(e.title)?(_(),s(`div`,{key:0,class:p([`w-3.5 h-3.5 relative`,n(e.title)])},null,2)):g(`v-if`,!0)}}),O=[`data-title`],k={key:0,class:`slidev-code-block-title`},A={class:`leading-1em`},j=[`title`],M=m({__name:`CodeBlockWrapper`,props:{ranges:{type:Array,default:()=>[]},finally:{type:[String,Number],default:`last`},startLine:{type:Number,default:1},lines:{type:Boolean,default:v.lineNumbers},at:{type:[String,Number],default:`+1`},maxHeight:{type:String,default:void 0},title:{type:String,default:``}},setup(m){let M=m,{$frontmatter:N,$clicksContext:P}=S(),F=l(()=>N?.codeCopy??v.codeCopy),I=d(),L=w();n(()=>{P.unregister(L)}),i(()=>{I.value?.classList.toggle(`slidev-code-line-numbers`,M.lines)}),t(()=>{if(!P||!M.ranges?.length)return;let e=P.calculateSince(M.at,M.ranges.length-1);P.register(L,e);let t=l(()=>e?Math.max(0,P.current-e.start+1):x),n=l(()=>M.finally===`last`?M.ranges.at(-1):M.finally.toString());i(()=>{if(!I.value)return;let e=M.ranges[t.value]??n.value,r=e===`hide`;I.value.classList.toggle(b,r),r&&(e=M.ranges[t.value+1]??n.value);let i=I.value.querySelector(`.shiki`),a=Array.from(i.querySelectorAll(`code > .line`)),o=a.length;if(C(e,o,M.startLine,e=>[a[e]]),M.maxHeight){let e=Array.from(i.querySelectorAll(`.line.highlighted`));e.reduce((e,t)=>t.offsetHeight+e,0)>I.value.offsetHeight?e[0].scrollIntoView({behavior:`smooth`,block:`start`}):e.length>0&&e[Math.round((e.length-1)/2)].scrollIntoView({behavior:`smooth`,block:`center`})}})});let{copied:R,copy:z}=y();function B(){let e=I.value?.querySelector(`.slidev-code`)?.textContent;e&&z(e)}let V=u(`activeTitle`,null),H=l(()=>V===null&&M.title);return(t,n)=>{let i=E,l=T;return _(),s(`div`,{ref_key:`el`,ref:I,class:p([`slidev-code-wrapper relative group`,{"slidev-code-line-numbers":M.lines,active:o(V)===m.title}]),style:c({"max-height":M.maxHeight,"overflow-y":M.maxHeight?`scroll`:void 0,"--start":M.startLine}),"data-title":m.title},[H.value?(_(),s(`div`,k,[r(D,{title:m.title},null,8,[`title`]),a(`div`,A,h(m.title.replace(/~([^~]+)~/g,``).trim()),1)])):g(`v-if`,!0),e(t.$slots,`default`),F.value?(_(),s(`button`,{key:1,class:p([`slidev-code-copy absolute right-0 transition opacity-0 group-hover:opacity-20 hover:!opacity-100`,H.value?`top-10`:`top-0`]),title:o(R)?`Copied`:`Copy`,onClick:n[0]||=e=>B()},[o(R)?(_(),f(i,{key:0,class:`p-2 w-8 h-8`})):(_(),f(l,{key:1,class:`p-2 w-8 h-8`}))],10,j)):g(`v-if`,!0)],14,O)}}});export{M as t};
|
lecture-06/class/assets/slidev/DrawingControls-cUpksemn.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{n as e}from"../rolldown-runtime-hePW80VL.js";import{$ as t,H as n,Q as r,S as i,V as a,_ as o,_t as s,b as c,bt as l,l as u,lt as d,m as f,r as p,u as m,v as h,vt as g,w as _,xt as v,y,z as b}from"../modules/shiki-BFOesOFp.js";import{d as x,g as S}from"../modules/vue-Ymwlffw_.js";import{t as C}from"../useDrawings-BkbsP5qo.js";import{t as w}from"./IconButton-B5keGFol.js";import{t as T}from"./VerticalDivider-D_ScNBWw.js";var E=_({__name:`Draggable`,props:{storageKey:{},initial:{}},setup(e){let t=e,r=d(null),i=t.initial??{x:0,y:0},a=t.storageKey?S(t.storageKey,i):d(i),{style:o}=x(r,{initialValue:a});return(e,t)=>(b(),c(`div`,{ref_key:`el`,ref:r,class:`fixed`,style:l(s(o))},[n(e.$slots,`default`)],4))}}),D={class:`flex bg-main p-2`},O={class:`inline-block w-7 text-center`},k={class:`pt-.5`},A={class:`i-carbon:pin-filled transform -rotate-45`},j={class:`i-carbon:pin`},M={class:`i-carbon:error`},N={class:`i-carbon:close-outline`},P=_({__name:`DrawingControls`,setup(e){let{brush:n,canClear:_,canRedo:x,canUndo:S,clear:P,drauu:F,drawingEnabled:I,drawingMode:L,drawingPinned:R,brushColors:z}=C(),B=d(!1);function V(e){e.preventDefault(),e.stopPropagation(),requestAnimationFrame(()=>{requestAnimationFrame(()=>{B.value=!0})})}function H(e){e.pointerType!==`mouse`&&V(e)}function U(){F.undo()}function W(){F.redo()}let G=`stylus`;function K(e){L.value=e,I.value=!0,e!==`eraseLine`&&(G=e)}function q(e){n.value.color=e,I.value=!0,L.value=G}return(e,d)=>s(I)||s(R)?(b(),h(E,{key:0,class:g([`flex flex-wrap text-xl p-2 gap-1 rounded-md bg-main shadow transition-opacity duration-200 z-nav border border-main`,!s(I)&&s(R)?`opacity-40 hover:opacity-90`:``]),"storage-key":`slidev-drawing-pos`,"initial-x":10,"initial-y":10},{default:r(()=>[i(w,{title:`Draw with stylus`,class:g({shallow:s(L)!==`stylus`}),onClick:d[0]||=e=>K(`stylus`)},{default:r(()=>[...d[14]||=[o(`div`,{class:`i-carbon:pen`},null,-1)]]),_:1},8,[`class`]),i(w,{title:`Draw a line`,class:g({shallow:s(L)!==`line`}),onClick:d[1]||=e=>K(`line`)},{default:r(()=>[...d[15]||=[o(`svg`,{width:`1em`,height:`1em`,class:`-mt-0.5`,preserveAspectRatio:`xMidYMid meet`,viewBox:`0 0 24 24`},[o(`path`,{d:`M21.71 3.29a1 1 0 0 0-1.42 0l-18 18a1 1 0 0 0 0 1.42a1 1 0 0 0 1.42 0l18-18a1 1 0 0 0 0-1.42z`,fill:`currentColor`})],-1)]]),_:1},8,[`class`]),i(w,{title:`Draw an arrow`,class:g({shallow:s(L)!==`arrow`}),onClick:d[2]||=e=>K(`arrow`)},{default:r(()=>[...d[16]||=[o(`div`,{class:`i-carbon:arrow-up-right`},null,-1)]]),_:1},8,[`class`]),i(w,{title:`Draw an ellipse`,class:g({shallow:s(L)!==`ellipse`}),onClick:d[3]||=e=>K(`ellipse`)},{default:r(()=>[...d[17]||=[o(`div`,{class:`i-carbon:radio-button`},null,-1)]]),_:1},8,[`class`]),i(w,{title:`Draw a rectangle`,class:g({shallow:s(L)!==`rectangle`}),onClick:d[4]||=e=>K(`rectangle`)},{default:r(()=>[...d[18]||=[o(`div`,{class:`i-carbon:checkbox`},null,-1)]]),_:1},8,[`class`]),i(w,{title:`Erase`,class:g({shallow:s(L)!==`eraseLine`}),onClick:d[5]||=e=>K(`eraseLine`)},{default:r(()=>[...d[19]||=[o(`div`,{class:`i-carbon:erase`},null,-1)]]),_:1},8,[`class`]),i(T),i(s(p),{shown:B.value,"onUpdate:shown":d[8]||=e=>B.value=e,triggers:[]},{popper:r(()=>[o(`div`,D,[o(`div`,O,v(s(n).size),1),o(`div`,k,[t(o(`input`,{"onUpdate:modelValue":d[6]||=e=>s(n).size=e,type:`range`,min:`1`,max:`15`,onChange:d[7]||=e=>L.value=s(G)},null,544),[[u,s(n).size]])])])]),default:r(()=>[i(w,{title:`Adjust stroke width`,class:g({shallow:s(L)===`eraseLine`}),onClick:V,onPointerdown:H,onPointerup:H,onTouchend:V},{default:r(()=>[...d[20]||=[o(`svg`,{viewBox:`0 0 32 32`,width:`1.2em`,height:`1.2em`},[o(`line`,{x1:`2`,y1:`15`,x2:`22`,y2:`4`,stroke:`currentColor`,"stroke-width":`1`,"stroke-linecap":`round`}),o(`line`,{x1:`2`,y1:`24`,x2:`28`,y2:`10`,stroke:`currentColor`,"stroke-width":`2`,"stroke-linecap":`round`}),o(`line`,{x1:`7`,y1:`31`,x2:`29`,y2:`19`,stroke:`currentColor`,"stroke-width":`3`,"stroke-linecap":`round`})],-1)]]),_:1},8,[`class`])]),_:1},8,[`shown`]),(b(!0),c(f,null,a(s(z),e=>(b(),h(w,{key:e,title:`Set brush color`,class:g(s(n).color===e&&s(L)!==`eraseLine`?`active`:`shallow`),onClick:t=>q(e)},{default:r(()=>[o(`div`,{class:g([`w-6 h-6 transition-all transform border`,s(n).color===e?`rounded-md border-gray-300/50`:`rounded-1/2 scale-85 border-white`]),style:l(s(I)?{background:e}:{borderColor:e})},null,6)]),_:2},1032,[`class`,`onClick`]))),128)),i(T),i(w,{title:`Undo`,class:g({disabled:!s(S)}),onClick:d[9]||=e=>U()},{default:r(()=>[...d[21]||=[o(`div`,{class:`i-carbon:undo`},null,-1)]]),_:1},8,[`class`]),i(w,{title:`Redo`,class:g({disabled:!s(x)}),onClick:d[10]||=e=>W()},{default:r(()=>[...d[22]||=[o(`div`,{class:`i-carbon:redo`},null,-1)]]),_:1},8,[`class`]),i(w,{title:`Delete`,class:g({disabled:!s(_)}),onClick:d[11]||=e=>s(P)()},{default:r(()=>[...d[23]||=[o(`div`,{class:`i-carbon:trash-can`},null,-1)]]),_:1},8,[`class`]),i(T),i(w,{title:s(R)?`Unpin drawing`:`Pin drawing`,class:g({shallow:!s(R)}),onClick:d[12]||=e=>R.value=!s(R)},{default:r(()=>[t(o(`div`,A,null,512),[[m,s(R)]]),t(o(`div`,j,null,512),[[m,!s(R)]])]),_:1},8,[`title`,`class`]),s(I)?(b(),h(w,{key:0,title:s(R)?`Drawing pinned`:`Drawing unpinned`,class:g({shallow:!s(I)}),onClick:d[13]||=e=>I.value=!s(I)},{default:r(()=>[t(o(`div`,M,null,512),[[m,s(R)]]),t(o(`div`,N,null,512),[[m,!s(R)]])]),_:1},8,[`title`,`class`])):y(`v-if`,!0)]),_:1},8,[`class`])):y(`v-if`,!0)}}),F=e({default:()=>I}),I=P;export{F as n,I as t};
|
lecture-06/class/assets/slidev/DrawingLayer-BLLWTnj4.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{F as e,L as t,Y as n,_t as r,b as i,lt as a,vt as o,w as s,z as c}from"../modules/shiki-BFOesOFp.js";import{n as l}from"./context-DrkI8qiD.js";import{t as u}from"../useDrawings-BkbsP5qo.js";var d=s({__name:`DrawingLayer`,setup(s){let{drauu:d,drawingEnabled:f,loadCanvas:p}=u(),m=l().$scale,h=a();return t(()=>{d.mount(h.value,h.value.parentElement),n(m,e=>d.options.coordinateScale=1/e,{immediate:!0}),p()}),e(()=>{d.unmount()}),(e,t)=>(c(),i(`svg`,{ref_key:`svg`,ref:h,class:o([`w-full h-full absolute top-0`,{"pointer-events-none":!r(f),"touch-none":r(f)}])},null,2))}});export{d as default};
|
lecture-06/class/assets/slidev/KaTexBlockWrapper-BIlBldVC.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{H as e,L as t,R as n,X as r,b as i,g as a,lt as o,w as s,z as c}from"../modules/shiki-BFOesOFp.js";import{H as l}from"../useNav-D8SbgLr9.js";import{a as u,c as d,l as f,n as p}from"./context-DrkI8qiD.js";import{u as m}from"../index-DvHcJSI0.js";var h=`.mtable > [class*=col-align]`,g=`:scope > .vlist-t > .vlist-r > .vlist > span > .mord`;function _(e){let t=Array.from(e.querySelectorAll(h)).filter(e=>!e.parentElement?.closest(h)),n=[];for(let e of t)Array.from(e.querySelectorAll(g)).forEach((e,t)=>{e&&(Array.isArray(n[t])?n[t].push(e):n[t]=[e])});return n}var v=s({__name:`KaTexBlockWrapper`,props:{ranges:{type:Array,default:()=>[]},finally:{type:[String,Number],default:`last`},startLine:{type:Number,default:1},at:{type:[String,Number],default:`+1`}},setup(s){let h=s,{$clicksContext:g}=p(),v=o(),y=m();return n(()=>{g.unregister(y)}),t(()=>{if(!g||!h.ranges?.length)return;let e=g.calculateSince(h.at,h.ranges.length-1);g.register(y,e);let t=a(()=>e?Math.max(0,g.current-e.start+1):f),n=a(()=>h.finally===`last`?h.ranges.at(-1):h.finally.toString());r(()=>{if(!v.value)return;let e=h.ranges[t.value]??n.value,r=e===`hide`;v.value.classList.toggle(u,r),r&&(e=h.ranges[t.value+1]??n.value);let i=_(v.value);if(!i.length)return;let a=h.startLine,o=l(i.length+a-1,e);i.forEach((e,t)=>{let n=o.includes(t+a);e.forEach(e=>{e.classList.toggle(d,!0),e.classList.toggle(`highlighted`,n),e.classList.toggle(`dishonored`,!n)})})})}),(t,n)=>(c(),i(`div`,{ref_key:`el`,ref:v,class:`slidev-katex-wrapper`},[e(t.$slots,`default`)],512))}});export{v as t};
|
lecture-06/class/assets/slidev/NoteDisplay-6LSM8xpP.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{L as e,N as t,V as n,X as r,Y as i,_ as a,_t as o,b as s,bt as c,g as l,it as u,lt as d,m as f,vt as p,w as m,xt as h,y as g,z as _}from"../modules/shiki-BFOesOFp.js";import{nt as v,rt as y}from"../useNav-D8SbgLr9.js";import{l as b}from"./context-DrkI8qiD.js";var x=[`title`],S={key:0,"flex-auto":``},C={"text-primary":``},w={op50:``,"text-sm":``},T={key:0,absolute:``,"inset-y-0":``,"right-0":``,"w-0.5":``,"bg-primary":``,"z-1":``},E=m({__name:`ClicksSlider`,props:{clicksContext:{},readonly:{type:Boolean},active:{type:Boolean,default:!0},resettable:{type:Boolean},compact:{type:Boolean},attached:{type:Boolean}},emits:[`activate`,`reset`],setup(e,{emit:t}){let r=e,i=t,m=l(()=>r.clicksContext.total),E=l(()=>v(0,r.clicksContext.clicksStart,m.value)),D=l(()=>m.value-E.value+1),O=l({get(){return r.resettable&&!r.active||r.clicksContext.current>m.value?-1:r.clicksContext.current},set(e){if(r.resettable&&e<0){i(`reset`),r.clicksContext.current=b;return}i(`activate`),r.clicksContext.current=e}}),k=l(()=>r.resettable&&O.value<0),A=l(()=>y(E.value,m.value+1)),j=d(),M;function N(e){let t=j.value.getBoundingClientRect();return(e.clientX-t.left)/Math.max(1,t.width)}function P(e,t){if(r.readonly||!j.value||!t&&!(e.buttons&1))return;let n=N(e);if(r.resettable&&n<0){O.value=-1;return}let i=v(0,n,t?.999999:1)*D.value,a=v(0,O.value-E.value,D.value-1),o=t?E.value+Math.floor(i):O.value;!t&&i>=a+1.5?o=E.value+Math.floor(i-.5):!t&&i<a-.5&&(o=E.value+Math.ceil(i-.5)),O.value=v(E.value,o,m.value)}function F(e){r.readonly||(j.value?.setPointerCapture(e.pointerId),M={id:e.pointerId,x:e.clientX,y:e.clientY},P(e,!0))}function I(e){if(M?.id===e.pointerId){if(Math.abs(e.clientX-M.x)<=3&&Math.abs(e.clientY-M.y)<=3)return;M=void 0}P(e,!1)}return(t,i)=>(_(),s(`div`,{class:p([`flex gap-1 select-none`,[e.attached?`items-end`:`items-center`,D.value&&r.clicksContext.isMounted?``:`op50`]]),title:`Clicks in this slide: ${D.value}`},[a(`div`,{class:p([`flex items-center font-mono`,[e.compact?`gap-1 min-w-0 mr0`:`gap-0.2 min-w-16 mr1`,e.attached?`h-[22px]`:``]])},[a(`div`,{class:p([`i-carbon:cursor-1 text-sm op50`,e.compact?`ml-1`:``])},null,2),O.value>=0&&O.value!==o(999999)&&e.active?(_(),s(f,{key:0},[e.compact?g(`v-if`,!0):(_(),s(`div`,S)),a(`span`,null,[a(`span`,C,h(O.value),1),i[2]||=a(`span`,{op25:``,"text-sm":``},`/`,-1),a(`span`,w,h(m.value),1)])],64)):(_(),s(`div`,{key:1,op50:``,class:p(e.compact?``:`flex-auto pl1`)},[a(`span`,{class:p(e.compact?`inline-block text-center`:``),style:c(e.compact?{width:`${String(m.value).length*2+1}ch`,marginLeft:`-0.25ch`}:void 0)},h(m.value),7)],2))],2),a(`div`,{ref_key:`sliderEl`,ref:j,relative:``,"flex-auto":``,"font-mono":``,flex:`~`,"touch-none":``,class:p([e.attached?`h-[22px]`:`h5`,k.value?`op80`:``]),onPointerdownCapture:F,onPointermove:I,onPointerup:i[0]||=e=>u(M)?M.value=void 0:M=void 0,onPointercancel:i[1]||=e=>u(M)?M.value=void 0:M=void 0},[(_(!0),s(f,null,n(A.value,t=>(_(),s(`div`,{key:t,border:`y main`,"of-hidden":``,relative:``,class:p([t===0?`border-l`:``,t===0?e.attached?`rounded-tl`:`rounded-l`:``,t===m.value?`border-r`:``,t===m.value&&+t!=+O.value?e.attached?`rounded-tr`:`rounded-r`:``,e.attached?`border-b-0`:``]),style:c({width:D.value>0?`${1/D.value*100}%`:`100%`})},[a(`div`,{absolute:``,"inset-0":``,class:p(t<=O.value&&e.active?`bg-primary op15`:``)},null,2),+t==+O.value&&e.active?(_(),s(`div`,T)):g(`v-if`,!0),a(`div`,{class:p([+t==+O.value&&e.active?`text-primary font-bold op100`:`op30`,t===m.value?``:`border-r-2 border-main`]),"w-full":``,"h-full":``,"text-xs":``,flex:``,"items-center":``,"justify-center":``,"z-1":``},h(t),3)],6))),128))],34)],10,x))}}),D=[`innerHTML`],O=[`textContent`],k=[`textContent`],A=`slidev-note-fade`,j=`slidev-note-click-mark`,M=m({__name:`NoteDisplay`,props:{class:{},noteHtml:{},note:{},highlight:{type:Boolean,default:!0},placeholder:{},clicksContext:{},autoScroll:{type:Boolean}},emits:[`markerDblclick`,`markerClick`],setup(n,{emit:o}){let c=n,u=o,f=l(()=>c.clicksContext!=null&&c.noteHtml?.includes(j)),m=d(null);function g(){if(!m.value||!f.value)return;let e=Array.from(m.value.querySelectorAll(`.${j}`)),t=new Map,n=new Map,r=0;for(let i of e){let e=Number(i.dataset.clicks);t.set(i,e);let a=i,o=i.parentElement;for(;o&&a!==m.value;)n.has(o)||n.set(o,[[null,r]]),n.get(o).push([a,e]),a=o,o=o.parentElement;r=e}let i=new Map;for(let[e,t]of n){let n=!1,r=0;for(let a of Array.from(e.childNodes)){let o=!1;for(;a===t[r+1]?.[0];)o=!0,r++;if(o)continue;let s=a;if(a.nodeType===3){if(!a.textContent?.trim())continue;s=document.createElement(`span`),s.textContent=a.textContent,e.insertBefore(s,a),a.remove()}n||=r===0,i.set(s,t[r][1])}n||(t[0][1]=-1)}return e=>{let r=c.highlight;for(let[t,i]of n)t.classList.toggle(A,r&&!i.some(([t,n])=>n===e));for(let[t,n]of i)t.classList.toggle(A,r&&n!==e);for(let[n,i]of t)n.classList.remove(A),n.classList.toggle(`${j}-past`,r&&i<e),n.classList.toggle(`${j}-active`,r&&i===e),n.classList.toggle(`${j}-next`,r&&i===e+1),n.classList.toggle(`${j}-future`,r&&i>e+1),n.ondblclick=e=>{r&&(u(`markerDblclick`,e,i),!e.defaultPrevented&&(c.clicksContext.current=i,e.stopPropagation(),e.stopImmediatePropagation()))},n.onclick=e=>{r&&u(`markerClick`,e,i)},r&&c.autoScroll&&i===e&&n.scrollIntoView({block:`center`,behavior:`smooth`})}}let v=d();return i(()=>[c.noteHtml,c.highlight],()=>{t(()=>{v.value=g()})},{immediate:!0}),e(()=>{g()}),r(()=>{let e=c.clicksContext?.current??999999;v.value?.(e)}),(e,t)=>n.noteHtml?(_(),s(`div`,{key:0,ref_key:`noteDisplay`,ref:m,class:p([`prose dark:prose-invert overflow-auto outline-none slidev-note`,[c.class,f.value?`slidev-note-with-clicks`:``]]),innerHTML:n.noteHtml},null,10,D)):n.note?(_(),s(`div`,{key:1,class:p([`prose dark:prose-invert overflow-auto outline-none slidev-note`,c.class])},[a(`p`,{textContent:h(n.note)},null,8,O)],2)):(_(),s(`div`,{key:2,class:p([`prose dark:prose-invert overflow-auto outline-none opacity-50 italic select-none slidev-note`,c.class])},[a(`p`,{textContent:h(c.placeholder||`No notes.`)},null,8,k)],2))}});export{E as n,M as t};
|
lecture-06/class/assets/slidev/NoteEditable-DTwQcEqa.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{$ as e,N as t,X as n,Y as r,_t as i,b as a,bt as o,d as s,l as c,lt as l,mt as u,v as d,vt as f,w as p,z as m}from"../modules/shiki-BFOesOFp.js";import{C as h,k as g,o as _}from"../modules/vue-Ymwlffw_.js";import{t as v}from"../index-DvHcJSI0.js";import{t as y}from"./NoteDisplay-6LSM8xpP.js";var b=[`placeholder`],x=p({__name:`NoteEditable`,props:{no:{type:Number,required:!0},class:{default:``},editing:{default:!1},style:{default:()=>({})},placeholder:{default:`No notes for this slide`},clicksContext:{type:Object},highlight:{default:!0},autoHeight:{default:!1}},emits:[`update:editing`,`markerDblclick`,`markerClick`],setup(p,{emit:x}){let S=p,C=x,w=h(S,`editing`,C,{passive:!0}),{info:T,update:E}=v(u(S,`no`)),D=l(``),O,{ignoreUpdates:k}=g(D,e=>{if(!w.value)return;let t=S.no;clearTimeout(O),O=setTimeout(()=>{E({note:e},t)},500)});r(()=>T.value?.note,(e=``)=>{w.value||(clearTimeout(O),k(()=>{D.value=e}))},{immediate:!0,flush:`sync`});let A=l(),j=l();n(()=>{w.value&&A.value?.focus()}),_(A,()=>{w.value=!1});function M(){!S.autoHeight||!A.value||!w.value||A.value.scrollHeight>A.value.clientHeight&&(A.value.style.height=`${A.value.scrollHeight}px`)}function N(e){w.value&&e.metaKey&&e.key===`s`&&(e.preventDefault(),E({note:D.value},S.no))}return r([D,w],()=>{t(()=>{M()})},{flush:`post`,immediate:!0}),(t,n)=>i(w)?e((m(),a(`textarea`,{key:1,ref_key:`inputEl`,ref:A,"onUpdate:modelValue":n[2]||=e=>D.value=e,class:f([`prose dark:prose-invert resize-none overflow-auto outline-none bg-transparent block border-primary border-2 slidev-note placeholder:op25`,[S.class,D.value?``:`italic`]]),style:o([S.style,j.value==null?{}:{height:`${j.value}px`}]),placeholder:p.placeholder,onKeydown:[n[3]||=s(e=>w.value=!1,[`esc`]),N]},null,46,b)),[[c,D.value]]):(m(),d(y,{key:0,class:f([`border-transparent border-2`,[S.class,D.value?``:`opacity-25 italic select-none`]]),style:o(S.style),note:D.value||p.placeholder,"note-html":i(T)?.noteHTML,"clicks-context":p.clicksContext,"auto-scroll":!p.autoHeight,highlight:S.highlight,onMarkerClick:n[0]||=(e,t)=>C(`markerClick`,e,t),onMarkerDblclick:n[1]||=(e,t)=>C(`markerDblclick`,e,t)},null,8,[`class`,`style`,`note`,`note-html`,`clicks-context`,`auto-scroll`,`highlight`]))}});export{x as t};
|
lecture-06/class/assets/slidev/TimerBar-1xtsicrJ.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{Q as e,S as t,V as n,_t as r,a as i,b as a,bt as o,g as s,m as c,mt as l,st as u,vt as d,w as f,y as p,z as m}from"../modules/shiki-BFOesOFp.js";import{U as h,t as g,tt as _}from"../useNav-D8SbgLr9.js";import{F as v}from"../modules/vue-Ymwlffw_.js";import{o as y}from"../index-DvHcJSI0.js";var b={class:`relative flex gap-px`},x=f({__name:`CurrentProgressBar`,props:{clicksContext:{},current:{}},setup(l){let u=l,f=g(),h=s(()=>u.clicksContext??f.clicksContext.value),_=s(()=>u.current??f.currentSlideNo.value),{total:v}=f;return(s,l)=>(m(),a(`div`,b,[(m(!0),a(c,null,n(r(v)-1,n=>(m(),a(`div`,{key:n,class:d([`border-x border-b border-main h-4px transition-all`,n<_.value?`bg-primary border-primary`:``]),style:o({width:`${1/(r(v)-1)*100}%`})},[t(i,{name:`fade`},{default:e(()=>[n===_.value?(m(),a(`div`,{key:0,class:`h-full bg-primary op75 transition-all`,style:o({width:`${h.value.total===0?0:h.value.current/(h.value.total+1)*100}%`})},null,4)):p(`v-if`,!0)]),_:2},1024)],6))),128))]))}});function S(){let e=s(()=>_.timer||`stopwatch`),t=s(()=>h(_.duration).seconds),n=v(100,{controls:!0}),r=l(y,`timer`),i=s(()=>r.value?.status),a=s(()=>(n.counter.value,r.value.status===`stopped`||!r.value.startedAt?0:r.value.status===`paused`?r.value.pausedAt-r.value.startedAt:Date.now()-r.value.startedAt)),o=s(()=>a.value/1e3),c=s(()=>o.value/t.value*100),u=s(()=>{if(e.value===`stopwatch`&&(r.value.status===`stopped`||!r.value.startedAt))return{h:``,m:`-`,s:`--`,ms:`-`};let n=e.value===`countdown`?t.value*1e3-a.value:a.value,i=Math.floor(n/1e3/60/60).toString();i===`0`&&(i=``);let o=Math.floor(n/1e3/60%60).toString();i&&(o=o.padStart(2,`0`));let s=Math.floor(n/1e3%60).toString().padStart(2,`0`),c=Math.floor(n%1e3/100).toString();return{h:i,m:o,s,ms:c}});function d(){n.pause(),r.value={status:`stopped`,slides:{},startedAt:0,pausedAt:0}}function f(){r.value&&(r.value?.status===`stopped`?(r.value.status=`running`,r.value.startedAt=Date.now()):r.value.status===`paused`&&(r.value.status=`running`,r.value.startedAt=Date.now()-(r.value.pausedAt-r.value.startedAt)),n.resume())}function p(){r.value.status=`paused`,r.value.pausedAt=Date.now(),n.pause()}function m(){r.value.status===`running`?p():f()}return{state:r,status:i,timer:u,reset:d,toggle:m,resume:f,pause:p,passed:o,percentage:c,duration:t,mode:e}}var C={class:`border-b mt-px border-main relative flex h-4px`},w=f({__name:`TimerBar`,setup(e){let t=u(S()),n=s(()=>t.status===`stopped`?`op50`:t.status===`paused`?`bg-blue`:t.percentage>100?`bg-red`:t.percentage>80?`bg-yellow`:`bg-green`);return(e,r)=>(m(),a(`div`,C,[t.status===`stopped`?p(`v-if`,!0):(m(),a(`div`,{key:0,class:d([`h-4px`,n.value]),style:o({width:`${t.percentage}%`})},null,6)),p(` {{ timesplits }} `)]))}});export{S as n,x as r,w as t};
|