File size: 8,163 Bytes
157ee49 | 1 | import{$ as e,K as t,S as n,St as r,V as i,_ as a,b as o,c as s,et as c,g as l,m as u,ut as d,v as f,vt as p,w as m,x as h,xt as g,y as _,z as v}from"./modules/shiki-CkJq9tSf.js";import{t as y}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as b}from"./slidev/context-DnYiQGy6.js";import{a as x,f as S}from"./useRecurrence-PhWWMav6.js";import{n as C}from"./interactive-CigWxCoZ.js";import{t as w}from"./Slider-Bf0ITYx4.js";import{n as T,r as E,t as D}from"./PlotLabel-DHDPmYNN.js";var O={class:`dl-curve`},k=[`d`,`fill`],A=[`d`,`stroke`,`stroke-width`,`stroke-dasharray`],j=m({__name:`PlotCurve`,props:{fn:{},points:{},samples:{default:160},color:{},width:{default:2.5},dashed:{type:Boolean,default:!1},fillTo:{},opacity:{default:1}},setup(e){let{$slidev:t,$nav:n,$clicksContext:r,$clicks:a,$page:s,$renderContext:c,$frontmatter:d}=b(),f=e,p=E(),m=l(()=>{if(f.points)return f.points.map(e=>Array.isArray(e)?{x:e[0],y:e[1]}:e);if(!f.fn)return[];let[e,t]=p.xDomain.value,n=Math.max(f.samples,2),r=[];for(let i=0;i<n;i++){let a=e+(t-e)*i/(n-1);r.push({x:a,y:f.fn(a)})}return r}),h=l(()=>{let[e,t]=p.yDomain.value,n=(t-e)*4,r=[],i=[];for(let a of m.value){if(!(Number.isFinite(a.y)&&a.y>e-n&&a.y<t+n)){i.length>1&&r.push(i.join(` `)),i=[];continue}i.push(`${i.length?`L`:`M`}${p.sx(a.x).toFixed(2)},${p.sy(a.y).toFixed(2)}`)}return i.length>1&&r.push(i.join(` `)),r}),y=l(()=>{if(f.fillTo===void 0||!m.value.length)return null;let[e,t]=p.yDomain.value,n=m.value.filter(e=>Number.isFinite(e.y));if(n.length<2)return null;let r=n=>Math.min(Math.max(n,e),t),i=n.map((e,t)=>`${t?`L`:`M`}${p.sx(e.x).toFixed(2)},${p.sy(r(e.y)).toFixed(2)}`).join(` `),a=p.sy(r(f.fillTo)).toFixed(2);return`${i} L${p.sx(n[n.length-1].x).toFixed(2)},${a} L${p.sx(n[0].x).toFixed(2)},${a} Z`});return(e,t)=>(v(),o(`g`,O,[y.value?(v(),o(`path`,{key:0,d:y.value,fill:f.color??`var(--dl-accent)`,style:g({opacity:.12*f.opacity}),stroke:`none`},null,12,k)):_(`v-if`,!0),(v(!0),o(u,null,i(h.value,(e,t)=>(v(),o(`path`,{key:t,d:e,fill:`none`,stroke:f.color??`var(--dl-accent)`,"stroke-width":f.width,"stroke-dasharray":f.dashed?`6 5`:void 0,style:g({opacity:f.opacity}),"stroke-linecap":`round`,"stroke-linejoin":`round`},null,12,A))),128))]))}}),M={key:0},N=[`fill`],P=[`x1`,`y1`,`x2`,`y2`,`stroke`,`stroke-width`,`stroke-dasharray`,`marker-end`],F=m({__name:`PlotLine`,props:{from:{},to:{},color:{},width:{default:2},dashed:{type:Boolean,default:!1},arrow:{type:Boolean,default:!1},opacity:{default:1}},setup(e){let{$slidev:n,$nav:r,$clicksContext:i,$clicks:s,$page:c,$renderContext:u,$frontmatter:d}=b(),f=e,p=E(),m=`dl-arrow-${t()}`,h=l(()=>({x:p.sx(f.from[0]),y:p.sy(f.from[1])})),y=l(()=>({x:p.sx(f.to[0]),y:p.sy(f.to[1])})),x=l(()=>f.color??`var(--dl-heading)`);return(e,t)=>(v(),o(`g`,{class:`dl-line`,style:g({opacity:f.opacity})},[f.arrow?(v(),o(`defs`,M,[a(`marker`,{id:m,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[a(`path`,{d:`M0,0 L7,3.5 L0,7 Z`,fill:x.value},null,8,N)])])):_(`v-if`,!0),a(`line`,{x1:h.value.x,y1:h.value.y,x2:y.value.x,y2:y.value.y,stroke:x.value,"stroke-width":f.width,"stroke-dasharray":f.dashed?`6 5`:void 0,"marker-end":f.arrow?`url(#${m})`:void 0,"stroke-linecap":`round`},null,8,P)],4))}}),I={key:1,class:`dl-gflow__toggle`},L=5,R=-12,z=4,B=y(m({__name:`GradientFlow`,props:{mode:{default:`decay`},steps:{default:25},w:{default:.6},forget:{default:.99},clip:{type:Boolean,default:!0}},setup(t){let{$slidev:i,$nav:m,$clicksContext:g,$clicks:y,$page:E,$renderContext:O,$frontmatter:k}=b(),A=t,M=d(A.w),N=d(A.forget),P=d(!1),B=Math.log10(L);function V(e,t){return x(e,A.steps).map((e,n)=>{let r=e===0?-13:Math.log10(e);return{x:n,y:t&&r>B?B:r}})}let H=l(()=>V(M.value,P.value&&A.clip)),U=l(()=>V(N.value,!1)),W=l(()=>Math.abs(M.value-1)<1e-9?`hold`:M.value<1?`vanish`:`explode`),G=l(()=>Math.abs(M.value)**Math.min(20,A.steps)),K=l(()=>Math.abs(N.value)**Math.min(20,A.steps)),q=l(()=>S(M.value,.01)),J={0:`⁰`,1:`¹`,2:`²`,3:`³`,4:`⁴`,5:`⁵`,6:`⁶`,7:`⁷`,8:`⁸`,9:`⁹`,"-":`⁻`,"+":``};function Y(e){if(e===0)return`0`;let t=Math.abs(e);if(t>=.001&&t<1)return String(Number(e.toPrecision(2)));if(t>=1&&t<1e4)return String(Number(e.toPrecision(3)));let[n,r]=e.toExponential(1).split(`e`);return`${n} \u00d7 10${[...r].map(e=>J[e]??e).join(``)}`}return(t,i)=>{let l=F,d=j,m=D,g=T,y=w,b=C;return v(),f(b,{"max-width":`34rem`},{controls:e(()=>[n(y,{modelValue:M.value,"onUpdate:modelValue":i[0]||=e=>M.value=e,label:`recurrent weight W`,min:.2,max:1.6,step:.05,precision:2},null,8,[`modelValue`]),A.mode===`gated`?(v(),f(y,{key:0,modelValue:N.value,"onUpdate:modelValue":i[1]||=e=>N.value=e,label:`forget gate f`,min:.8,max:1,step:.005,precision:3},null,8,[`modelValue`])):_(`v-if`,!0),A.clip&&A.mode===`decay`?(v(),o(`label`,I,[c(a(`input`,{"onUpdate:modelValue":i[2]||=e=>P.value=e,type:`checkbox`},null,512),[[s,P.value]]),i[3]||=h(` gradient clipping `,-1)])):_(`v-if`,!0)]),readout:e(()=>[A.mode===`gated`?(v(),o(u,{key:0},[i[4]||=h(` Twenty steps back, the plain recurrence has multiplied by `,-1),a(`strong`,null,r(Y(G.value)),1),i[5]||=h(`; the cell state, by `,-1),a(`strong`,null,r(Y(K.value)),1),i[6]||=h(`. Push `,-1),i[7]||=a(`em`,null,`f`,-1),i[8]||=h(` to 1.000 and the factor is exactly `,-1),i[9]||=a(`strong`,null,`1`,-1),i[10]||=h(` — the gradient reaches the start of the sequence undamaged. `,-1)],64)):W.value===`vanish`?(v(),o(u,{key:1},[i[11]||=a(`strong`,null,`Vanishing.`,-1),h(` Twenty steps back the gradient carries a factor of `+r(M.value.toFixed(2)),1),i[12]||=a(`sup`,null,`20`,-1),i[13]||=h(` = `,-1),a(`strong`,null,r(Y(G.value)),1),i[14]||=h(`. It is still worth 1% of its original size about `,-1),a(`strong`,null,r(q.value),1),i[15]||=h(` steps back — past that, the end of the sentence cannot teach the weights that read its beginning. `,-1)],64)):W.value===`hold`?(v(),o(u,{key:2},[i[16]||=a(`strong`,null,`The knife edge.`,-1),i[17]||=h(` At exactly 1 the gradient neither shrinks nor grows. Nothing keeps a trained weight there, and this is one number out of a whole matrix. `,-1)],64)):(v(),o(u,{key:3},[i[21]||=a(`strong`,null,`Exploding.`,-1),h(` `+r(M.value.toFixed(2)),1),i[22]||=a(`sup`,null,`20`,-1),i[23]||=h(` = `,-1),a(`strong`,null,r(Y(G.value)),1),h(`, and by step `+r(A.steps)+` it is `+r(Y(Math.abs(M.value)**A.steps))+`. One long sequence and the update overshoots — the loss goes to NaN in a single batch. `,1),A.clip?(v(),o(u,{key:0},[i[18]||=h(` Tick `,-1),i[19]||=a(`strong`,null,`gradient clipping`,-1),i[20]||=h(`: the ceiling caps it. Notice it does nothing at all to the vanishing end. `,-1)],64)):_(`v-if`,!0)],64))]),default:e(()=>[n(g,{"x-domain":[0,A.steps],"y-domain":[R,z],"x-ticks":[0,5,10,15,20,25].filter(e=>e<=A.steps),"y-ticks":[-12,-9,-6,-3,0,3],"x-label":`time steps back through the sequence`,"y-label":`log₁₀ of the gradient's size`,width:520,height:290},{default:e(()=>[_(` Unity: the gradient neither grows nor shrinks. `),n(l,{from:[0,0],to:[A.steps,0],color:`var(--dl-muted)`,width:1.2,dashed:``},null,8,[`to`]),P.value&&A.clip&&A.mode===`decay`?(v(),f(l,{key:0,from:[0,p(B)],to:[A.steps,p(B)],color:`var(--dl-danger)`,width:1.2,dashed:``},null,8,[`from`,`to`])):_(`v-if`,!0),n(d,{points:H.value,color:A.mode===`gated`?`var(--dl-danger)`:`var(--dl-accent)`,width:2.6},null,8,[`points`,`color`]),A.mode===`gated`?(v(),f(d,{key:1,points:U.value,color:`var(--dl-accent)`,width:2.6},null,8,[`points`])):_(`v-if`,!0),A.mode===`gated`?(v(),f(m,{key:2,at:[A.steps,U.value[A.steps].y],dx:-6,dy:-10,anchor:`end`,color:`var(--dl-accent)`,text:`LSTM cell state — multiplied by the forget gate`,size:11,bold:``},null,8,[`at`])):_(`v-if`,!0),A.mode===`gated`?(v(),f(m,{key:3,at:[A.steps,Math.min(Math.max(H.value[A.steps].y,-11),3)],dx:-6,dy:22,anchor:`end`,color:`var(--dl-danger)`,text:`plain RNN — multiplied by W`,size:11,bold:``},null,8,[`at`])):_(`v-if`,!0),P.value&&A.clip&&A.mode===`decay`?(v(),f(m,{key:4,at:[1,p(B)],dx:4,dy:-6,color:`var(--dl-danger)`,text:`clipped at ${L}×`,size:11},null,8,[`at`,`text`])):_(`v-if`,!0)]),_:1},8,[`x-domain`,`y-domain`,`x-ticks`])]),_:1})}}}),[[`__scopeId`,`data-v-d7ea900e`]]);export{B as t}; |