vlbthambawita commited on
Commit
c1a36a7
·
verified ·
1 Parent(s): d3896c0

Release lecture-08-v1

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. lecture-08/404.html +30 -0
  2. lecture-08/_redirects +1 -0
  3. lecture-08/assets/404-bkedjYjP.css +1 -0
  4. lecture-08/assets/Citation-CcKhBxpe.js +1 -0
  5. lecture-08/assets/Citation-rca7ok54.css +1 -0
  6. lecture-08/assets/DeckFooter-B6vKMo6W.css +1 -0
  7. lecture-08/assets/DeckFooter-BmHxwiXu.js +1 -0
  8. lecture-08/assets/DqnFlow-B46GQSY5.css +1 -0
  9. lecture-08/assets/DqnFlow-DlxgPWo3.js +1 -0
  10. lecture-08/assets/DrawingControls-Dhbx0szf.css +1 -0
  11. lecture-08/assets/GridWorld-CYBTngYz.css +1 -0
  12. lecture-08/assets/GridWorld-D5i5gg4u.js +1 -0
  13. lecture-08/assets/KaTeX_AMS-Regular-BQhdFMY1.woff2 +0 -0
  14. lecture-08/assets/KaTeX_AMS-Regular-DMm9YOAa.woff +0 -0
  15. lecture-08/assets/KaTeX_AMS-Regular-DRggAlZN.ttf +0 -0
  16. lecture-08/assets/KaTeX_Caligraphic-Bold-ATXxdsX0.ttf +0 -0
  17. lecture-08/assets/KaTeX_Caligraphic-Bold-BEiXGLvX.woff +0 -0
  18. lecture-08/assets/KaTeX_Caligraphic-Bold-Dq_IR9rO.woff2 +0 -0
  19. lecture-08/assets/KaTeX_Caligraphic-Regular-CTRA-rTL.woff +0 -0
  20. lecture-08/assets/KaTeX_Caligraphic-Regular-Di6jR-x-.woff2 +0 -0
  21. lecture-08/assets/KaTeX_Caligraphic-Regular-wX97UBjC.ttf +0 -0
  22. lecture-08/assets/KaTeX_Fraktur-Bold-BdnERNNW.ttf +0 -0
  23. lecture-08/assets/KaTeX_Fraktur-Bold-BsDP51OF.woff +0 -0
  24. lecture-08/assets/KaTeX_Fraktur-Bold-CL6g_b3V.woff2 +0 -0
  25. lecture-08/assets/KaTeX_Fraktur-Regular-CB_wures.ttf +0 -0
  26. lecture-08/assets/KaTeX_Fraktur-Regular-CTYiF6lA.woff2 +0 -0
  27. lecture-08/assets/KaTeX_Fraktur-Regular-Dxdc4cR9.woff +0 -0
  28. lecture-08/assets/KaTeX_Main-Bold-Cx986IdX.woff2 +0 -0
  29. lecture-08/assets/KaTeX_Main-Bold-Jm3AIy58.woff +0 -0
  30. lecture-08/assets/KaTeX_Main-Bold-waoOVXN0.ttf +0 -0
  31. lecture-08/assets/KaTeX_Main-BoldItalic-DxDJ3AOS.woff2 +0 -0
  32. lecture-08/assets/KaTeX_Main-BoldItalic-DzxPMmG6.ttf +0 -0
  33. lecture-08/assets/KaTeX_Main-BoldItalic-SpSLRI95.woff +0 -0
  34. lecture-08/assets/KaTeX_Main-Italic-3WenGoN9.ttf +0 -0
  35. lecture-08/assets/KaTeX_Main-Italic-BMLOBm91.woff +0 -0
  36. lecture-08/assets/KaTeX_Main-Italic-NWA7e6Wa.woff2 +0 -0
  37. lecture-08/assets/KaTeX_Main-Regular-B22Nviop.woff2 +0 -0
  38. lecture-08/assets/KaTeX_Main-Regular-Dr94JaBh.woff +0 -0
  39. lecture-08/assets/KaTeX_Main-Regular-ypZvNtVU.ttf +0 -0
  40. lecture-08/assets/KaTeX_Math-BoldItalic-B3XSjfu4.ttf +0 -0
  41. lecture-08/assets/KaTeX_Math-BoldItalic-CZnvNsCZ.woff2 +0 -0
  42. lecture-08/assets/KaTeX_Math-BoldItalic-iY-2wyZ7.woff +0 -0
  43. lecture-08/assets/KaTeX_Math-Italic-DA0__PXp.woff +0 -0
  44. lecture-08/assets/KaTeX_Math-Italic-flOr_0UB.ttf +0 -0
  45. lecture-08/assets/KaTeX_Math-Italic-t53AETM-.woff2 +0 -0
  46. lecture-08/assets/KaTeX_SansSerif-Bold-CFMepnvq.ttf +0 -0
  47. lecture-08/assets/KaTeX_SansSerif-Bold-D1sUS0GD.woff2 +0 -0
  48. lecture-08/assets/KaTeX_SansSerif-Bold-DbIhKOiC.woff +0 -0
  49. lecture-08/assets/KaTeX_SansSerif-Italic-C3H0VqGB.woff2 +0 -0
  50. lecture-08/assets/KaTeX_SansSerif-Italic-DN2j7dab.woff +0 -0
lecture-08/404.html ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>Deep Reinforcement Learning - Slidev</title>
7
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Inter:wght@200;400;600&family=Source+Serif+4:wght@200;400;600&family=JetBrains+Mono:wght@200;400;600&display=swap" type="text/css">
8
+ <link rel="icon" href="https://cdn.jsdelivr.net/gh/slidevjs/slidev/assets/favicon.png">
9
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
10
+ <meta property="slidev:version" content="52.19.0">
11
+ <meta name="description" content="&quot;PGR207 Deep Learning 2026 — Lecture 08&quot;">
12
+ <meta name="author" content="&quot;Vajira Thambawita&quot;">
13
+ <meta property="og:title" content="Deep Reinforcement Learning - Slidev">
14
+ <meta property="og:description" content="&quot;PGR207 Deep Learning 2026 — Lecture 08&quot;"> <script type="module" crossorigin src="./assets/index-CvCioxNn.js"></script>
15
+ <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-hePW80VL.js">
16
+ <link rel="modulepreload" crossorigin href="./assets/modules/shiki-CJZsCeMT.js">
17
+ <link rel="modulepreload" crossorigin href="./assets/_plugin-vue_export-helper-BDNMzG2s.js">
18
+ <link rel="modulepreload" crossorigin href="./assets/modules/vue-Dnfgf7Ad.js">
19
+ <link rel="modulepreload" crossorigin href="./assets/slidev/context-DB-9afGB.js">
20
+ <link rel="modulepreload" crossorigin href="./assets/useNav-B0kQsolV.js">
21
+ <link rel="modulepreload" crossorigin href="./assets/syncState-3Tc2ZArb.js">
22
+ <link rel="stylesheet" crossorigin href="./assets/modules/shiki-BrmBd2_v.css">
23
+ <link rel="stylesheet" crossorigin href="./assets/index-CSMQn_X0.css">
24
+ </head>
25
+ <body>
26
+ <div id="app"></div>
27
+ <div id="mermaid-rendering-container"></div>
28
+
29
+ </body>
30
+ </html>
lecture-08/_redirects ADDED
@@ -0,0 +1 @@
 
 
1
+ ./* ./index.html 200
lecture-08/assets/404-bkedjYjP.css ADDED
@@ -0,0 +1 @@
 
 
1
+ .page-link[data-v-b584852f]{background-color:#9ca3af1a;border-radius:.25rem;padding:.5rem 1rem}.page-link[data-v-b584852f]:hover{background-color:#9ca3af33}
lecture-08/assets/Citation-CcKhBxpe.js ADDED
@@ -0,0 +1 @@
 
 
1
+ import{B as e,Ct as t,S as n,T as r,U as i,b as a,m as o}from"./modules/shiki-CJZsCeMT.js";import{t as s}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as c}from"./slidev/context-DB-9afGB.js";var l={class:`dl-citation`},u=[`href`],d=s(r({__name:`Citation`,props:{source:{},url:{}},setup(r){let{$slidev:s,$nav:d,$clicksContext:f,$clicks:p,$page:m,$renderContext:h,$frontmatter:g}=c(),_=r;return(r,s)=>(e(),a(`span`,l,[_.url?(e(),a(`a`,{key:0,href:_.url,target:`_blank`,rel:`noopener`},t(_.source),9,u)):(e(),a(o,{key:1},[n(t(_.source),1)],64)),i(r.$slots,`default`,{},void 0,!0)]))}}),[[`__scopeId`,`data-v-11a8cabc`]]);export{d as t};
lecture-08/assets/Citation-rca7ok54.css ADDED
@@ -0,0 +1 @@
 
 
1
+ .dl-citation[data-v-11a8cabc]{font-size:var(--dl-fs-citation);color:var(--dl-muted)}.dl-citation a[data-v-11a8cabc]{color:inherit;border-bottom:1px dotted}
lecture-08/assets/DeckFooter-B6vKMo6W.css ADDED
@@ -0,0 +1 @@
 
 
1
+ .dl-footer[data-v-fe908050]{letter-spacing:.04em;color:var(--dl-muted);pointer-events:none;justify-content:space-between;align-items:center;gap:1rem;font-size:.72rem;display:flex;position:absolute;bottom:1rem;left:3rem;right:3rem}.dl-footer__left[data-v-fe908050]{align-items:center;gap:.4rem;min-width:0;display:flex}.dl-footer__code[data-v-fe908050]{color:var(--dl-accent);font-weight:700}.dl-footer__sep[data-v-fe908050]{opacity:.5}.dl-footer__title[data-v-fe908050]{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.dl-footer__right[data-v-fe908050]{font-variant-numeric:tabular-nums;flex-shrink:0}
lecture-08/assets/DeckFooter-BmHxwiXu.js ADDED
@@ -0,0 +1 @@
 
 
1
+ import{B as e,Ct as t,T as n,_ as r,b as i,g as a,y as o,yt as s}from"./modules/shiki-CJZsCeMT.js";import{t as c}from"./useNav-B0kQsolV.js";import{t as l}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as u}from"./slidev/context-DB-9afGB.js";import"./slidev/client-beF_Q0xG.js";var d={class:`dl-footer`},f={class:`dl-footer__left`},p={key:0,class:`dl-footer__code`},m={key:1,class:`dl-footer__sep`},h={key:2},g={key:3,class:`dl-footer__sep`},_={key:4,class:`dl-footer__title`},v={class:`dl-footer__right`},y=l(n({__name:`DeckFooter`,setup(n){let{$slidev:l}=u(),y=c(),b=a(()=>l?.themeConfigs??{}),x=a(()=>b.value.courseCode??``),S=a(()=>b.value.lecture??``),C=a(()=>b.value.lectureTitle??``);return(n,a)=>(e(),i(`footer`,d,[r(`div`,f,[x.value?(e(),i(`span`,p,t(x.value),1)):o(`v-if`,!0),S.value?(e(),i(`span`,m,`·`)):o(`v-if`,!0),S.value?(e(),i(`span`,h,`Lecture `+t(S.value),1)):o(`v-if`,!0),C.value?(e(),i(`span`,g,`·`)):o(`v-if`,!0),C.value?(e(),i(`span`,_,t(C.value),1)):o(`v-if`,!0)]),r(`div`,v,t(s(y).currentPage.value)+` / `+t(s(y).total.value),1)]))}}),[[`__scopeId`,`data-v-fe908050`]]);export{y as t};
lecture-08/assets/DqnFlow-B46GQSY5.css ADDED
@@ -0,0 +1 @@
 
 
1
+ .dl-dqn[data-v-10cdf00f]{width:100%;max-width:42rem;margin:0}.dl-dqn svg[data-v-10cdf00f]{width:100%;height:auto}.dl-dqn g[data-v-10cdf00f]{transition:opacity .25s}.dl-dqn__env[data-v-10cdf00f]{fill:color-mix(in srgb, var(--dl-enc) 14%, var(--dl-bg));stroke:var(--dl-enc);stroke-width:1.8px}.dl-dqn__box[data-v-10cdf00f]{fill:var(--dl-surface);stroke:var(--dl-border);stroke-width:1.6px}.dl-dqn__box.is-accent[data-v-10cdf00f]{stroke:var(--dl-accent)}.dl-dqn__row[data-v-10cdf00f]{fill:var(--dl-bg);stroke:var(--dl-border)}.dl-dqn__row.is-pick[data-v-10cdf00f]{fill:var(--dl-accent-soft);stroke:var(--dl-accent)}.dl-dqn__net[data-v-10cdf00f]{fill:var(--dl-accent-soft);stroke:var(--dl-accent);stroke-width:1.8px}.dl-dqn__net.is-target[data-v-10cdf00f]{fill:color-mix(in srgb, var(--dl-q) 12%, var(--dl-bg));stroke:var(--dl-q)}.dl-dqn__loss[data-v-10cdf00f]{fill:var(--dl-bg);stroke:var(--dl-heading);stroke-width:1.8px}.dl-dqn__lab[data-v-10cdf00f]{fill:var(--dl-heading);font-size:13px;font-weight:600}.dl-dqn__in[data-v-10cdf00f]{fill:var(--dl-accent-strong);font-size:13px;font-weight:700}.dl-dqn__in.is-target[data-v-10cdf00f]{fill:var(--dl-q)}.dl-dqn__small[data-v-10cdf00f]{fill:var(--dl-muted);font-size:11px}.dl-dqn__tiny[data-v-10cdf00f]{fill:var(--dl-muted);font-size:9px}.dl-dqn__small.is-current[data-v-10cdf00f]{fill:var(--dl-k);font-weight:600}.dl-dqn__small.is-target[data-v-10cdf00f]{fill:var(--dl-q);font-weight:600}.dl-dqn__small.is-bad[data-v-10cdf00f]{fill:var(--dl-danger);font-weight:600}.dl-dqn__arrow[data-v-10cdf00f]{fill:none;stroke:var(--dl-body);stroke-width:1.8px}.dl-dqn__arrow.is-current[data-v-10cdf00f]{stroke:var(--dl-k);stroke-width:2.4px}.dl-dqn__arrow.is-target[data-v-10cdf00f]{stroke:var(--dl-q);stroke-width:2.4px}.dl-dqn__grad[data-v-10cdf00f]{fill:none;stroke:var(--dl-danger);stroke-width:2px;stroke-dasharray:5 3}.dl-dqn__copy[data-v-10cdf00f]{fill:none;stroke:var(--dl-body);stroke-width:1.8px;stroke-dasharray:6 4}.dl-dqn__head[data-v-10cdf00f]{fill:var(--dl-body)}.dl-dqn__head.is-current[data-v-10cdf00f]{fill:var(--dl-k)}.dl-dqn__head.is-target[data-v-10cdf00f]{fill:var(--dl-q)}.dl-dqn__head.is-bad[data-v-10cdf00f]{fill:var(--dl-danger)}.dl-dqn__cap[data-v-10cdf00f]{color:var(--dl-body);min-height:1.3rem;margin-top:.3rem;font-size:.85rem}
lecture-08/assets/DqnFlow-DlxgPWo3.js ADDED
@@ -0,0 +1 @@
 
 
1
+ import{B as e,Ct as t,H as n,St as r,T as i,_ as a,b as o,bt as s,g as c,m as l,q as u,y as d,yt as f}from"./modules/shiki-CJZsCeMT.js";import{t as p}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as m}from"./slidev/context-DB-9afGB.js";var h={class:`dl-dqn`},g={viewBox:`0 0 680 318`,role:`img`,"aria-label":`The deep Q-network training loop: environment, replay buffer, batch, Q-network, target network and loss`},_=[`id`],v=[`id`],y=[`id`],b=[`id`],x=[`y`],S=[`marker-end`],C=[`marker-end`],w=[`marker-end`],T=[`marker-end`],E=[`marker-end`],D=[`marker-end`],O=[`marker-end`],k=[`marker-end`],A=[`marker-end`],j={class:`dl-dqn__cap`},M=p(i({__name:`DqnFlow`,props:{step:{default:0}},setup(i){let{$slidev:p,$nav:M,$clicksContext:N,$clicks:P,$page:F,$renderContext:I,$frontmatter:L}=m(),R=i,z=u(),B=[`The whole loop. Acting and learning run side by side.`,`1 · The Q-network picks an action (ε-greedy). The environment returns a reward and the next state.`,`2 · The move (s, a, r, s′) is stored in the replay buffer.`,`3 · A random batch of stored moves is drawn — not the latest ones.`,`4 · The Q-network predicts Q(s, a) for each move in the batch.`,`5 · The target network gives max Q(s′, ·). Target = r + γ · max.`,`6 · Loss = mean of (target − predicted)² over the batch.`,`7 · Backpropagation updates the Q-network. The target network does not change.`,`8 · Every T steps, copy the Q-network's weights into the target network.`],V=(...e)=>({opacity:R.step===0||e.includes(R.step)?1:.18}),H=c(()=>B[Math.min(8,Math.max(0,R.step))]);return(i,c)=>(e(),o(`figure`,h,[(e(),o(`svg`,g,[a(`defs`,null,[a(`marker`,{id:`dq-${f(z)}`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[...c[0]||=[a(`path`,{d:`M0 0 L7 3.5 L0 7 z`,class:`dl-dqn__head`},null,-1)]],8,_),a(`marker`,{id:`dqc-${f(z)}`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[...c[1]||=[a(`path`,{d:`M0 0 L7 3.5 L0 7 z`,class:`dl-dqn__head is-current`},null,-1)]],8,v),a(`marker`,{id:`dqt-${f(z)}`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[...c[2]||=[a(`path`,{d:`M0 0 L7 3.5 L0 7 z`,class:`dl-dqn__head is-target`},null,-1)]],8,y),a(`marker`,{id:`dqb-${f(z)}`,markerWidth:`7`,markerHeight:`7`,refX:`6`,refY:`3.5`,orient:`auto`},[...c[3]||=[a(`path`,{d:`M0 0 L7 3.5 L0 7 z`,class:`dl-dqn__head is-bad`},null,-1)]],8,b)]),d(` environment `),a(`g`,{style:r(V(1,2))},[...c[4]||=[a(`rect`,{x:`14`,y:`20`,width:`132`,height:`60`,rx:`10`,class:`dl-dqn__env`},null,-1),a(`text`,{x:`80`,y:`46`,"text-anchor":`middle`,class:`dl-dqn__lab`},`environment`,-1),a(`text`,{x:`80`,y:`64`,"text-anchor":`middle`,class:`dl-dqn__small`},`the robot's floor`,-1)]],4),d(` replay buffer `),a(`g`,{style:r(V(2,3))},[c[5]||=a(`rect`,{x:`236`,y:`14`,width:`168`,height:`96`,rx:`6`,class:`dl-dqn__box`},null,-1),c[6]||=a(`text`,{x:`320`,y:`31`,"text-anchor":`middle`,class:`dl-dqn__lab`},`replay buffer`,-1),(e(),o(l,null,n(4,e=>a(`rect`,{key:e,x:`248`,y:38+(e-1)*17,width:`144`,height:`13`,rx:`2`,class:s([`dl-dqn__row`,{"is-pick":e===2||e===4}])},null,10,x)),64)),c[7]||=a(`text`,{x:`320`,y:`48`,"text-anchor":`middle`,class:`dl-dqn__tiny`},`(s, a, r, s′)`,-1)],4),d(` batch `),a(`g`,{style:r(V(3,4,5))},[...c[8]||=[a(`rect`,{x:`490`,y:`30`,width:`150`,height:`56`,rx:`6`,class:`dl-dqn__box is-accent`},null,-1),a(`text`,{x:`565`,y:`48`,"text-anchor":`middle`,class:`dl-dqn__lab`},`random batch`,-1),a(`rect`,{x:`502`,y:`56`,width:`126`,height:`11`,rx:`2`,class:`dl-dqn__row is-pick`},null,-1),a(`rect`,{x:`502`,y:`70`,width:`126`,height:`11`,rx:`2`,class:`dl-dqn__row is-pick`},null,-1)]],4),d(` Q-network `),a(`g`,{style:r(V(1,4,7,8))},[...c[9]||=[a(`polygon`,{points:`190,168 320,184 320,240 190,256`,class:`dl-dqn__net`},null,-1),a(`text`,{x:`255`,y:`210`,"text-anchor":`middle`,class:`dl-dqn__in`},`Q-network`,-1),a(`text`,{x:`255`,y:`226`,"text-anchor":`middle`,class:`dl-dqn__small`},`weights w`,-1)]],4),d(` target network `),a(`g`,{style:r(V(5,8))},[...c[10]||=[a(`polygon`,{points:`470,168 600,184 600,240 470,256`,class:`dl-dqn__net is-target`},null,-1),a(`text`,{x:`535`,y:`210`,"text-anchor":`middle`,class:`dl-dqn__in is-target`},`target network`,-1),a(`text`,{x:`535`,y:`226`,"text-anchor":`middle`,class:`dl-dqn__small`},`weights w⁻ (frozen)`,-1)]],4),d(` loss `),a(`g`,{style:r(V(6,7))},[...c[11]||=[a(`circle`,{cx:`395`,cy:`286`,r:`22`,class:`dl-dqn__loss`},null,-1),a(`text`,{x:`395`,y:`291`,"text-anchor":`middle`,class:`dl-dqn__lab`},`MSE`,-1)]],4),d(` 1: act `),a(`g`,{style:r(V(1))},[a(`path`,{d:`M190 200 C110 200, 80 150, 80 88`,class:`dl-dqn__arrow is-current`,"marker-end":`url(#dqc-${f(z)})`},null,8,S),c[12]||=a(`text`,{x:`40`,y:`150`,class:`dl-dqn__small is-current`},`action a`,-1),c[13]||=a(`text`,{x:`40`,y:`164`,class:`dl-dqn__small`},`(ε-greedy)`,-1)],4),d(` 2: store `),a(`g`,{style:r(V(2))},[a(`path`,{d:`M148 50 H230`,class:`dl-dqn__arrow`,"marker-end":`url(#dq-${f(z)})`},null,8,C),c[14]||=a(`text`,{x:`189`,y:`42`,"text-anchor":`middle`,class:`dl-dqn__small`},`store`,-1)],4),d(` 3: sample `),a(`g`,{style:r(V(3))},[a(`path`,{d:`M406 60 H484`,class:`dl-dqn__arrow`,"marker-end":`url(#dq-${f(z)})`},null,8,w),c[15]||=a(`text`,{x:`445`,y:`52`,"text-anchor":`middle`,class:`dl-dqn__small`},`sample`,-1)],4),d(` 4: batch s, a → Q-net `),a(`g`,{style:r(V(4))},[a(`path`,{d:`M520 88 C470 130, 330 120, 290 166`,class:`dl-dqn__arrow`,"marker-end":`url(#dq-${f(z)})`},null,8,T),c[16]||=a(`text`,{x:`402`,y:`134`,"text-anchor":`middle`,class:`dl-dqn__small`},`s, a`,-1),a(`path`,{d:`M322 230 C350 250, 362 266, 372 278`,class:`dl-dqn__arrow is-current`,"marker-end":`url(#dqc-${f(z)})`},null,8,E),c[17]||=a(`text`,{x:`366`,y:`306`,"text-anchor":`end`,class:`dl-dqn__small is-current`},`predicted Q(s, a)`,-1)],4),d(` 5: batch s′, r → target `),a(`g`,{style:r(V(5))},[a(`path`,{d:`M590 88 V164`,class:`dl-dqn__arrow`,"marker-end":`url(#dq-${f(z)})`},null,8,D),c[18]||=a(`text`,{x:`598`,y:`130`,class:`dl-dqn__small`},`s′, r`,-1),a(`path`,{d:`M470 236 C450 258, 432 274, 418 280`,class:`dl-dqn__arrow is-target`,"marker-end":`url(#dqt-${f(z)})`},null,8,O),c[19]||=a(`text`,{x:`442`,y:`296`,class:`dl-dqn__small is-target`},`target r + γ max Q(s′, ·)`,-1)],4),d(` 7: backprop `),a(`g`,{style:r(V(7))},[a(`path`,{d:`M374 296 C300 312, 230 300, 224 262`,class:`dl-dqn__grad`,"marker-end":`url(#dqb-${f(z)})`},null,8,k),c[20]||=a(`text`,{x:`150`,y:`300`,class:`dl-dqn__small is-bad`},`backprop → w`,-1)],4),d(` 8: copy `),a(`g`,{style:r(V(8))},[a(`path`,{d:`M322 196 H464`,class:`dl-dqn__copy`,"marker-end":`url(#dq-${f(z)})`},null,8,A),c[21]||=a(`text`,{x:`393`,y:`190`,"text-anchor":`middle`,class:`dl-dqn__small`},`copy w → w⁻ every T steps`,-1)],4)])),a(`figcaption`,j,t(H.value),1)]))}}),[[`__scopeId`,`data-v-10cdf00f`]]);export{M as t};
lecture-08/assets/DrawingControls-Dhbx0szf.css ADDED
@@ -0,0 +1 @@
 
 
1
+ .v-popper--theme-dropdown .v-popper__arrow-inner{border-color:#9ca3af33}
lecture-08/assets/GridWorld-CYBTngYz.css ADDED
@@ -0,0 +1 @@
 
 
1
+ .dl-grid[data-v-9e69cef1]{max-width:100%;height:auto;font-family:inherit}.dl-grid__cell[data-v-9e69cef1]{fill:var(--dl-surface);stroke:var(--dl-border);stroke-width:1.5px}.dl-grid__cell.is-charger[data-v-9e69cef1]{fill:color-mix(in srgb, var(--dl-enc) 16%, var(--dl-bg));stroke:var(--dl-enc)}.dl-grid__cell.is-hole[data-v-9e69cef1]{fill:color-mix(in srgb, var(--dl-danger) 12%, var(--dl-bg));stroke:var(--dl-danger)}.dl-grid__cell.is-start[data-v-9e69cef1]{fill:var(--dl-accent-soft)}.dl-grid__cell.is-hl[data-v-9e69cef1]{stroke:var(--dl-accent);stroke-width:3px}.dl-grid__battery[data-v-9e69cef1]{fill:var(--dl-bg);stroke:var(--dl-enc);stroke-width:2.4px}.dl-grid__battery-cap[data-v-9e69cef1],.dl-grid__bolt[data-v-9e69cef1]{fill:var(--dl-enc)}.dl-grid__hole[data-v-9e69cef1]{fill:var(--dl-body);stroke:var(--dl-danger);stroke-width:2px}.dl-grid__hole-in[data-v-9e69cef1]{fill:var(--dl-bg);fill-opacity:.15}.dl-grid__reward[data-v-9e69cef1]{font-size:12px;font-weight:700}.dl-grid__reward.is-good[data-v-9e69cef1]{fill:var(--dl-enc)}.dl-grid__reward.is-bad[data-v-9e69cef1]{fill:var(--dl-danger)}.dl-grid__coord[data-v-9e69cef1]{fill:var(--dl-muted);font-size:9.5px}.dl-grid__value[data-v-9e69cef1]{fill:var(--dl-heading);font-variant-numeric:tabular-nums;font-weight:700}.dl-grid__q[data-v-9e69cef1]{fill:var(--dl-muted);font-variant-numeric:tabular-nums;font-weight:600}.dl-grid__q.is-pos[data-v-9e69cef1]{fill:var(--dl-accent-strong)}.dl-grid__q.is-neg[data-v-9e69cef1]{fill:var(--dl-danger)}.dl-grid__diag[data-v-9e69cef1]{stroke:var(--dl-border);stroke-width:1px}.dl-grid__mark[data-v-9e69cef1]{fill:var(--dl-bg);stroke-width:2.2px}.dl-grid__mark.is-current[data-v-9e69cef1]{stroke:var(--dl-k)}.dl-grid__mark.is-target[data-v-9e69cef1]{stroke:var(--dl-q)}.dl-grid__arrow[data-v-9e69cef1]{stroke:var(--dl-accent);stroke-width:3px;fill:none;stroke-linecap:round}.dl-grid__arrowhead[data-v-9e69cef1]{fill:var(--dl-accent)}.dl-grid__path[data-v-9e69cef1]{fill:none;stroke:var(--dl-k);stroke-width:3px;stroke-linejoin:round;stroke-linecap:round;stroke-dasharray:7 4}.dl-grid__pathhead[data-v-9e69cef1]{fill:var(--dl-k)}.dl-grid__slip[data-v-9e69cef1]{stroke:var(--dl-k);stroke-width:1.6px;stroke-dasharray:3 3}.dl-grid__slip.is-main[data-v-9e69cef1]{stroke-width:3px;stroke-dasharray:none}.dl-grid__sliplab[data-v-9e69cef1]{fill:var(--dl-k);font-size:11px;font-weight:700}.dl-grid__robot-body[data-v-9e69cef1]{fill:var(--dl-accent)}.dl-grid__robot-line[data-v-9e69cef1]{stroke:var(--dl-accent);stroke-width:2px}.dl-grid__robot-eye[data-v-9e69cef1]{fill:var(--dl-bg)}.dl-grid__robot-wheels[data-v-9e69cef1]{fill:var(--dl-body)}
lecture-08/assets/GridWorld-D5i5gg4u.js ADDED
@@ -0,0 +1 @@
 
 
1
+ import{B as e,Ct as t,H as n,St as r,T as i,_ as a,b as o,bt as s,g as c,m as l,q as u,x as d,y as f,yt as p}from"./modules/shiki-CJZsCeMT.js";import{t as m}from"./_plugin-vue_export-helper-BDNMzG2s.js";import{n as h}from"./slidev/context-DB-9afGB.js";import{a as g,h as ee,l as _}from"./useReinforcement-11RE6Te1.js";var v=[`viewBox`,`width`,`height`,`aria-label`],y=[`id`],b=[`id`],x=[`x`,`y`,`width`,`height`],te=[`x`,`y`],ne=[`transform`],re=[`x`,`y`],ie=[`transform`],ae=[`x`,`y`],oe=[`x`,`y`],S=[`x`,`y`],C=[`x1`,`y1`,`x2`,`y2`],w=[`x1`,`y1`,`x2`,`y2`],T=[`x`,`y`,`width`,`height`],se=[`x`,`y`],ce=[`d`,`marker-end`],le={key:5},ue=[`d`,`marker-end`],de=[`d`,`marker-end`],fe=[`d`,`marker-end`],E=[`x`,`y`],D=[`x`,`y`],O=[`x`,`y`],k=[`d`,`marker-end`],A=[`transform`],j=4,M=m(i({__name:`GridWorld`,props:{cell:{default:72},path:{},pathStep:{default:99},values:{default:null},q:{default:null},arrows:{default:null},greedy:{type:Boolean,default:!1},marks:{default:()=>[]},highlight:{default:()=>[]},robot:{},slip:{type:Boolean,default:!1},coords:{type:Boolean,default:!1},precision:{default:2},plain:{type:Boolean,default:!1}},setup(i){let{$slidev:m,$nav:M,$clicksContext:pe,$clicks:me,$page:he,$renderContext:ge,$frontmatter:_e}=h(),N=i,P=u(),F=c(()=>N.cell*3+8),I=e=>j+e%3*N.cell,L=e=>j+Math.floor(e/3)*N.cell,R=e=>I(e)+N.cell/2,z=e=>L(e)+N.cell/2,B=Array.from({length:9},(e,t)=>t),V=c(()=>N.path?ee(N.path.slice(0,Math.max(0,N.pathStep))).cells:null),H=c(()=>N.robot===void 0?V.value?V.value[V.value.length-1]:7:N.robot),ve=c(()=>{let e=H.value??7,t=N.cell/72;return N.q?`translate(${I(e)+N.cell*.2}, ${L(e)+N.cell*.24}) scale(${t*.42})`:N.values?`translate(${R(e)}, ${z(e)}) scale(${t*.72})`:`translate(${R(e)}, ${z(e)-N.cell*.06}) scale(${t*(N.slip?.75:1)})`}),U=c(()=>{let e=V.value;if(!e||e.length<2)return``;let t=e.map(e=>[R(e),z(e)]),[n,r]=t[t.length-1],[i,a]=t[t.length-2],o=Math.hypot(n-i,r-a)||1,s=N.cell*.3;return t[t.length-1]=[n-(n-i)/o*s,r-(r-a)/o*s],t.map((e,t)=>`${t?`L`:`M`}${e[0].toFixed(1)} ${e[1].toFixed(1)}`).join(` `)}),W=c(()=>{let e=[];for(let t of B){if(_(t))continue;let n=[];N.greedy&&N.q?n=g(N.q[t]).length<4?g(N.q[t]):[]:N.arrows&&N.arrows[t]!==null&&N.arrows[t]!==void 0&&(n=[].concat(N.arrows[t]));for(let r of n)e.push({s:t,a:r})}return e}),G=[[0,-1],[0,1],[-1,0],[1,0]],K=c(()=>!!N.values&&W.value.length>0);function ye(e,t){let[n,r]=G[t],i=N.cell*(N.q?.06:K.value?.04:.1),a=N.cell*(N.q||K.value?.2:.34),o=z(e)-(K.value?N.cell*.16:0);return`M${R(e)+n*i} ${o+r*i} L${R(e)+n*a} ${o+r*a}`}let q=e=>e.length*.6*$.value;function J(e,t){let[n,r]=G[t],i=q(X(N.q?.[e][t]??0))/2,a=n?Math.min(N.cell*.34,N.cell/2-i-4):N.cell*.34;return{x:R(e)+n*a,y:z(e)+r*a+4}}let Y=(e,t)=>Math.max(N.cell*.34,q(X(N.q?.[e][t]??0))+8);function X(e){return Math.abs(e)<1e-9?`0`:Number(e.toFixed(N.precision)).toString().replace(`-`,`−`)}function Z(e,t){return N.marks.find(n=>n.s===e&&n.a===t)?.tone}let Q=c(()=>Math.max(10,Math.round(N.cell*.17))),$=c(()=>Math.max(9,Math.round(N.cell*.13)));return(c,u)=>(e(),o(`svg`,{class:`dl-grid`,viewBox:`0 0 ${F.value} ${F.value}`,width:F.value,height:F.value,role:`img`,"aria-label":`A 3 by 3 floor: the robot, its charger top right worth plus ${p(5)}, a hole in the middle worth ${p(-10)}`},[a(`defs`,null,[a(`marker`,{id:`gw-path-${p(P)}`,markerWidth:`7`,markerHeight:`7`,refX:`5`,refY:`3.5`,orient:`auto`},[...u[0]||=[a(`path`,{d:`M0 0 L7 3.5 L0 7 z`,class:`dl-grid__pathhead`},null,-1)]],8,y),a(`marker`,{id:`gw-arrow-${p(P)}`,markerWidth:`6`,markerHeight:`6`,refX:`4`,refY:`3`,orient:`auto`},[...u[1]||=[a(`path`,{d:`M0 0 L6 3 L0 6 z`,class:`dl-grid__arrowhead`},null,-1)]],8,b)]),f(` floor `),(e(!0),o(l,null,n(p(B),n=>(e(),o(`g`,{key:`c${n}`},[a(`rect`,{x:I(n),y:L(n),width:i.cell,height:i.cell,class:s([`dl-grid__cell`,{"is-charger":n===p(2),"is-hole":n===p(4),"is-start":n===p(7),"is-hl":i.highlight.includes(n)}])},null,10,x),i.coords&&!p(_)(n)?(e(),o(`text`,{key:0,x:I(n)+4,y:L(n)+11,class:`dl-grid__coord`},`(`+t(Math.floor(n/p(3))+1)+`,`+t(n%p(3)+1)+`)`,9,te)):f(`v-if`,!0)]))),128)),f(` the charger: a battery with a bolt `),a(`g`,{transform:`translate(${R(p(2))}, ${z(p(2))-(i.plain?0:i.cell*.08)}) scale(${i.cell/72})`},[...u[2]||=[a(`rect`,{x:`-13`,y:`-17`,width:`26`,height:`30`,rx:`4`,class:`dl-grid__battery`},null,-1),a(`rect`,{x:`-5`,y:`-21`,width:`10`,height:`5`,rx:`1.5`,class:`dl-grid__battery-cap`},null,-1),a(`path`,{d:`M2 -12 L-6 0 H0 L-2 9 L7 -4 H1 Z`,class:`dl-grid__bolt`},null,-1)]],8,ne),i.plain?f(`v-if`,!0):(e(),o(`text`,{key:0,x:R(p(2)),y:L(p(2))+i.cell-6,"text-anchor":`middle`,class:`dl-grid__reward is-good`},`+`+t(p(5)),9,re)),f(` the hole `),a(`g`,{transform:`translate(${R(p(4))}, ${z(p(4))-(i.plain?0:i.cell*.08)}) scale(${i.cell/72})`},[...u[3]||=[a(`ellipse`,{cx:`0`,cy:`0`,rx:`22`,ry:`13`,class:`dl-grid__hole`},null,-1),a(`ellipse`,{cx:`0`,cy:`3`,rx:`15`,ry:`7`,class:`dl-grid__hole-in`},null,-1)]],8,ie),i.plain?f(`v-if`,!0):(e(),o(`text`,{key:1,x:R(p(4)),y:L(p(4))+i.cell-6,"text-anchor":`middle`,class:`dl-grid__reward is-bad`},t(p(-10).toString().replace(`-`,`−`)),9,ae)),!i.values&&!i.q?(e(),o(`text`,{key:2,x:R(p(7)),y:L(p(7))+i.cell-5,"text-anchor":`middle`,class:`dl-grid__coord`},`start`,8,oe)):f(`v-if`,!0),f(` state values `),i.values?(e(!0),o(l,{key:3},n(p(B).filter(e=>!p(_)(e)),n=>(e(),o(`text`,{key:`v${n}`,x:R(n),y:K.value?L(n)+i.cell*.8:z(n)+(H.value===n?i.cell*.36:Q.value/3),"text-anchor":`middle`,class:`dl-grid__value`,style:r({fontSize:`${Q.value}px`})},t(X(i.values[n])),13,S))),128)):f(`v-if`,!0),f(` action values `),i.q?(e(!0),o(l,{key:4},n(p(B).filter(e=>!p(_)(e)),c=>(e(),o(`g`,{key:`q${c}`},[a(`line`,{x1:I(c)+3,y1:L(c)+3,x2:I(c)+i.cell-3,y2:L(c)+i.cell-3,class:`dl-grid__diag`},null,8,C),a(`line`,{x1:I(c)+i.cell-3,y1:L(c)+3,x2:I(c)+3,y2:L(c)+i.cell-3,class:`dl-grid__diag`},null,8,w),(e(),o(l,null,n([0,1,2,3],n=>a(`g`,{key:n},[Z(c,n)?(e(),o(`rect`,{key:0,x:J(c,n).x-Y(c,n)/2,y:J(c,n).y-$.value-1,width:Y(c,n),height:$.value+6,rx:`3`,class:s([`dl-grid__mark`,`is-${Z(c,n)}`])},null,10,T)):f(`v-if`,!0),a(`text`,{x:J(c,n).x,y:J(c,n).y,"text-anchor":`middle`,class:s([`dl-grid__q`,{"is-neg":i.q[c][n]<-1e-9,"is-pos":i.q[c][n]>1e-9}]),style:r({fontSize:`${$.value}px`})},t(X(i.q[c][n])),15,se)])),64))]))),128)):f(`v-if`,!0),f(` policy arrows `),(e(!0),o(l,null,n(W.value,(t,n)=>(e(),o(`path`,{key:`a${n}`,d:ye(t.s,t.a),class:`dl-grid__arrow`,"marker-end":`url(#gw-arrow-${p(P)})`},null,8,ce))),128)),f(` the slippery floor `),i.slip?(e(),o(`g`,le,[a(`path`,{d:`M${R(5)} ${z(5)-8} L${R(5)} ${L(5)-i.cell*.28}`,class:`dl-grid__slip is-main`,"marker-end":`url(#gw-path-${p(P)})`},null,8,ue),a(`path`,{d:`M${R(5)-8} ${z(5)} L${I(5)-i.cell*.28} ${z(5)}`,class:`dl-grid__slip`,"marker-end":`url(#gw-path-${p(P)})`},null,8,de),a(`path`,{d:`M${R(5)+8} ${z(5)} L${I(5)+i.cell-4} ${z(5)}`,class:`dl-grid__slip`,"marker-end":`url(#gw-path-${p(P)})`},null,8,fe),a(`text`,{x:R(5)+6,y:L(5)+12,class:`dl-grid__sliplab is-main`},`0.8`,8,E),a(`text`,{x:I(5)+3,y:z(5)-6,class:`dl-grid__sliplab`},`0.1`,8,D),a(`text`,{x:I(5)+i.cell-22,y:z(5)+16,class:`dl-grid__sliplab`},`0.1`,8,O)])):f(`v-if`,!0),f(` the path walked so far `),U.value?(e(),o(`path`,{key:6,d:U.value,class:`dl-grid__path`,"marker-end":`url(#gw-path-${p(P)})`},null,8,k)):f(`v-if`,!0),f(` the robot `),f(` With Q-values the four edges are taken, so the robot sits small in a corner. `),H.value!==null&&H.value!==void 0?(e(),o(`g`,{key:7,transform:ve.value,class:`dl-grid__robot`},[...u[4]||=[d(`<line x1="0" y1="-20" x2="0" y2="-14" class="dl-grid__robot-line" data-v-9e69cef1></line><circle cx="0" cy="-21" r="2.6" class="dl-grid__robot-body" data-v-9e69cef1></circle><rect x="-14" y="-14" width="28" height="22" rx="6" class="dl-grid__robot-body" data-v-9e69cef1></rect><circle cx="-5.5" cy="-4" r="3" class="dl-grid__robot-eye" data-v-9e69cef1></circle><circle cx="5.5" cy="-4" r="3" class="dl-grid__robot-eye" data-v-9e69cef1></circle><rect x="-10" y="9" width="20" height="5" rx="2.5" class="dl-grid__robot-wheels" data-v-9e69cef1></rect>`,6)]],8,A)):f(`v-if`,!0)],8,v))}}),[[`__scopeId`,`data-v-9e69cef1`]]);export{M as t};
lecture-08/assets/KaTeX_AMS-Regular-BQhdFMY1.woff2 ADDED
Binary file (28.1 kB). View file
 
lecture-08/assets/KaTeX_AMS-Regular-DMm9YOAa.woff ADDED
Binary file (33.5 kB). View file
 
lecture-08/assets/KaTeX_AMS-Regular-DRggAlZN.ttf ADDED
Binary file (63.6 kB). View file
 
lecture-08/assets/KaTeX_Caligraphic-Bold-ATXxdsX0.ttf ADDED
Binary file (12.4 kB). View file
 
lecture-08/assets/KaTeX_Caligraphic-Bold-BEiXGLvX.woff ADDED
Binary file (7.72 kB). View file
 
lecture-08/assets/KaTeX_Caligraphic-Bold-Dq_IR9rO.woff2 ADDED
Binary file (6.91 kB). View file
 
lecture-08/assets/KaTeX_Caligraphic-Regular-CTRA-rTL.woff ADDED
Binary file (7.66 kB). View file
 
lecture-08/assets/KaTeX_Caligraphic-Regular-Di6jR-x-.woff2 ADDED
Binary file (6.91 kB). View file
 
lecture-08/assets/KaTeX_Caligraphic-Regular-wX97UBjC.ttf ADDED
Binary file (12.3 kB). View file
 
lecture-08/assets/KaTeX_Fraktur-Bold-BdnERNNW.ttf ADDED
Binary file (19.6 kB). View file
 
lecture-08/assets/KaTeX_Fraktur-Bold-BsDP51OF.woff ADDED
Binary file (13.3 kB). View file
 
lecture-08/assets/KaTeX_Fraktur-Bold-CL6g_b3V.woff2 ADDED
Binary file (11.3 kB). View file
 
lecture-08/assets/KaTeX_Fraktur-Regular-CB_wures.ttf ADDED
Binary file (19.6 kB). View file
 
lecture-08/assets/KaTeX_Fraktur-Regular-CTYiF6lA.woff2 ADDED
Binary file (11.3 kB). View file
 
lecture-08/assets/KaTeX_Fraktur-Regular-Dxdc4cR9.woff ADDED
Binary file (13.2 kB). View file
 
lecture-08/assets/KaTeX_Main-Bold-Cx986IdX.woff2 ADDED
Binary file (25.3 kB). View file
 
lecture-08/assets/KaTeX_Main-Bold-Jm3AIy58.woff ADDED
Binary file (29.9 kB). View file
 
lecture-08/assets/KaTeX_Main-Bold-waoOVXN0.ttf ADDED
Binary file (51.3 kB). View file
 
lecture-08/assets/KaTeX_Main-BoldItalic-DxDJ3AOS.woff2 ADDED
Binary file (16.8 kB). View file
 
lecture-08/assets/KaTeX_Main-BoldItalic-DzxPMmG6.ttf ADDED
Binary file (33 kB). View file
 
lecture-08/assets/KaTeX_Main-BoldItalic-SpSLRI95.woff ADDED
Binary file (19.4 kB). View file
 
lecture-08/assets/KaTeX_Main-Italic-3WenGoN9.ttf ADDED
Binary file (33.6 kB). View file
 
lecture-08/assets/KaTeX_Main-Italic-BMLOBm91.woff ADDED
Binary file (19.7 kB). View file
 
lecture-08/assets/KaTeX_Main-Italic-NWA7e6Wa.woff2 ADDED
Binary file (17 kB). View file
 
lecture-08/assets/KaTeX_Main-Regular-B22Nviop.woff2 ADDED
Binary file (26.3 kB). View file
 
lecture-08/assets/KaTeX_Main-Regular-Dr94JaBh.woff ADDED
Binary file (30.8 kB). View file
 
lecture-08/assets/KaTeX_Main-Regular-ypZvNtVU.ttf ADDED
Binary file (53.6 kB). View file
 
lecture-08/assets/KaTeX_Math-BoldItalic-B3XSjfu4.ttf ADDED
Binary file (31.2 kB). View file
 
lecture-08/assets/KaTeX_Math-BoldItalic-CZnvNsCZ.woff2 ADDED
Binary file (16.4 kB). View file
 
lecture-08/assets/KaTeX_Math-BoldItalic-iY-2wyZ7.woff ADDED
Binary file (18.7 kB). View file
 
lecture-08/assets/KaTeX_Math-Italic-DA0__PXp.woff ADDED
Binary file (18.7 kB). View file
 
lecture-08/assets/KaTeX_Math-Italic-flOr_0UB.ttf ADDED
Binary file (31.3 kB). View file
 
lecture-08/assets/KaTeX_Math-Italic-t53AETM-.woff2 ADDED
Binary file (16.4 kB). View file
 
lecture-08/assets/KaTeX_SansSerif-Bold-CFMepnvq.ttf ADDED
Binary file (24.5 kB). View file
 
lecture-08/assets/KaTeX_SansSerif-Bold-D1sUS0GD.woff2 ADDED
Binary file (12.2 kB). View file
 
lecture-08/assets/KaTeX_SansSerif-Bold-DbIhKOiC.woff ADDED
Binary file (14.4 kB). View file
 
lecture-08/assets/KaTeX_SansSerif-Italic-C3H0VqGB.woff2 ADDED
Binary file (12 kB). View file
 
lecture-08/assets/KaTeX_SansSerif-Italic-DN2j7dab.woff ADDED
Binary file (14.1 kB). View file