Spaces:
Running
Running
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>ML Field Manual — Vol. 1</title> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link href="https://fonts.googleapis.com/css2?family=Archivo:wght@500;700;900&family=IBM+Plex+Sans:ital,wght@0,400;0,600;0,700;1,400&family=IBM+Plex+Mono:wght@400;600&display=swap" rel="stylesheet"> | |
| <style> | |
| :root{ | |
| --paper:#F4F6F7; --card:#FFFFFF; --ink:#16262E; --ink-soft:#3E5461; | |
| --steel:#33566B; --flame:#E8551A; --flame-soft:#FBE9DF; | |
| --gauge:#1E7A5A; --gauge-soft:#E3F1EB; | |
| --warn:#B3541E; --warn-soft:#FCEFE3; | |
| --def:#2C5F8A; --def-soft:#E7F0F7; | |
| --line:#C9D3D8; --tagbg:#16262E; | |
| --mono:'IBM Plex Mono',monospace; --disp:'Archivo',sans-serif; --body:'IBM Plex Sans',sans-serif; | |
| } | |
| *{margin:0;padding:0;box-sizing:border-box} | |
| html{scroll-behavior:smooth} | |
| body{background:var(--paper);color:var(--ink);font-family:var(--body);font-size:16.5px;line-height:1.65} | |
| .wrap{display:grid;grid-template-columns:290px 1fr;min-height:100vh} | |
| /* ---------- SIDEBAR ---------- */ | |
| aside{background:var(--ink);color:#DCE6EB;padding:26px 0 40px;position:sticky;top:0;height:100vh;overflow-y:auto} | |
| .brand{padding:0 24px 18px;border-bottom:1px solid #2C424E} | |
| .brand .kicker{font-family:var(--mono);font-size:11px;letter-spacing:.18em;color:var(--flame)} | |
| .brand h1{font-family:var(--disp);font-weight:900;font-size:21px;line-height:1.15;margin-top:6px;color:#fff} | |
| .brand .sub{font-size:12.5px;color:#8FA6B2;margin-top:6px} | |
| .toc{padding:14px 12px} | |
| .toc a{display:flex;gap:10px;align-items:baseline;padding:8px 12px;border-radius:6px;color:#B9CAD3;text-decoration:none;font-size:13.5px;line-height:1.35} | |
| .toc a .n{font-family:var(--mono);font-size:11px;color:#6E8794;min-width:26px} | |
| .toc a.live{color:#fff;background:#233A46} | |
| .toc a.live .n{color:var(--flame)} | |
| .toc a.locked{opacity:.45;cursor:default} | |
| .toc a.locked .st{font-family:var(--mono);font-size:9px;letter-spacing:.08em;color:#7E96A2;border:1px solid #3A5260;border-radius:3px;padding:1px 5px;margin-left:auto;white-space:nowrap} | |
| .toc a.live .st{font-family:var(--mono);font-size:9px;letter-spacing:.08em;color:var(--gauge);border:1px solid var(--gauge);border-radius:3px;padding:1px 5px;margin-left:auto} | |
| /* ---------- MANIFOLD progress strip ---------- */ | |
| .manifold{padding:16px 24px;border-top:1px solid #2C424E;margin-top:8px} | |
| .manifold .lbl{font-family:var(--mono);font-size:10px;letter-spacing:.16em;color:#6E8794;margin-bottom:8px} | |
| .valves{display:flex;gap:5px;flex-wrap:wrap} | |
| .valve{width:13px;height:13px;border-radius:50%;border:2px solid #3A5260;position:relative} | |
| .valve.open{border-color:var(--gauge);background:var(--gauge)} | |
| .valve.open::after{content:'';position:absolute;inset:3px;border-radius:50%;background:#fff;opacity:.85} | |
| /* ---------- MAIN ---------- */ | |
| main{max-width:820px;padding:0 44px 90px;margin:0 auto;width:100%} | |
| /* Cover */ | |
| .cover{padding:64px 0 40px;border-bottom:3px solid var(--ink)} | |
| .cover .tagline{font-family:var(--mono);font-size:12px;letter-spacing:.2em;color:var(--flame);text-transform:uppercase} | |
| .cover h2{font-family:var(--disp);font-weight:900;font-size:clamp(34px,5vw,52px);line-height:1.03;margin:14px 0 16px;letter-spacing:-.01em} | |
| .cover p{max-width:600px;color:var(--ink-soft);font-size:17px} | |
| .nameplate{display:inline-flex;gap:0;margin-top:26px;border:2px solid var(--ink);border-radius:6px;overflow:hidden;font-family:var(--mono);font-size:12px} | |
| .nameplate div{padding:8px 14px} | |
| .nameplate .k{background:var(--ink);color:#fff;letter-spacing:.1em} | |
| .nameplate .v{background:#fff;color:var(--ink)} | |
| /* Chapter plate */ | |
| .plate{display:flex;align-items:center;gap:16px;margin:70px 0 8px} | |
| .plate .tag{font-family:var(--mono);font-weight:600;font-size:13px;background:var(--tagbg);color:#fff;padding:7px 13px;border-radius:5px;letter-spacing:.12em} | |
| .plate .status{font-family:var(--mono);font-size:10px;letter-spacing:.14em;color:var(--gauge);border:1.5px solid var(--gauge);border-radius:4px;padding:3px 8px} | |
| h2.ch{font-family:var(--disp);font-weight:900;font-size:clamp(28px,4vw,38px);line-height:1.08;margin:6px 0 10px;letter-spacing:-.01em} | |
| .ch-intro{font-size:18px;color:var(--ink-soft);max-width:640px;margin-bottom:8px} | |
| h3{font-family:var(--disp);font-weight:700;font-size:23px;margin:52px 0 14px;padding-top:18px;border-top:1px solid var(--line)} | |
| h3 .sn{font-family:var(--mono);font-size:13px;color:var(--flame);margin-right:10px;font-weight:600} | |
| h4{font-family:var(--disp);font-weight:700;font-size:17px;margin:26px 0 8px} | |
| p{margin:0 0 14px} | |
| strong{font-weight:700} | |
| em{font-style:italic} | |
| ul,ol{margin:0 0 16px 22px} | |
| li{margin-bottom:7px} | |
| /* Callouts */ | |
| .co{border-radius:10px;padding:16px 18px;margin:20px 0;border:1px solid var(--line);background:var(--card)} | |
| .co .co-t{font-family:var(--mono);font-size:11px;letter-spacing:.14em;font-weight:600;display:flex;align-items:center;gap:8px;margin-bottom:8px;text-transform:uppercase} | |
| .co.def{background:var(--def-soft);border-color:#BAD2E4}.co.def .co-t{color:var(--def)} | |
| .co.analogy{background:var(--gauge-soft);border-color:#BBDACB}.co.analogy .co-t{color:var(--gauge)} | |
| .co.warn{background:var(--warn-soft);border-color:#EBCBAA}.co.warn .co-t{color:var(--warn)} | |
| .co.plant{background:var(--flame-soft);border-color:#F0C4AC}.co.plant .co-t{color:var(--flame)} | |
| .co p:last-child{margin-bottom:0} | |
| /* Jargon decoder */ | |
| .jd{border:2px solid var(--ink);border-radius:12px;margin:26px 0;overflow:hidden;background:var(--card)} | |
| .jd .jd-head{background:var(--ink);color:#fff;padding:11px 18px;display:flex;align-items:baseline;gap:12px} | |
| .jd .jd-head .w{font-family:var(--disp);font-weight:900;font-size:19px} | |
| .jd .jd-head .ar{font-size:14px;color:#9FB6C2;margin-left:auto} | |
| .jd table{width:100%;border-collapse:collapse;font-size:14.5px} | |
| .jd td{padding:10px 18px;border-top:1px solid var(--line);vertical-align:top} | |
| .jd td:first-child{font-family:var(--mono);font-size:11px;letter-spacing:.08em;color:var(--steel);width:118px;font-weight:600;text-transform:uppercase;padding-top:12px} | |
| /* Tables */ | |
| table.tb{width:100%;border-collapse:collapse;font-size:14px;margin:20px 0;background:var(--card);border:1px solid var(--line);border-radius:10px;overflow:hidden} | |
| table.tb th{background:var(--ink);color:#fff;text-align:left;padding:10px 12px;font-family:var(--mono);font-size:11px;letter-spacing:.1em;text-transform:uppercase;font-weight:600} | |
| table.tb td{padding:10px 12px;border-top:1px solid var(--line);vertical-align:top} | |
| table.tb tr:nth-child(even) td{background:#F8FAFB} | |
| .pill{display:inline-block;font-family:var(--mono);font-size:10.5px;padding:2px 8px;border-radius:20px;font-weight:600;letter-spacing:.04em} | |
| .pill.g{background:var(--gauge-soft);color:var(--gauge)} | |
| .pill.o{background:var(--flame-soft);color:var(--flame)} | |
| .pill.r{background:#F7E1E1;color:#A33} | |
| .pill.b{background:var(--def-soft);color:var(--def)} | |
| /* Figures */ | |
| figure{margin:28px 0;background:var(--card);border:1px solid var(--line);border-radius:12px;padding:22px 20px 14px} | |
| figcaption{font-family:var(--mono);font-size:11.5px;color:var(--steel);margin-top:12px;letter-spacing:.04em;border-top:1px dashed var(--line);padding-top:10px} | |
| figcaption .fn{color:var(--flame);font-weight:600} | |
| /* Interactive type-picker */ | |
| .picker{background:var(--card);border:2px solid var(--ink);border-radius:12px;padding:20px;margin:26px 0} | |
| .picker h4{margin-top:0} | |
| .picker .q-btns{display:flex;flex-wrap:wrap;gap:8px;margin:12px 0} | |
| .picker button{font-family:var(--body);font-size:14px;padding:9px 14px;border:1.5px solid var(--steel);background:#fff;color:var(--steel);border-radius:8px;cursor:pointer;font-weight:600} | |
| .picker button:hover{background:var(--def-soft)} | |
| .picker .ans{border-radius:8px;padding:14px;font-size:14.5px;display:none;margin-top:6px} | |
| .picker .ans.show{display:block} | |
| .picker .ans .t{font-family:var(--mono);font-size:11px;letter-spacing:.12em;font-weight:600;text-transform:uppercase;margin-bottom:5px} | |
| /* End-of-chapter blocks */ | |
| .eoc{margin-top:60px;border-top:3px solid var(--ink);padding-top:6px} | |
| .eoc-block{background:var(--card);border:1px solid var(--line);border-radius:12px;margin:22px 0;overflow:hidden} | |
| .eoc-block>.hd{padding:13px 20px;font-family:var(--disp);font-weight:700;font-size:16px;display:flex;align-items:center;gap:10px;border-bottom:1px solid var(--line);background:#FBFCFD} | |
| .eoc-block>.hd .ic{font-family:var(--mono);font-size:10px;letter-spacing:.1em;background:var(--ink);color:#fff;padding:3px 8px;border-radius:4px} | |
| .eoc-block>.bd{padding:18px 20px} | |
| .eoc-block>.bd> :last-child{margin-bottom:0} | |
| /* Quiz */ | |
| .quiz-q{margin-bottom:22px;padding-bottom:18px;border-bottom:1px dashed var(--line)} | |
| .quiz-q:last-of-type{border-bottom:none} | |
| .quiz-q .qt{font-weight:700;margin-bottom:9px} | |
| .quiz-q label{display:flex;gap:9px;align-items:flex-start;padding:7px 10px;border-radius:7px;cursor:pointer;font-size:14.5px} | |
| .quiz-q label:hover{background:#F2F5F7} | |
| .quiz-q input{margin-top:4px;accent-color:var(--flame)} | |
| .quiz-q.correct label.right{background:var(--gauge-soft);outline:1.5px solid var(--gauge)} | |
| .quiz-q.wrong label.right{background:var(--gauge-soft);outline:1.5px solid var(--gauge)} | |
| .quiz-q.wrong label.picked{background:#F7E1E1;outline:1.5px solid #A33} | |
| .quiz-q .exp{display:none;font-size:13.5px;color:var(--ink-soft);background:#F4F6F7;border-left:3px solid var(--steel);padding:9px 12px;border-radius:0 8px 8px 0;margin-top:8px} | |
| .quiz-q.revealed .exp{display:block} | |
| .q-actions{display:flex;align-items:center;gap:14px;margin-top:8px} | |
| .btn{font-family:var(--disp);font-weight:700;font-size:14px;background:var(--flame);color:#fff;border:none;padding:11px 20px;border-radius:8px;cursor:pointer;letter-spacing:.02em} | |
| .btn:hover{background:#CE4712} | |
| .score{font-family:var(--mono);font-size:14px;font-weight:600} | |
| /* Prompts */ | |
| .prompt{background:var(--ink);color:#D9E4EA;border-radius:10px;margin:14px 0;overflow:hidden} | |
| .prompt .ph{display:flex;align-items:center;padding:9px 14px;background:#22343E;font-family:var(--mono);font-size:11px;letter-spacing:.1em;color:#9FB6C2} | |
| .prompt .ph button{margin-left:auto;font-family:var(--mono);font-size:11px;background:none;border:1px solid #4A6474;color:#B9CAD3;border-radius:5px;padding:3px 10px;cursor:pointer} | |
| .prompt .ph button:hover{border-color:var(--flame);color:var(--flame)} | |
| .prompt pre{padding:14px 16px;font-family:var(--mono);font-size:13px;line-height:1.55;white-space:pre-wrap;word-break:break-word} | |
| /* Cheat sheet grid */ | |
| .cs{display:grid;grid-template-columns:1fr 1fr;gap:12px} | |
| .cs .cell{border:1px solid var(--line);border-radius:9px;padding:12px 14px;background:#FBFCFD;font-size:13.5px} | |
| .cs .cell .k{font-family:var(--mono);font-size:11px;color:var(--flame);font-weight:600;letter-spacing:.06em;text-transform:uppercase;margin-bottom:4px} | |
| /* Details / expandables */ | |
| details{border:1px solid var(--line);border-radius:9px;background:var(--card);margin:10px 0;overflow:hidden} | |
| details summary{cursor:pointer;padding:12px 16px;font-weight:600;font-size:14.5px;list-style:none;display:flex;gap:10px;align-items:center} | |
| details summary::before{content:'+';font-family:var(--mono);color:var(--flame);font-weight:600;font-size:16px} | |
| details[open] summary::before{content:'–'} | |
| details .dbody{padding:0 16px 14px;font-size:14.5px;color:var(--ink-soft)} | |
| /* footer nav */ | |
| .next{margin-top:60px;border:2px dashed var(--line);border-radius:12px;padding:22px;text-align:center;color:var(--ink-soft)} | |
| .next .t{font-family:var(--mono);font-size:11px;letter-spacing:.16em;color:var(--flame);text-transform:uppercase;margin-bottom:6px} | |
| @media (max-width:960px){ | |
| .wrap{grid-template-columns:1fr} | |
| aside{position:static;height:auto} | |
| .toc{display:flex;flex-wrap:wrap;gap:2px} | |
| .toc a{font-size:12px;padding:6px 9px} | |
| .toc a .st{display:none} | |
| main{padding:0 20px 70px} | |
| .cs{grid-template-columns:1fr} | |
| } | |
| @media (prefers-reduced-motion:reduce){html{scroll-behavior:auto}} | |
| /* paging */ | |
| article.chx{display:none} | |
| article.chx.active{display:block} | |
| .cover.hidden{display:none} | |
| /* sidebar toggle */ | |
| #sbtoggle{position:fixed;top:14px;left:14px;z-index:50;font-family:var(--mono);font-size:11px;letter-spacing:.08em;background:var(--ink);color:#fff;border:none;border-radius:7px;padding:8px 12px;cursor:pointer;box-shadow:0 2px 8px rgba(0,0,0,.25)} | |
| #sbtoggle:hover{background:var(--flame)} | |
| body.nosb .wrap{grid-template-columns:1fr} | |
| body.nosb aside{display:none} | |
| /* footer nav */ | |
| .chfoot{margin-top:54px;display:flex;gap:12px;align-items:center;border-top:3px solid var(--ink);padding-top:22px;flex-wrap:wrap} | |
| .chfoot .ghost{font-family:var(--disp);font-weight:700;font-size:14px;background:#fff;color:var(--steel);border:1.5px solid var(--steel);padding:10px 18px;border-radius:8px;cursor:pointer} | |
| .chfoot .ghost:hover{background:var(--def-soft)} | |
| .chfoot .done{margin-left:auto} | |
| .toc a{cursor:pointer} | |
| .toc a.done .n{color:var(--gauge)} | |
| .toc a.done::after{content:'✓';color:var(--gauge);margin-left:auto;font-family:var(--mono);font-size:11px} | |
| /* collapsible eoc blocks */ | |
| .eoc-block.clps>.hd{cursor:pointer;user-select:none} | |
| .eoc-block.clps>.hd::after{content:'SHOW';margin-left:auto;font-family:var(--mono);font-size:10px;letter-spacing:.1em;color:var(--flame);border:1px solid var(--flame);border-radius:4px;padding:2px 8px} | |
| .eoc-block.clps.open>.hd::after{content:'HIDE'} | |
| .eoc-block.clps>.bd{display:none} | |
| .eoc-block.clps.open>.bd{display:block} | |
| /* code */ | |
| pre.code{background:var(--ink);color:#D9E4EA;border-radius:10px;padding:14px 16px;font-family:var(--mono);font-size:13px;line-height:1.6;overflow-x:auto;margin:14px 0;white-space:pre} | |
| pre.code .c{color:#7E96A2} | |
| pre.code .k{color:#F09B6E} | |
| pre.code .s{color:#8FD0B2} | |
| @media (max-width:960px){#sbtoggle{top:10px;left:10px}} | |
| </style> | |
| </head> | |
| <body> | |
| <button id="sbtoggle" onclick="document.body.classList.toggle('nosb')">☰ CONTENTS</button> | |
| <div class="wrap"> | |
| <aside> | |
| <div class="brand"> | |
| <div class="kicker">FIELD MANUAL · ML-FM-001</div> | |
| <h1>Machine Learning<br>Field Manual</h1> | |
| <div class="sub">From plant floor to production models · built for a Planning Engineer</div> | |
| </div> | |
| <nav class="toc" id="toc"> | |
| <a data-ch="1"><span class="n">01</span> What ML Actually Is</a> | |
| <a data-ch="2"><span class="n">02</span> Python for ML</a> | |
| <a data-ch="3"><span class="n">03</span> The Data Science Ecosystem</a> | |
| <a data-ch="4"><span class="n">04</span> Data</a> | |
| <a data-ch="5"><span class="n">05</span> Exploratory Data Analysis</a> | |
| <a data-ch="6"><span class="n">06</span> Statistics for ML</a> | |
| <a data-ch="7"><span class="n">07</span> ML Algorithms</a> | |
| <a data-ch="8"><span class="n">08</span> Deep Learning</a> | |
| <a data-ch="9"><span class="n">09</span> Model Evaluation</a> | |
| <a data-ch="10"><span class="n">10</span> Model Improvement</a> | |
| <a data-ch="11"><span class="n">11</span> Production ML</a> | |
| <a data-ch="12"><span class="n">12</span> Generative AI</a> | |
| <a data-ch="13"><span class="n">13</span> ML at the LPG Plant</a> | |
| <a data-ch="14"><span class="n">14</span> First Projects</a> | |
| <a data-ch="15"><span class="n">15</span> Working With AI Assistants</a> | |
| <a data-ch="16"><span class="n">16</span> Learning Roadmap</a> | |
| </nav> | |
| <div class="manifold"> | |
| <div class="lbl">READING MANIFOLD — <span id="mcount">0</span> / 16 COMPLETED</div> | |
| <div class="valves"><div class="valve" id="v1"></div><div class="valve" id="v2"></div><div class="valve" id="v3"></div><div class="valve" id="v4"></div><div class="valve" id="v5"></div><div class="valve" id="v6"></div><div class="valve" id="v7"></div><div class="valve" id="v8"></div><div class="valve" id="v9"></div><div class="valve" id="v10"></div><div class="valve" id="v11"></div><div class="valve" id="v12"></div><div class="valve" id="v13"></div><div class="valve" id="v14"></div><div class="valve" id="v15"></div><div class="valve" id="v16"></div></div> | |
| </div> | |
| </aside> | |
| <main> | |
| <section class="cover"> | |
| <div class="tagline">Complete Edition</div> | |
| <h2>Machine Learning,<br>engineered for the plant.</h2> | |
| <p>A practitioner's textbook written for one reader: a Planning Engineer at an LPG cylinder filling plant who will use Claude and ChatGPT to write the code — but insists on truly understanding every concept behind it.</p> | |
| <div class="nameplate"> | |
| <div class="k">DOC</div><div class="v">ML-FM-001</div> | |
| <div class="k">REV</div><div class="v">1.0</div> | |
| <div class="k">STATUS</div><div class="v">16 CHAPTERS · FULL EDITION</div> | |
| </div> | |
| <div style="margin-top:28px"><button class="btn" onclick="go(1)">Open Chapter 1 →</button></div> | |
| </section> | |
| <!-- ================= CHAPTER 1 ================= --> | |
| <article id="ch1" class="chx"> | |
| <div class="plate"><span class="tag">CH-01</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">What Machine Learning Actually Is</h2> | |
| <p class="ch-intro">Before touching a single line of code, you need one mental model so solid that no jargon can shake it. This chapter builds it.</p> | |
| <h3><span class="sn">1.1</span>The one-sentence definition</h3> | |
| <p>Traditional software works because a human writes explicit rules: <em>if pressure > 8 bar, open the relief valve</em>. Machine Learning flips this. Instead of writing the rules, you show the computer many <strong>examples</strong>, and it discovers the rules by itself.</p> | |
| <div class="co def"> | |
| <div class="co-t">◈ Formal Definition</div> | |
| <p>Tom Mitchell's classic textbook definition (1997): a program learns from <strong>experience E</strong> with respect to a <strong>task T</strong> and a <strong>performance measure P</strong>, if its performance at T, measured by P, improves with E. In plain words: <em>the more good examples it sees, the better it gets at the job.</em></p> | |
| </div> | |
| <div class="co analogy"> | |
| <div class="co-t">⬡ Plant Analogy</div> | |
| <p>A new operator on the filling carousel doesn't memorize a rulebook for every possible situation. They watch thousands of cylinders pass, and gradually develop an instinct: <em>"that hissing sound plus that gauge flicker means a bad valve seat."</em> Nobody wrote that rule down — it was learned from experience. ML gives software the same ability, at industrial scale.</p> | |
| </div> | |
| <figure> | |
| <svg viewBox="0 0 720 250" width="100%" role="img" aria-label="Traditional programming vs machine learning"> | |
| <defs> | |
| <marker id="arr" markerWidth="9" markerHeight="9" refX="7" refY="4.5" orient="auto"><path d="M0,0 L9,4.5 L0,9 z" fill="#33566B"/></marker> | |
| </defs> | |
| <text x="10" y="24" font-family="Archivo" font-weight="900" font-size="15" fill="#16262E">TRADITIONAL PROGRAMMING</text> | |
| <rect x="10" y="42" width="130" height="44" rx="8" fill="#E7F0F7" stroke="#2C5F8A"/><text x="75" y="68" text-anchor="middle" font-family="IBM Plex Sans" font-size="13" fill="#16262E">Rules</text> | |
| <rect x="10" y="96" width="130" height="44" rx="8" fill="#E7F0F7" stroke="#2C5F8A"/><text x="75" y="122" text-anchor="middle" font-family="IBM Plex Sans" font-size="13" fill="#16262E">Data</text> | |
| <line x1="140" y1="64" x2="215" y2="85" stroke="#33566B" stroke-width="2" marker-end="url(#arr)"/> | |
| <line x1="140" y1="118" x2="215" y2="97" stroke="#33566B" stroke-width="2" marker-end="url(#arr)"/> | |
| <rect x="220" y="66" width="130" height="50" rx="8" fill="#16262E"/><text x="285" y="96" text-anchor="middle" font-family="Archivo" font-weight="700" font-size="13" fill="#fff">Computer</text> | |
| <line x1="350" y1="91" x2="425" y2="91" stroke="#33566B" stroke-width="2" marker-end="url(#arr)"/> | |
| <rect x="430" y="66" width="140" height="50" rx="8" fill="#E3F1EB" stroke="#1E7A5A"/><text x="500" y="96" text-anchor="middle" font-family="IBM Plex Sans" font-size="13" fill="#16262E">Answers</text> | |
| <text x="10" y="178" font-family="Archivo" font-weight="900" font-size="15" fill="#E8551A">MACHINE LEARNING</text> | |
| <rect x="10" y="192" width="130" height="40" rx="8" fill="#FBE9DF" stroke="#E8551A"/><text x="75" y="216" text-anchor="middle" font-family="IBM Plex Sans" font-size="13" fill="#16262E">Data</text> | |
| <rect x="160" y="192" width="130" height="40" rx="8" fill="#FBE9DF" stroke="#E8551A"/><text x="225" y="216" text-anchor="middle" font-family="IBM Plex Sans" font-size="13" fill="#16262E">Answers</text> | |
| <line x1="290" y1="212" x2="345" y2="212" stroke="#33566B" stroke-width="2" marker-end="url(#arr)"/> | |
| <rect x="350" y="187" width="130" height="50" rx="8" fill="#16262E"/><text x="415" y="217" text-anchor="middle" font-family="Archivo" font-weight="700" font-size="13" fill="#fff">Computer</text> | |
| <line x1="480" y1="212" x2="535" y2="212" stroke="#33566B" stroke-width="2" marker-end="url(#arr)"/> | |
| <rect x="540" y="187" width="140" height="50" rx="8" fill="#E3F1EB" stroke="#1E7A5A"/><text x="610" y="212" text-anchor="middle" font-family="Archivo" font-weight="700" font-size="13" fill="#1E7A5A">RULES</text> | |
| <text x="610" y="228" text-anchor="middle" font-family="IBM Plex Mono" font-size="10" fill="#33566B">(the "model")</text> | |
| </svg> | |
| <figcaption><span class="fn">FIG 1.1</span> — The fundamental inversion. In ML, rules come OUT of the computer instead of going in. This single diagram is the whole field.</figcaption> | |
| </figure> | |
| <p>Now decode the six words you will hear more than any others. Each card follows the same 5-step format used across the entire manual.</p> | |
| <div class="jd"> | |
| <div class="jd-head"><span class="w">Model</span><span class="ar">نموذج</span></div> | |
| <table> | |
| <tr><td>Technical</td><td>A mathematical function, learned from data, that maps inputs to outputs.</td></tr> | |
| <tr><td>Simple</td><td>The "rules" the computer discovered, saved as a file you can reuse.</td></tr> | |
| <tr><td>Why it matters</td><td>The model IS the product. Everything else — data cleaning, training, tuning — exists to produce a good model.</td></tr> | |
| <tr><td>Analogy</td><td>The experienced operator's instinct, bottled. You can copy it to every shift.</td></tr> | |
| <tr><td>You'll use it</td><td>"Claude, train a model to predict tomorrow's cylinder demand from this CSV."</td></tr> | |
| </table> | |
| </div> | |
| <div class="jd"> | |
| <div class="jd-head"><span class="w">Training</span><span class="ar">التدريب</span></div> | |
| <table> | |
| <tr><td>Technical</td><td>The optimization process where the algorithm adjusts internal parameters to minimize error on example data.</td></tr> | |
| <tr><td>Simple</td><td>Showing the computer many examples until its guesses stop being wrong.</td></tr> | |
| <tr><td>Why it matters</td><td>Training quality decides everything. Garbage examples → garbage model, no exceptions.</td></tr> | |
| <tr><td>Analogy</td><td>Commissioning a new compressor: run it, measure, adjust, repeat until performance meets spec.</td></tr> | |
| <tr><td>You'll use it</td><td>You'll split your plant data, train on one part, and keep the rest hidden to test honestly.</td></tr> | |
| </table> | |
| </div> | |
| <div class="jd"> | |
| <div class="jd-head"><span class="w">Features</span><span class="ar">الخصائص / المدخلات</span></div> | |
| <table> | |
| <tr><td>Technical</td><td>The input variables the model uses to make a prediction.</td></tr> | |
| <tr><td>Simple</td><td>The columns of your spreadsheet that describe each example.</td></tr> | |
| <tr><td>Why it matters</td><td>Good features beat fancy algorithms. This is where YOUR plant knowledge becomes a superpower no data scientist has.</td></tr> | |
| <tr><td>Analogy</td><td>The readings you'd check before diagnosing a pump: vibration, temperature, hours since service.</td></tr> | |
| <tr><td>You'll use it</td><td>Deciding that "days since last maintenance" belongs in the demand model — that decision is feature engineering.</td></tr> | |
| </table> | |
| </div> | |
| <div class="jd"> | |
| <div class="jd-head"><span class="w">Label</span><span class="ar">التصنيف / الإجابة</span></div> | |
| <table> | |
| <tr><td>Technical</td><td>The known correct answer attached to each training example (also called the "target").</td></tr> | |
| <tr><td>Simple</td><td>The answer column. What you want the model to predict.</td></tr> | |
| <tr><td>Why it matters</td><td>No labels = no supervised learning. Most workplace ML projects die because labels don't exist or are unreliable.</td></tr> | |
| <tr><td>Analogy</td><td>QC inspection results stamped on each cylinder: PASS / FAIL. That stamp is the label.</td></tr> | |
| <tr><td>You'll use it</td><td>Your historical "actual filled quantity" column becomes the label when forecasting production.</td></tr> | |
| </table> | |
| </div> | |
| <div class="jd"> | |
| <div class="jd-head"><span class="w">Algorithm</span><span class="ar">الخوارزمية</span></div> | |
| <table> | |
| <tr><td>Technical</td><td>The learning procedure (e.g., Random Forest, Linear Regression) that produces a model from data.</td></tr> | |
| <tr><td>Simple</td><td>The recipe. Data goes in, model comes out.</td></tr> | |
| <tr><td>Why it matters</td><td>Beginners obsess over algorithms; practitioners know data quality matters 10× more. Chapter 7 gives you a decision tree so you never guess.</td></tr> | |
| <tr><td>Analogy</td><td>Algorithm : Model :: Filling procedure : A filled cylinder. One is the process, the other is the output.</td></tr> | |
| <tr><td>You'll use it</td><td>"Claude, compare a Random Forest and Gradient Boosting on this dataset and tell me which wins."</td></tr> | |
| </table> | |
| </div> | |
| <div class="jd"> | |
| <div class="jd-head"><span class="w">Inference / Prediction</span><span class="ar">الاستدلال / التنبؤ</span></div> | |
| <table> | |
| <tr><td>Technical</td><td>Running new, unseen inputs through a trained model to get outputs.</td></tr> | |
| <tr><td>Simple</td><td>Using the model. Training is school; inference is the job.</td></tr> | |
| <tr><td>Why it matters</td><td>Training happens once (or occasionally); inference runs daily in production. Their requirements differ completely — this drives Chapter 11.</td></tr> | |
| <tr><td>Analogy</td><td>Training = certifying the operator. Inference = the operator working today's shift.</td></tr> | |
| <tr><td>You'll use it</td><td>Your morning Power BI dashboard showing tomorrow's predicted demand — that number came from inference.</td></tr> | |
| </table> | |
| </div> | |
| <h3><span class="sn">1.2</span>AI vs Machine Learning vs Deep Learning</h3> | |
| <p>These three terms are used interchangeably in meetings, and that causes real confusion in procurement documents and project scoping. They are <strong>nested circles</strong>, not synonyms.</p> | |
| <figure> | |
| <svg viewBox="0 0 720 330" width="100%" role="img" aria-label="Nested circles of AI, ML, DL, GenAI"> | |
| <ellipse cx="270" cy="168" rx="255" ry="152" fill="#E7F0F7" stroke="#2C5F8A" stroke-width="2"/> | |
| <text x="270" y="52" text-anchor="middle" font-family="Archivo" font-weight="900" font-size="16" fill="#2C5F8A">ARTIFICIAL INTELLIGENCE</text> | |
| <text x="270" y="70" text-anchor="middle" font-family="IBM Plex Sans" font-size="11.5" fill="#33566B">Any technique that makes machines act "smart" — incl. old rule-based systems</text> | |
| <ellipse cx="255" cy="192" rx="185" ry="108" fill="#E3F1EB" stroke="#1E7A5A" stroke-width="2"/> | |
| <text x="255" y="112" text-anchor="middle" font-family="Archivo" font-weight="900" font-size="15" fill="#1E7A5A">MACHINE LEARNING</text> | |
| <text x="255" y="129" text-anchor="middle" font-family="IBM Plex Sans" font-size="11.5" fill="#1E7A5A">Learns rules from data instead of being programmed</text> | |
| <ellipse cx="240" cy="222" rx="118" ry="64" fill="#FBE9DF" stroke="#E8551A" stroke-width="2"/> | |
| <text x="240" y="205" text-anchor="middle" font-family="Archivo" font-weight="900" font-size="14" fill="#E8551A">DEEP LEARNING</text> | |
| <text x="240" y="222" text-anchor="middle" font-family="IBM Plex Sans" font-size="11" fill="#B3541E">ML using multi-layer neural networks</text> | |
| <ellipse cx="228" cy="252" rx="66" ry="26" fill="#16262E"/> | |
| <text x="228" y="250" text-anchor="middle" font-family="Archivo" font-weight="700" font-size="11" fill="#fff">GENERATIVE AI</text> | |
| <text x="228" y="263" text-anchor="middle" font-family="IBM Plex Mono" font-size="9" fill="#9FB6C2">ChatGPT · Claude</text> | |
| <line x1="530" y1="120" x2="620" y2="120" stroke="#2C5F8A" stroke-width="1.5" stroke-dasharray="4 3"/> | |
| <text x="628" y="112" font-family="IBM Plex Mono" font-size="10.5" fill="#2C5F8A">e.g. a thermostat</text> | |
| <text x="628" y="126" font-family="IBM Plex Mono" font-size="10.5" fill="#2C5F8A">rule: IF temp>25 → cool</text> | |
| <text x="628" y="140" font-family="IBM Plex Mono" font-size="10.5" fill="#2C5F8A">= AI but NOT ML</text> | |
| </svg> | |
| <figcaption><span class="fn">FIG 1.2</span> — Every deep learning system is ML; every ML system is AI. The reverse is false. Claude and ChatGPT live in the innermost ring.</figcaption> | |
| </figure> | |
| <div class="co warn"> | |
| <div class="co-t">⚠ Meeting-Room Trap</div> | |
| <p>When a vendor says "our system uses AI", it may be a 1990s-style rulebook. Ask one question: <strong>"What data was it trained on?"</strong> If there's no training data, it's not machine learning — and it won't improve over time.</p> | |
| </div> | |
| <h3><span class="sn">1.3</span>A 70-year history in one timeline</h3> | |
| <p>You don't need dates memorized — you need to understand <em>why ML exploded now</em> and not in 1980. Three ingredients had to arrive together: <strong>data</strong> (digitization), <strong>compute</strong> (GPUs), and <strong>algorithms</strong> (backpropagation → transformers).</p> | |
| <figure> | |
| <svg viewBox="0 0 720 300" width="100%" role="img" aria-label="ML history timeline"> | |
| <line x1="40" y1="150" x2="690" y2="150" stroke="#16262E" stroke-width="3"/> | |
| <g font-family="IBM Plex Mono" font-size="11" fill="#16262E"> | |
| <circle cx="60" cy="150" r="6" fill="#2C5F8A"/><text x="60" y="130" text-anchor="middle" font-weight="600">1950</text><text x="60" y="180" text-anchor="middle" font-size="10">Turing asks</text><text x="60" y="193" text-anchor="middle" font-size="10">"Can machines</text><text x="60" y="206" text-anchor="middle" font-size="10">think?"</text> | |
| <circle cx="150" cy="150" r="6" fill="#2C5F8A"/><text x="150" y="130" text-anchor="middle" font-weight="600">1959</text><text x="150" y="103" text-anchor="middle" font-size="10">Samuel coins</text><text x="150" y="116" text-anchor="middle" font-size="10">"machine learning"</text> | |
| <circle cx="250" cy="150" r="6" fill="#2C5F8A"/><text x="250" y="130" text-anchor="middle" font-weight="600">1986</text><text x="250" y="180" text-anchor="middle" font-size="10">Backpropagation</text><text x="250" y="193" text-anchor="middle" font-size="10">popularized —</text><text x="250" y="206" text-anchor="middle" font-size="10">neural nets trainable</text> | |
| <circle cx="340" cy="150" r="6" fill="#1E7A5A"/><text x="340" y="130" text-anchor="middle" font-weight="600">1997</text><text x="340" y="103" text-anchor="middle" font-size="10">Deep Blue beats</text><text x="340" y="116" text-anchor="middle" font-size="10">Kasparov (chess)</text> | |
| <circle cx="440" cy="150" r="7" fill="#E8551A"/><text x="440" y="130" text-anchor="middle" font-weight="600" fill="#E8551A">2012</text><text x="440" y="180" text-anchor="middle" font-size="10">AlexNet wins ImageNet —</text><text x="440" y="193" text-anchor="middle" font-size="10">deep learning era begins</text><text x="440" y="206" text-anchor="middle" font-size="10">(GPUs + big data)</text> | |
| <circle cx="540" cy="150" r="7" fill="#E8551A"/><text x="540" y="130" text-anchor="middle" font-weight="600" fill="#E8551A">2017</text><text x="540" y="103" text-anchor="middle" font-size="10">"Attention Is All</text><text x="540" y="116" text-anchor="middle" font-size="10">You Need" — the</text><text x="540" y="90" text-anchor="middle" font-size="10">Transformer</text> | |
| <circle cx="640" cy="150" r="7" fill="#16262E"/><text x="640" y="130" text-anchor="middle" font-weight="600">2022+</text><text x="640" y="180" text-anchor="middle" font-size="10">ChatGPT, Claude —</text><text x="640" y="193" text-anchor="middle" font-size="10">GenAI reaches</text><text x="640" y="206" text-anchor="middle" font-size="10">everyone</text> | |
| </g> | |
| <rect x="40" y="245" width="650" height="40" rx="8" fill="#F8FAFB" stroke="#C9D3D8"/> | |
| <text x="365" y="262" text-anchor="middle" font-family="IBM Plex Sans" font-size="12" fill="#33566B">Why now? Data (sensors, ERP, digitization) + Compute (GPUs) + Algorithms (transformers)</text> | |
| <text x="365" y="278" text-anchor="middle" font-family="IBM Plex Sans" font-weight="700" font-size="12" fill="#E8551A">All three arrived together only after ~2012.</text> | |
| </svg> | |
| <figcaption><span class="fn">FIG 1.3</span> — The ideas are old; the enabling conditions are new. Your plant's SCADA and ERP data are exactly the fuel this era runs on.</figcaption> | |
| </figure> | |
| <h3><span class="sn">1.4</span>The three families of Machine Learning</h3> | |
| <p>Almost every ML technique belongs to one of three families, defined by <strong>what kind of feedback the algorithm learns from</strong>.</p> | |
| <table class="tb"> | |
| <tr><th>Family</th><th>Learns from</th><th>Question it answers</th><th>Plant example</th></tr> | |
| <tr><td><strong>Supervised</strong><br><span class="pill g">~80% of business ML</span></td><td>Examples with known answers (labels)</td><td>"Given these inputs, what is the output?"</td><td>Predict next month's 12kg cylinder demand from 5 years of history</td></tr> | |
| <tr><td><strong>Unsupervised</strong><br><span class="pill b">Pattern discovery</span></td><td>Examples with NO answers</td><td>"What structure hides in this data?"</td><td>Group filling-line stoppages into natural clusters → discover unknown failure modes</td></tr> | |
| <tr><td><strong>Reinforcement</strong><br><span class="pill o">Rare in business</span></td><td>Trial, error, and rewards</td><td>"What sequence of actions maximizes reward?"</td><td>Optimizing a filling carousel's speed schedule (advanced — not a first project)</td></tr> | |
| </table> | |
| <p>Supervised learning further splits into two tasks, and telling them apart is a skill you'll use in every project scoping meeting:</p> | |
| <ul> | |
| <li><strong>Regression</strong> → predict a <em>number</em> (tomorrow's demand: 4,120 cylinders)</li> | |
| <li><strong>Classification</strong> → predict a <em>category</em> (this cylinder: PASS or FAIL)</li> | |
| </ul> | |
| <div class="picker"> | |
| <h4>⚙ Interactive — Which family is it?</h4> | |
| <p style="font-size:14px;color:var(--ink-soft)">Tap a real plant problem. Check your instinct before revealing.</p> | |
| <div class="q-btns"> | |
| <button onclick="pick(0)">Predict pump failure in next 7 days</button> | |
| <button onclick="pick(1)">Find unusual patterns in energy bills</button> | |
| <button onclick="pick(2)">Sort incoming cylinders: refurbish vs scrap</button> | |
| <button onclick="pick(3)">Forecast LPG demand for Ramadan</button> | |
| <button onclick="pick(4)">Group customers by ordering behavior</button> | |
| </div> | |
| <div class="ans" id="a0" style="background:var(--gauge-soft)"><div class="t" style="color:var(--gauge)">Supervised · Classification</div>You have historical labels (failed / didn't fail within 7 days). Predicting a category → classification.</div> | |
| <div class="ans" id="a1" style="background:var(--def-soft)"><div class="t" style="color:var(--def)">Unsupervised · Anomaly detection</div>No labels for "unusual" — the algorithm learns what normal looks like and flags deviations.</div> | |
| <div class="ans" id="a2" style="background:var(--gauge-soft)"><div class="t" style="color:var(--gauge)">Supervised · Classification</div>Past QC decisions are your labels. Two categories → binary classification.</div> | |
| <div class="ans" id="a3" style="background:var(--gauge-soft)"><div class="t" style="color:var(--gauge)">Supervised · Regression (time series)</div>Predicting a quantity → regression. The Ramadan seasonality makes it a time-series problem (Ch. 7).</div> | |
| <div class="ans" id="a4" style="background:var(--def-soft)"><div class="t" style="color:var(--def)">Unsupervised · Clustering</div>No predefined groups exist — you're asking the data to reveal them.</div> | |
| </div> | |
| <h3><span class="sn">1.5</span>Where ML creates real business value</h3> | |
| <p>McKinsey, Google, and Microsoft case studies converge on the same pattern: ML pays off where decisions are <strong>frequent, repetitive, data-rich, and currently made by rough rules of thumb</strong>. It does NOT pay off on rare, one-off strategic decisions.</p> | |
| <table class="tb"> | |
| <tr><th>Value pattern</th><th>Generic example</th><th>Your world (GFB-U / GFB-S)</th></tr> | |
| <tr><td>Predict to plan earlier</td><td>Retail demand forecasting</td><td>Cylinder demand & filling schedule by season</td></tr> | |
| <tr><td>Detect before failure</td><td>Aircraft engine monitoring</td><td>Compressor / carousel predictive maintenance</td></tr> | |
| <tr><td>Inspect at scale</td><td>PCB visual inspection</td><td>Cylinder valve & body defect detection (camera)</td></tr> | |
| <tr><td>Optimize allocation</td><td>Route optimization</td><td>Truck dispatch between Umm Al-Aish & Al-Shuaiba</td></tr> | |
| <tr><td>Find the needle</td><td>Fraud detection</td><td>Anomalies in OPEX line items or energy consumption</td></tr> | |
| <tr><td>Unlock documents</td><td>Contract analysis</td><td>RAG over your SharePoint spec corpus — already on your radar</td></tr> | |
| </table> | |
| <h3><span class="sn">1.6</span>Where ML fails — the honest section</h3> | |
| <p>Most ML failures are predictable before a single model is trained. Industry post-mortems (Google's ML guides are blunt about this) repeat the same causes:</p> | |
| <table class="tb"> | |
| <tr><th>Failure condition</th><th>Why it kills the project</th><th>Pre-flight check</th></tr> | |
| <tr><td>Too little data</td><td>Models need hundreds–thousands of examples of the thing you predict. 6 failures in 5 years ≠ a dataset.</td><td>Count the rows of the <em>rare</em> class first.</td></tr> | |
| <tr><td>The answer isn't in the data</td><td>If demand is driven by a ministry decision, no sensor data will predict it.</td><td>Ask: could a smart human predict this from these columns?</td></tr> | |
| <tr><td>The world changes (drift)</td><td>A model trained pre-2020 died in 2020. Patterns expire.</td><td>Plan for monitoring & retraining from day one (Ch. 11).</td></tr> | |
| <tr><td>Labels are wrong</td><td>If QC stamps were inconsistent, the model learns the inconsistency perfectly.</td><td>Audit 50 random labels manually before training.</td></tr> | |
| <tr><td>A simple rule already works</td><td>If "average of last 4 weeks" forecasts within 3%, ML adds cost, not value.</td><td>Always build the dumb baseline first — this is Rule #1 of the whole manual.</td></tr> | |
| </table> | |
| <div class="co plant"> | |
| <div class="co-t">▲ Rule #1 of this manual</div> | |
| <p><strong>Never deploy a model that can't beat a simple baseline.</strong> Before any ML: compute the naive answer (last value, moving average, "always predict PASS"). Every model must earn its complexity by beating it. You'll see this rule again in Chapters 7, 9, and 14 — it is the single habit that separates practitioners from hobbyists.</p> | |
| </div> | |
| <h3><span class="sn">1.7</span>Ethics, bias, hallucinations & explainability</h3> | |
| <p>Four risk concepts every practitioner must carry — especially in a safety-critical plant environment.</p> | |
| <div class="jd"> | |
| <div class="jd-head"><span class="w">Bias</span><span class="ar">التحيّز</span></div> | |
| <table> | |
| <tr><td>Technical</td><td>Systematic error where a model performs worse for certain groups or conditions, usually inherited from unrepresentative training data.</td></tr> | |
| <tr><td>Simple</td><td>The model is unfair or blind to situations it rarely saw.</td></tr> | |
| <tr><td>Why it matters</td><td>A defect detector trained only on summer daylight photos will miss defects on night shift. Bias isn't only a social issue — it's an operational one.</td></tr> | |
| <tr><td>Analogy</td><td>An inspector who only ever worked day shift judging night-shift conditions.</td></tr> | |
| <tr><td>You'll use it</td><td>Checklist question in every project: "Which situations are under-represented in my training data?"</td></tr> | |
| </table> | |
| </div> | |
| <div class="jd"> | |
| <div class="jd-head"><span class="w">Hallucination</span><span class="ar">الهلوسة</span></div> | |
| <table> | |
| <tr><td>Technical</td><td>A generative model (LLM) producing fluent, confident output that is factually false.</td></tr> | |
| <tr><td>Simple</td><td>Claude/ChatGPT sometimes makes things up — smoothly.</td></tr> | |
| <tr><td>Why it matters</td><td>You will rely on AI assistants for code. Hallucinated function names, invented library parameters, and fake citations are your #1 daily risk. Chapter 15 is a full defense course.</td></tr> | |
| <tr><td>Analogy</td><td>A confident contractor quoting a spec clause that doesn't exist. Verify against the document, not the confidence.</td></tr> | |
| <tr><td>You'll use it</td><td>Habit: run AI-generated code on a tiny sample and check one output by hand before trusting it.</td></tr> | |
| </table> | |
| </div> | |
| <div class="jd"> | |
| <div class="jd-head"><span class="w">Explainability</span><span class="ar">قابلية التفسير</span></div> | |
| <table> | |
| <tr><td>Technical</td><td>The degree to which a model's predictions can be understood by humans; tools like SHAP and LIME attribute predictions to input features.</td></tr> | |
| <tr><td>Simple</td><td>Can the model tell you WHY it decided that?</td></tr> | |
| <tr><td>Why it matters</td><td>In a plant, "the AI said shut down line 2" is unacceptable without a reason. Management, HSE, and auditors will demand explanations. Interpretable models often win in industry even when slightly less accurate.</td></tr> | |
| <tr><td>Analogy</td><td>A junior engineer recommending a shutdown must show the readings that led there. Same standard for models.</td></tr> | |
| <tr><td>You'll use it</td><td>"Claude, add SHAP analysis showing the top 5 drivers of each prediction" — a prompt you'll reuse constantly.</td></tr> | |
| </table> | |
| </div> | |
| <div class="co warn"> | |
| <div class="co-t">⚠ Ethics in one paragraph</div> | |
| <p>Three commitments cover 90% of workplace AI ethics: <strong>(1)</strong> never let a model make a safety-critical decision without a human in the loop, <strong>(2)</strong> never feed personal or confidential data into external AI tools without clearance — your on-prem LLM plan exists precisely for this, and <strong>(3)</strong> always disclose when a number in a report is a model's prediction, not a measurement.</p> | |
| </div> | |
| <!-- ============ END OF CHAPTER MODULES ============ --> | |
| <div class="eoc"> | |
| <div class="plate" style="margin-top:34px"><span class="tag">CH-01 · CLOSE-OUT</span></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"> | |
| <ul> | |
| <li>ML inverts programming: <strong>data + answers → rules</strong>, instead of rules + data → answers.</li> | |
| <li>AI ⊃ ML ⊃ Deep Learning ⊃ Generative AI. They are nested, not synonyms.</li> | |
| <li>~80% of business ML is <strong>supervised learning</strong>: regression (numbers) or classification (categories).</li> | |
| <li>ML thrives on frequent, repetitive, data-rich decisions — not rare strategic ones.</li> | |
| <li>Rule #1: <strong>no model deploys without beating a simple baseline.</strong></li> | |
| <li>Your unfair advantage is domain knowledge: choosing features is where a Planning Engineer beats a generic data scientist.</li> | |
| </ul> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>What Beginners Usually Misunderstand</div><div class="bd"> | |
| <details><summary>"ML is smart like a human"</summary><div class="dbody">It's pattern-matching at scale, with zero understanding. It will happily learn a nonsense pattern if your data contains one. Treat it like a brilliant but extremely literal intern.</div></details> | |
| <details><summary>"More complex algorithm = better results"</summary><div class="dbody">In tabular business data, simple models (linear regression, gradient-boosted trees) routinely match or beat deep learning — while being faster and explainable. Complexity is a cost you must justify.</div></details> | |
| <details><summary>"The model is done once it's trained"</summary><div class="dbody">Training is ~20% of a real project. Data preparation before, and monitoring/retraining after, are the other 80%. Patterns drift; models expire.</div></details> | |
| <details><summary>"ChatGPT/Claude = Machine Learning"</summary><div class="dbody">They are one product OF machine learning (generative AI). Your demand forecast will use classical ML — a completely different toolset that the same assistants can help you write.</div></details> | |
| <details><summary>"We need big data"</summary><div class="dbody">A clean spreadsheet with 2,000 rows can power a genuinely useful forecasting model. You need <em>enough relevant</em> data, not "big" data.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Practical Workplace Applications</div><div class="bd"> | |
| <table class="tb"> | |
| <tr><th>Idea</th><th>Family</th><th>Data you already have</th><th>Difficulty</th></tr> | |
| <tr><td>Monthly cylinder demand forecast</td><td>Regression / time series</td><td>Filling records, seasonality, holidays</td><td><span class="pill g">Starter</span></td></tr> | |
| <tr><td>Energy-consumption anomaly alerts</td><td>Unsupervised</td><td>Utility bills, production volumes</td><td><span class="pill g">Starter</span></td></tr> | |
| <tr><td>Downtime-cause classification from logs</td><td>Classification / NLP</td><td>Maintenance & stoppage logs</td><td><span class="pill o">Medium</span></td></tr> | |
| <tr><td>Valve defect detection by camera</td><td>Deep learning / vision</td><td>Needs new labeled images</td><td><span class="pill r">Advanced</span></td></tr> | |
| </table> | |
| <p style="font-size:14px;color:var(--ink-soft)">Chapter 13 ranks ~30 of these by ROI, data readiness, and difficulty.</p> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd" id="quiz"> | |
| <div class="quiz-q" data-a="1"> | |
| <div class="qt">1. What is the fundamental difference between ML and traditional programming?</div> | |
| <label><input type="radio" name="q1" value="0">ML runs on faster computers</label> | |
| <label><input type="radio" name="q1" value="1">ML learns rules from data instead of having rules written by hand</label> | |
| <label><input type="radio" name="q1" value="2">ML always uses neural networks</label> | |
| <div class="exp">The inversion in FIG 1.1: data + answers go in, rules (the model) come out.</div> | |
| </div> | |
| <div class="quiz-q" data-a="2"> | |
| <div class="qt">2. Predicting the exact number of cylinders needed next Tuesday is…</div> | |
| <label><input type="radio" name="q2" value="0">Classification</label> | |
| <label><input type="radio" name="q2" value="1">Clustering</label> | |
| <label><input type="radio" name="q2" value="2">Regression</label> | |
| <div class="exp">Predicting a quantity (a number) = regression. A category (PASS/FAIL) = classification.</div> | |
| </div> | |
| <div class="quiz-q" data-a="0"> | |
| <div class="qt">3. You have 5 years of stoppage logs but no categories assigned. Grouping them into natural clusters is…</div> | |
| <label><input type="radio" name="q3" value="0">Unsupervised learning</label> | |
| <label><input type="radio" name="q3" value="1">Supervised learning</label> | |
| <label><input type="radio" name="q3" value="2">Reinforcement learning</label> | |
| <div class="exp">No labels → unsupervised. The algorithm finds structure you didn't define.</div> | |
| </div> | |
| <div class="quiz-q" data-a="1"> | |
| <div class="qt">4. In a demand-forecast dataset, the column "actual quantity filled" is the…</div> | |
| <label><input type="radio" name="q4" value="0">Feature</label> | |
| <label><input type="radio" name="q4" value="1">Label (target)</label> | |
| <label><input type="radio" name="q4" value="2">Algorithm</label> | |
| <div class="exp">It's the answer the model must learn to predict. The describing columns (date, season, temperature…) are features.</div> | |
| </div> | |
| <div class="quiz-q" data-a="2"> | |
| <div class="qt">5. A vendor's "AI system" has no training data behind it. What is it most likely?</div> | |
| <label><input type="radio" name="q5" value="0">Deep learning</label> | |
| <label><input type="radio" name="q5" value="1">Generative AI</label> | |
| <label><input type="radio" name="q5" value="2">A rule-based system (AI, but not ML)</label> | |
| <div class="exp">No data → nothing was learned. It sits in the outer AI ring of FIG 1.2 and will never improve with experience.</div> | |
| </div> | |
| <div class="quiz-q" data-a="0"> | |
| <div class="qt">6. Before deploying any model, Rule #1 says you must…</div> | |
| <label><input type="radio" name="q6" value="0">Beat a simple baseline (e.g., moving average)</label> | |
| <label><input type="radio" name="q6" value="1">Use deep learning</label> | |
| <label><input type="radio" name="q6" value="2">Collect at least 1 million rows</label> | |
| <div class="exp">A model must earn its complexity. If a 4-week average forecasts equally well, ship the average.</div> | |
| </div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score" id="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"> | |
| <ol> | |
| <li><strong>Decision inventory (30 min).</strong> List 10 decisions made weekly at GFB-U/GFB-S. For each, mark: frequent? data exists? currently rule-of-thumb? Any row with 3 ✓ is an ML candidate.</li> | |
| <li><strong>Family sorting.</strong> Take your 10 decisions and assign each to supervised-regression, supervised-classification, unsupervised, or "not ML" — using the table in §1.4.</li> | |
| <li><strong>Baseline drill.</strong> Pick one metric you report monthly. Compute what a naive forecast (last month's value) would have predicted for the past 6 months. Note the average error — this is the number any future model must beat.</li> | |
| <li><strong>Label audit.</strong> Find one dataset at work with a "result" column. Sample 20 rows: would two colleagues assign the same label? Estimate the % of disagreement.</li> | |
| </ol> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection Questions</div><div class="bd"> | |
| <ul> | |
| <li>Which decision at your plant is currently made by pure experience, where a wrong call costs the most money?</li> | |
| <li>Where does your domain knowledge exceed anything written in the data? How could that knowledge become a feature?</li> | |
| <li>If a model contradicted your 10-year intuition, what evidence would make you trust it?</li> | |
| </ul> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"> | |
| <ul> | |
| <li>Starting with an algorithm ("let's use deep learning!") instead of a decision that needs improving.</li> | |
| <li>Calling a project "AI" in a proposal without specifying the task, the data, and the metric.</li> | |
| <li>Trusting a confident LLM answer about your own plant data — it has never seen your data.</li> | |
| <li>Skipping the baseline and celebrating a model that a moving average would beat.</li> | |
| <li>Assuming labels are correct because they're in the system.</li> | |
| </ul> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet — Chapter 1</div><div class="bd"> | |
| <div class="cs"> | |
| <div class="cell"><div class="k">Core inversion</div>Data + Answers → Computer → Rules (model)</div> | |
| <div class="cell"><div class="k">Nesting</div>AI ⊃ ML ⊃ DL ⊃ GenAI</div> | |
| <div class="cell"><div class="k">Predict a number</div>Regression (demand, energy, cost)</div> | |
| <div class="cell"><div class="k">Predict a category</div>Classification (pass/fail, will-fail/won't)</div> | |
| <div class="cell"><div class="k">No labels</div>Unsupervised (clustering, anomaly detection)</div> | |
| <div class="cell"><div class="k">Good ML problem</div>Frequent + repetitive + data-rich + rule-of-thumb today</div> | |
| <div class="cell"><div class="k">Rule #1</div>Beat the naive baseline or don't ship</div> | |
| <div class="cell"><div class="k">4 risks</div>Bias · Hallucination · Drift · No explainability</div> | |
| </div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts You Can Use Today</div><div class="bd"> | |
| <div class="prompt"><div class="ph">PROMPT 1 · PROBLEM SCOPING<button onclick="cp(this)">Copy</button></div> | |
| <pre>I am a Planning Engineer at an LPG cylinder filling plant. Here is a decision we make regularly: [describe decision]. The data we have: [list columns/sources]. | |
| 1) Is this a machine learning problem? | |
| 2) If yes, is it regression, classification, clustering, or anomaly detection — and why? | |
| 3) What simple non-ML baseline should I compute first? | |
| 4) What data problems should I expect? | |
| Explain in simple English. Do not write code yet.</pre></div> | |
| <div class="prompt"><div class="ph">PROMPT 2 · JARGON DECODER<button onclick="cp(this)">Copy</button></div> | |
| <pre>Explain the term "[TERM]" to me using this exact format: | |
| 1) Technical definition, 2) Simple English, 3) Why it matters in practice, 4) An analogy from an industrial plant, 5) How I will actually use it as a beginner working with plant data. Keep each part under 3 sentences.</pre></div> | |
| <div class="prompt"><div class="ph">PROMPT 3 · VENDOR/PROPOSAL STRESS-TEST<button onclick="cp(this)">Copy</button></div> | |
| <pre>Here is a description of an "AI solution" proposed for our plant: [paste]. Act as a skeptical ML engineer. Tell me: (a) is this actually machine learning or a rule-based system, (b) what training data it would require, (c) what questions I should ask the vendor, (d) what a cheap in-house alternative might look like.</pre></div> | |
| <div class="prompt"><div class="ph">PROMPT 4 · BASELINE FIRST<button onclick="cp(this)">Copy</button></div> | |
| <pre>I will paste a small CSV of monthly [metric] data. Before any ML: compute two naive baselines (last value, and 3-month moving average), report their average absolute error, and tell me the error number any future model must beat. Show results as a small table and explain like I'm new to this.</pre></div> | |
| </div></div> | |
| </div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Chapter 1 Bibliography</div><div class="bd" style="font-size:13.5px"> | |
| <ul style="margin-left:18px"> | |
| <li>Mitchell, T. (1997). <em>Machine Learning.</em> McGraw-Hill — source of the E/T/P definition (§1.1).</li> | |
| <li>Samuel, A. L. (1959). "Some Studies in Machine Learning Using the Game of Checkers." <em>IBM Journal of Research and Development</em> — origin of the term (§1.3).</li> | |
| <li>Turing, A. M. (1950). "Computing Machinery and Intelligence." <em>Mind</em> (§1.3).</li> | |
| <li>Rumelhart, Hinton & Williams (1986). "Learning representations by back-propagating errors." <em>Nature</em> (§1.3).</li> | |
| <li>Krizhevsky, Sutskever & Hinton (2012). "ImageNet Classification with Deep Convolutional Neural Networks." <em>NeurIPS</em> — AlexNet (§1.3).</li> | |
| <li>Vaswani et al. (2017). "Attention Is All You Need." <em>NeurIPS</em> — the Transformer (§1.3).</li> | |
| <li>Google Developers — <em>Machine Learning Crash Course</em> & <em>Rules of ML</em> (developers.google.com/machine-learning) — basis for §1.6 failure conditions and the baseline rule.</li> | |
| <li>Ng, A. — <em>Machine Learning Specialization</em>, DeepLearning.AI / Stanford — supervised/unsupervised framing (§1.4).</li> | |
| <li>Lundberg & Lee (2017). "A Unified Approach to Interpreting Model Predictions" (SHAP), <em>NeurIPS</em>; Ribeiro et al. (2016). "Why Should I Trust You?" (LIME), <em>KDD</em> (§1.7).</li> | |
| <li>scikit-learn documentation (scikit-learn.org) — terminology alignment for estimator/model/fit.</li> | |
| </ul> | |
| <p style="color:var(--ink-soft)">Full consolidated bibliography will appear as an appendix when the final section is commissioned.</p> | |
| </div></div> | |
| <div class="chfoot"><button class="ghost" onclick="go(0)">← Cover</button><button class="btn done" onclick="done(1)">Mark complete → Ch. 2</button></div></article> | |
| <!-- ================= CHAPTER 2 ================= --> | |
| <article id="ch2" class="chx"> | |
| <div class="plate"><span class="tag">CH-02</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Python for Machine Learning</h2> | |
| <p class="ch-intro">You won't write most of your code — Claude will. But you must <strong>read it, verify it, and fix it</strong>. This chapter teaches exactly that reading fluency, nothing more.</p> | |
| <h3><span class="sn">2.1</span>Why Python, and how much of it you need</h3> | |
| <p>Python won ML for one reason: it reads almost like English, and every major ML library is built for it. You need roughly <strong>15% of the Python language</strong> to be a fully effective ML practitioner. This chapter is that 15%.</p> | |
| <div class="co analogy"><div class="co-t">⬡ Plant Analogy</div><p>You don't need to machine your own valves to run a filling plant — you need to read P&IDs fluently and know when something looks wrong. Python fluency for you means <em>reading</em> code like you read a P&ID.</p></div> | |
| <h3><span class="sn">2.2</span>Variables and the four containers</h3> | |
| <pre class="code"><span class="c"># A variable is a labeled box holding a value</span> | |
| plant = <span class="s">"GFB-U"</span> <span class="c"># text (string)</span> | |
| daily_target = <span class="k">4200</span> <span class="c"># whole number (int)</span> | |
| fill_rate = <span class="k">0.94</span> <span class="c"># decimal (float)</span> | |
| line_active = <span class="k">True</span> <span class="c"># yes/no (boolean)</span></pre> | |
| <table class="tb"> | |
| <tr><th>Container</th><th>Syntax</th><th>Rules</th><th>You'll use it for</th></tr> | |
| <tr><td><strong>List</strong></td><td><code>[10, 12, 50]</code></td><td>Ordered, changeable, duplicates OK</td><td>90% of cases: rows of readings, filenames, results</td></tr> | |
| <tr><td><strong>Tuple</strong></td><td><code>(29.3, 47.9)</code></td><td>Ordered, <em>frozen</em> after creation</td><td>Fixed pairs: coordinates, (width, height). Rarely written by you</td></tr> | |
| <tr><td><strong>Set</strong></td><td><code>{"GFB-U","GFB-S"}</code></td><td>No duplicates, no order</td><td>De-duplicating; "is X in this group?" checks</td></tr> | |
| <tr><td><strong>Dictionary</strong></td><td><code>{"size": 12, "qty": 480}</code></td><td>Key → value pairs</td><td>Everywhere: configs, one record, JSON from APIs</td></tr> | |
| </table> | |
| <div class="co def"><div class="co-t">◈ Reading shortcut</div><p><code>[ ]</code> = list · <code>( )</code> = tuple · <code>{ }</code> with <code>:</code> = dictionary · <code>{ }</code> without <code>:</code> = set. This one line lets you identify 95% of data structures in AI-generated code instantly.</p></div> | |
| <h3><span class="sn">2.3</span>Loops, conditions, functions</h3> | |
| <pre class="code"><span class="c"># Loop: do something for each item</span> | |
| <span class="k">for</span> cyl <span class="k">in</span> cylinders: | |
| <span class="k">if</span> cyl[<span class="s">"weight"</span>] < <span class="k">14.5</span>: <span class="c"># condition</span> | |
| flag(cyl) | |
| <span class="c"># Function: a reusable named block. def = define</span> | |
| <span class="k">def</span> fill_efficiency(filled, target): | |
| <span class="k">return</span> filled / target * <span class="k">100</span> | |
| fill_efficiency(<span class="k">3990</span>, <span class="k">4200</span>) <span class="c"># → 95.0</span></pre> | |
| <p><strong>Indentation is grammar in Python.</strong> The spaces at the start of a line define what belongs inside the loop/function. When AI-generated code fails with <code>IndentationError</code>, a line is mis-aligned — the most common copy-paste bug you'll meet.</p> | |
| <h3><span class="sn">2.4</span>Classes — the 5-minute version</h3> | |
| <p>A <strong>class</strong> is a blueprint bundling data + functions. You will rarely <em>write</em> classes, but you <em>use</em> them constantly: every scikit-learn model is one.</p> | |
| <pre class="code">model = RandomForestRegressor() <span class="c"># create an object from the class blueprint</span> | |
| model.fit(X, y) <span class="c"># .fit() = a method (function attached to the object)</span> | |
| model.predict(X_new) <span class="c"># the dot means "belonging to model"</span></pre> | |
| <p>That's the entire mental model you need: <em>object dot method</em>. The pattern <code>.fit()</code> then <code>.predict()</code> repeats across all of scikit-learn — learn it once, use it for every algorithm in Chapter 7.</p> | |
| <h3><span class="sn">2.5</span>Imports, pip, and virtual environments</h3> | |
| <table class="tb"> | |
| <tr><th>Term</th><th>Simple English</th><th>Analogy</th></tr> | |
| <tr><td><code>import pandas as pd</code></td><td>Load a toolbox into your session; nickname it <code>pd</code></td><td>Bringing the right toolkit to the job site</td></tr> | |
| <tr><td><code>pip install pandas</code></td><td>Download a library from the internet (once)</td><td>Procuring the toolkit before it can be brought on-site</td></tr> | |
| <tr><td>Virtual environment (<code>venv</code>)</td><td>An isolated folder of libraries per project, so projects don't break each other</td><td>Each project gets its own dedicated tool crib — no shared, mixed-up tools</td></tr> | |
| </table> | |
| <div class="co warn"><div class="co-t">⚠ #1 beginner error</div><p><code>ModuleNotFoundError: No module named 'X'</code> simply means: run <code>pip install X</code>. Not a bug — a missing procurement step.</p></div> | |
| <h3><span class="sn">2.6</span>Where you'll actually work</h3> | |
| <table class="tb"> | |
| <tr><th>Tool</th><th>What it is</th><th>When you use it</th></tr> | |
| <tr><td><strong>Google Colab</strong></td><td>Free Jupyter notebooks in the browser — zero installation, free GPU</td><td><span class="pill g">Start here.</span> All Chapter 14 projects run in Colab</td></tr> | |
| <tr><td><strong>Jupyter Notebook</strong></td><td>Documents mixing code cells + results + notes; run cells one at a time</td><td>Exploration & EDA — see each step's output immediately</td></tr> | |
| <tr><td><strong>VS Code</strong></td><td>Professional code editor</td><td>Later: real scripts, apps, and when you outgrow notebooks</td></tr> | |
| <tr><td><strong>Git / GitHub</strong></td><td>Version control = "track changes" for code; GitHub = the cloud home for it</td><td>Four commands cover you: <code>clone · add · commit · push</code>. You already use GitHub for discovery — now use it for backup</td></tr> | |
| </table> | |
| <h3><span class="sn">2.7</span>Files, CSV, Excel, JSON, APIs</h3> | |
| <pre class="code"><span class="k">import</span> pandas <span class="k">as</span> pd | |
| df = pd.read_csv(<span class="s">"filling_log.csv"</span>) <span class="c"># CSV → table</span> | |
| df = pd.read_excel(<span class="s">"opex_2026.xlsx"</span>) <span class="c"># Excel → table</span> | |
| df.to_excel(<span class="s">"report.xlsx"</span>) <span class="c"># table → Excel</span> | |
| <span class="k">import</span> requests, json | |
| r = requests.get(<span class="s">"https://api.example.com/prices"</span>) | |
| data = r.json() <span class="c"># API response → dictionary</span></pre> | |
| <ul> | |
| <li><strong>CSV</strong> — plain-text table; the universal currency of ML data.</li> | |
| <li><strong>JSON</strong> — nested dictionaries as text; how APIs and configs speak. If you can read a Python dict, you can read JSON.</li> | |
| <li><strong>API</strong> — a service you call over the internet that returns data. <code>requests.get(url)</code> is the whole pattern.</li> | |
| </ul> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>You need reading fluency, not authorship: ~15% of Python covers all of ML practice.</li> | |
| <li>Brackets tell you the structure: <code>[]</code> list, <code>{}</code>+<code>:</code> dict, <code>()</code> tuple.</li> | |
| <li>The universal ML pattern is <em>object.method()</em>: <code>model.fit()</code> → <code>model.predict()</code>.</li> | |
| <li>pip installs once; import loads each session; venv keeps projects isolated.</li> | |
| <li>Start in Google Colab — zero setup, and Claude's code pastes straight in.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"I must memorize syntax"</summary><div class="dbody">No. You must recognize structures and read error messages. Claude writes; you verify. Memorization comes free with repetition.</div></details> | |
| <details><summary>"Errors mean I failed"</summary><div class="dbody">Errors are the normal workflow. Professionals see dozens daily. The skill is pasting the full error back to Claude with context.</div></details> | |
| <details><summary>"Notebooks are for beginners only"</summary><div class="dbody">Professional data scientists live in notebooks for exploration. Scripts come later, for automation.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>Read your monthly OPEX Excel into pandas and automate the summary you currently build by hand.</li> | |
| <li>A Colab notebook that ingests the GFB-U filling log CSV and produces the charts for your report.</li> | |
| <li>A GitHub private repo as the single home for all your plant analysis notebooks.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="1"><div class="qt">1. <code>{"line": 2, "status": "run"}</code> is a…</div> | |
| <label><input type="radio" name="c2q1" value="0">List</label><label><input type="radio" name="c2q1" value="1">Dictionary</label><label><input type="radio" name="c2q1" value="2">Set</label> | |
| <div class="exp">Curly braces with key: value pairs = dictionary.</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">2. <code>ModuleNotFoundError: No module named 'seaborn'</code> — the fix is:</div> | |
| <label><input type="radio" name="c2q2" value="0"><code>pip install seaborn</code></label><label><input type="radio" name="c2q2" value="1">Restart the computer</label><label><input type="radio" name="c2q2" value="2">Rewrite the code</label> | |
| <div class="exp">Missing library = missing procurement. Install once, then import works.</div></div> | |
| <div class="quiz-q" data-a="2"><div class="qt">3. In <code>model.fit(X, y)</code>, <code>.fit</code> is a…</div> | |
| <label><input type="radio" name="c2q3" value="0">Variable</label><label><input type="radio" name="c2q3" value="1">Library</label><label><input type="radio" name="c2q3" value="2">Method — a function belonging to the model object</label> | |
| <div class="exp">The dot means "belonging to". fit = train this model on data X with answers y.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Open colab.research.google.com, create a notebook, run <code>print("GFB-U online")</code>.</li> | |
| <li>Ask Claude for a 10-row fake cylinder-filling CSV, upload it to Colab, load it with <code>pd.read_csv</code>, and run <code>df.head()</code>.</li> | |
| <li>Deliberately break the code (delete a bracket), read the error, fix it yourself before asking Claude.</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Which Excel task do you repeat monthly that a 20-line script could kill forever?</li> | |
| <li>What in AI-generated code would you currently accept blindly because you can't read it?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Installing Python locally and fighting setup for days — use Colab first.</li> | |
| <li>Retyping code from a screenshot instead of copy-pasting (indentation breaks).</li> | |
| <li>Pasting only the last error line to Claude — always paste the <em>full</em> traceback.</li> | |
| <li>Skipping venv, then breaking an old project by upgrading a library for a new one.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Load table</div><code>pd.read_csv("f.csv")</code> / <code>read_excel</code></div> | |
| <div class="cell"><div class="k">Peek</div><code>df.head()</code>, <code>df.info()</code>, <code>df.describe()</code></div> | |
| <div class="cell"><div class="k">Install / load</div><code>pip install X</code> → <code>import X</code></div> | |
| <div class="cell"><div class="k">Function</div><code>def name(inputs): return output</code></div> | |
| <div class="cell"><div class="k">Loop + condition</div><code>for x in items:</code> · <code>if cond:</code></div> | |
| <div class="cell"><div class="k">The ML pattern</div><code>model.fit(X,y)</code> → <code>model.predict(X)</code></div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">CODE EXPLAINER<button onclick="cp(this)">Copy</button></div><pre>Explain this Python code line by line, in simple English, for a non-programmer engineer. For each line: what it does, and what would break if it were removed. Then list the 3 lines I should verify most carefully. | |
| [paste code]</pre></div> | |
| <div class="prompt"><div class="ph">ERROR FIXER<button onclick="cp(this)">Copy</button></div><pre>I ran your code in Google Colab and got this full error. Explain the cause in one sentence, give the fixed code, and tell me how to prevent this class of error in future. | |
| [paste FULL traceback]</pre></div> | |
| <div class="prompt"><div class="ph">COLAB SETUP<button onclick="cp(this)">Copy</button></div><pre>Give me a Google Colab starter cell for a data analysis project: install and import pandas, numpy, matplotlib, seaborn; set display options for wide tables; include a test that prints library versions. Add a comment above every line.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>Python official documentation & tutorial — docs.python.org (language semantics, §2.2–2.4).</li> | |
| <li>pip & venv official guides — packaging.python.org (§2.5).</li> | |
| <li>Project Jupyter documentation — jupyter.org; Google Colab documentation — research.google.com/colaboratory (§2.6).</li> | |
| <li>pandas official documentation, IO tools — pandas.pydata.org (§2.7).</li> | |
| <li>Pro Git (Chacon & Straub) — git-scm.com/book (§2.6).</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(1)">← Ch. 1</button><button class="btn done" onclick="done(2)">Mark complete → Ch. 3</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 3 ================= --> | |
| <article id="ch3" class="chx"> | |
| <div class="plate"><span class="tag">CH-03</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">The Data Science Ecosystem</h2> | |
| <p class="ch-intro">Sixteen libraries look intimidating until you see them as one plant flow diagram: raw material in, refined product out, each unit doing one job.</p> | |
| <h3><span class="sn">3.1</span>The ecosystem as a process flow</h3> | |
| <figure> | |
| <svg viewBox="0 0 720 240" width="100%" role="img" aria-label="Data science library pipeline"> | |
| <defs><marker id="ar3" markerWidth="9" markerHeight="9" refX="7" refY="4.5" orient="auto"><path d="M0,0 L9,4.5 L0,9 z" fill="#33566B"/></marker></defs> | |
| <g font-family="IBM Plex Sans" font-size="12"> | |
| <rect x="10" y="80" width="120" height="72" rx="9" fill="#E7F0F7" stroke="#2C5F8A"/><text x="70" y="105" text-anchor="middle" font-weight="700">STORAGE</text><text x="70" y="122" text-anchor="middle" font-size="10.5">SQL · DuckDB</text><text x="70" y="136" text-anchor="middle" font-size="10.5">CSV · Excel · PyArrow</text> | |
| <line x1="130" y1="116" x2="165" y2="116" stroke="#33566B" stroke-width="2" marker-end="url(#ar3)"/> | |
| <rect x="170" y="80" width="120" height="72" rx="9" fill="#E3F1EB" stroke="#1E7A5A"/><text x="230" y="105" text-anchor="middle" font-weight="700">WRANGLING</text><text x="230" y="122" text-anchor="middle" font-size="10.5">pandas · Polars</text><text x="230" y="136" text-anchor="middle" font-size="10.5">NumPy underneath</text> | |
| <line x1="290" y1="116" x2="325" y2="116" stroke="#33566B" stroke-width="2" marker-end="url(#ar3)"/> | |
| <rect x="330" y="80" width="120" height="72" rx="9" fill="#FBE9DF" stroke="#E8551A"/><text x="390" y="105" text-anchor="middle" font-weight="700">MODELING</text><text x="390" y="122" text-anchor="middle" font-size="10.5">scikit-learn</text><text x="390" y="136" text-anchor="middle" font-size="10.5">SciPy · statsmodels</text> | |
| <line x1="450" y1="116" x2="485" y2="116" stroke="#33566B" stroke-width="2" marker-end="url(#ar3)"/> | |
| <rect x="490" y="80" width="120" height="72" rx="9" fill="#F4F6F7" stroke="#16262E"/><text x="550" y="105" text-anchor="middle" font-weight="700">DELIVERY</text><text x="550" y="122" text-anchor="middle" font-size="10.5">Matplotlib · Seaborn</text><text x="550" y="136" text-anchor="middle" font-size="10.5">Plotly · joblib · ONNX</text> | |
| <rect x="170" y="185" width="440" height="40" rx="9" fill="#16262E"/><text x="390" y="210" text-anchor="middle" fill="#fff" font-size="11.5" font-family="IBM Plex Mono">OPS LAYER: MLflow (experiments) · DVC (data versioning) — runs under everything</text> | |
| <line x1="390" y1="152" x2="390" y2="182" stroke="#33566B" stroke-width="2" stroke-dasharray="4 3"/> | |
| <text x="70" y="60" font-family="IBM Plex Mono" font-size="10" fill="#33566B">RAW MATERIAL</text> | |
| <text x="550" y="60" font-family="IBM Plex Mono" font-size="10" fill="#33566B">FINISHED PRODUCT</text> | |
| </g></svg> | |
| <figcaption><span class="fn">FIG 3.1</span> — Storage → Wrangling → Modeling → Delivery, with an ops layer beneath. Every library below slots into exactly one box.</figcaption> | |
| </figure> | |
| <h3><span class="sn">3.2</span>The 16-library register</h3> | |
| <table class="tb"> | |
| <tr><th>Library</th><th>What / why it exists</th><th>Use when</th><th>Avoid when</th></tr> | |
| <tr><td><strong>NumPy</strong></td><td>Fast math on arrays of numbers; the engine every other library runs on</td><td>Rarely directly — it's under the hood</td><td>For labeled tables → use pandas on top</td></tr> | |
| <tr><td><strong>pandas</strong></td><td>Excel-like tables (DataFrames) in code; the daily workhorse</td><td>Every project, every day</td><td>Data > RAM (millions of rows crawling) → Polars/DuckDB</td></tr> | |
| <tr><td><strong>Matplotlib</strong></td><td>The original plotting library; full control, verbose</td><td>Static report figures, fine-grained control</td><td>Quick statistical charts → Seaborn is faster</td></tr> | |
| <tr><td><strong>Seaborn</strong></td><td>Beautiful statistical charts on top of Matplotlib in one line</td><td>EDA: distributions, correlations, boxplots</td><td>Interactive dashboards → Plotly</td></tr> | |
| <tr><td><strong>Plotly</strong></td><td>Interactive charts (hover, zoom) for HTML/dashboards</td><td>Anything a manager will click</td><td>Static PDF/print figures</td></tr> | |
| <tr><td><strong>scikit-learn</strong></td><td>THE classical-ML library: all Chapter 7 algorithms, one consistent API</td><td>Every tabular ML project</td><td>Deep learning (→ PyTorch) or huge data</td></tr> | |
| <tr><td><strong>SciPy</strong></td><td>Scientific computing: stats tests, optimization, signal processing</td><td>A specific statistical test or optimization</td><td>General ML — sklearn wraps what you need</td></tr> | |
| <tr><td><strong>statsmodels</strong></td><td>Statistics-first modeling: p-values, confidence intervals, ARIMA</td><td>When you need to EXPLAIN (inference), classic time series</td><td>When you only need to PREDICT accurately</td></tr> | |
| <tr><td><strong>Polars</strong></td><td>pandas' younger, multi-core, much faster rival</td><td>pandas becomes slow on large files</td><td>Starting out — pandas has 100× more tutorials/AI training data</td></tr> | |
| <tr><td><strong>DuckDB</strong></td><td>SQL engine on your laptop; queries huge CSV/Parquet instantly, no server</td><td>SQL on big local files; pre-aggregating before pandas</td><td>Multi-user production DB → real SQL server</td></tr> | |
| <tr><td><strong>SQL</strong></td><td>The language of company databases; how you'll pull COGNOS/ERP data</td><td>Extracting plant data at the source</td><td>— (non-negotiable skill; your text-to-SQL pilot proves it)</td></tr> | |
| <tr><td><strong>PyArrow</strong></td><td>Columnar memory format + Parquet files; the plumbing between tools</td><td>Reading/writing Parquet; pandas↔DuckDB handoff</td><td>You'll rarely call it directly</td></tr> | |
| <tr><td><strong>joblib</strong></td><td>Saves trained sklearn models to a file</td><td><code>joblib.dump(model,"m.pkl")</code> after every successful training</td><td>Cross-platform model exchange → ONNX</td></tr> | |
| <tr><td><strong>MLflow</strong></td><td>Logs every experiment: parameters, metrics, model versions</td><td>Once you train the same model 10+ times and lose track</td><td>Your very first project — a notes cell is enough</td></tr> | |
| <tr><td><strong>DVC</strong></td><td>Git for datasets — versions large data files alongside code</td><td>Datasets change and results must be reproducible</td><td>Small static CSVs — overkill</td></tr> | |
| <tr><td><strong>ONNX</strong></td><td>Universal saved-model format, portable across languages/tools</td><td>Deploying a model into non-Python systems</td><td>Simple Python-to-Python workflows → joblib</td></tr> | |
| </table> | |
| <div class="co plant"><div class="co-t">▲ Your starter loadout</div><p>For the next 3 months you need exactly five: <strong>pandas + scikit-learn + Seaborn + Matplotlib + joblib</strong>, all preinstalled in Colab. Everything else enters when a specific pain appears — Polars when pandas slows, MLflow when experiments multiply, DuckDB when SQL-on-files beckons. Tools on demand, not tools in advance.</p></div> | |
| <h3><span class="sn">3.3</span>How they connect in one real script</h3> | |
| <pre class="code"><span class="k">import</span> pandas <span class="k">as</span> pd <span class="c"># wrangling</span> | |
| <span class="k">from</span> sklearn.ensemble <span class="k">import</span> RandomForestRegressor | |
| <span class="k">import</span> seaborn <span class="k">as</span> sns, joblib | |
| df = pd.read_csv(<span class="s">"filling_log.csv"</span>) <span class="c"># STORAGE → table</span> | |
| df = df.dropna() <span class="c"># WRANGLING</span> | |
| sns.lineplot(data=df, x=<span class="s">"date"</span>, y=<span class="s">"qty"</span>) <span class="c"># DELIVERY (EDA chart)</span> | |
| model = RandomForestRegressor().fit(X, y) <span class="c"># MODELING</span> | |
| joblib.dump(model, <span class="s">"demand_model.pkl"</span>) <span class="c"># DELIVERY (saved model)</span></pre> | |
| <p>Five lines, four ecosystem boxes. Every project in Chapter 14 is an elaboration of this skeleton.</p> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>The ecosystem is a pipeline: Storage → Wrangling (pandas) → Modeling (sklearn) → Delivery (charts/saved models).</li> | |
| <li>NumPy is the invisible engine; pandas is your daily interface to data.</li> | |
| <li>scikit-learn's uniform <code>.fit()/.predict()</code> API means learning one algorithm teaches you all of them.</li> | |
| <li>statsmodels explains (p-values); sklearn predicts (accuracy). Different questions, different tools.</li> | |
| <li>Start with 5 libraries; adopt the rest only when a specific pain arrives.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"I must learn all 16 before starting"</summary><div class="dbody">Five carry you for months. The register above is a map for later, not a syllabus for now.</div></details> | |
| <details><summary>"pandas is a database"</summary><div class="dbody">pandas holds data in RAM for analysis. Databases store data permanently for many users. You pull FROM databases INTO pandas.</div></details> | |
| <details><summary>"Newer = better (Polars over pandas)"</summary><div class="dbody">AI assistants were trained on oceans of pandas code — their pandas answers are far more reliable. As an AI-assisted practitioner, ecosystem maturity IS a feature.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>DuckDB to run SQL directly on years of exported filling-log CSVs — no IT ticket required.</li> | |
| <li>Plotly charts embedded in HTML reports for management (interactive, like your dashboards).</li> | |
| <li>joblib-saved demand model that a scheduled script reloads every morning for the Power BI feed.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="1"><div class="qt">1. Your daily tool for loading and cleaning tabular plant data:</div> | |
| <label><input type="radio" name="c3q1" value="0">NumPy</label><label><input type="radio" name="c3q1" value="1">pandas</label><label><input type="radio" name="c3q1" value="2">ONNX</label> | |
| <div class="exp">pandas is the Excel-like DataFrame layer; NumPy powers it underneath.</div></div> | |
| <div class="quiz-q" data-a="2"><div class="qt">2. You need p-values and confidence intervals to EXPLAIN a relationship for an audit. Reach for:</div> | |
| <label><input type="radio" name="c3q2" value="0">Plotly</label><label><input type="radio" name="c3q2" value="1">joblib</label><label><input type="radio" name="c3q2" value="2">statsmodels</label> | |
| <div class="exp">statsmodels is statistics-first (inference); sklearn is prediction-first.</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">3. Saving a trained sklearn model for tomorrow's use:</div> | |
| <label><input type="radio" name="c3q3" value="0"><code>joblib.dump(model, "m.pkl")</code></label><label><input type="radio" name="c3q3" value="1">Screenshot the notebook</label><label><input type="radio" name="c3q3" value="2">Retrain it daily from scratch</label> | |
| <div class="exp">joblib serializes the model object to disk; reload with joblib.load.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>In Colab: <code>import pandas, numpy, sklearn, seaborn</code> and print each <code>.__version__</code>.</li> | |
| <li>Take any work CSV, load with pandas, plot one Seaborn chart, and save the figure to PNG.</li> | |
| <li>Ask Claude to rewrite one of your repetitive Excel calculations as pandas — compare results cell by cell.</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Which box of FIG 3.1 is weakest in your current workflow — storage, wrangling, modeling, or delivery?</li> | |
| <li>Where does your data actually live today (COGNOS, SharePoint, Excel folders), and which tool bridges it into pandas?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Tool-collecting: installing MLflow+DVC+Polars before finishing one project with pandas+sklearn.</li> | |
| <li>Fighting Matplotlib syntax for an hour when <code>sns.barplot(...)</code> is one line.</li> | |
| <li>Retraining a model every run because nobody saved it with joblib.</li> | |
| <li>Copy-pasting between Excel and Python instead of <code>read_excel</code> — silent data corruption.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Tables</div>pandas (default) · Polars (speed) · DuckDB (SQL on files)</div> | |
| <div class="cell"><div class="k">Charts</div>Seaborn (EDA) · Matplotlib (control) · Plotly (interactive)</div> | |
| <div class="cell"><div class="k">Models</div>sklearn (predict) · statsmodels (explain) · SciPy (tests)</div> | |
| <div class="cell"><div class="k">Persistence</div>joblib (save model) · PyArrow/Parquet (save data)</div> | |
| <div class="cell"><div class="k">Ops</div>MLflow (experiments) · DVC (data versions) · ONNX (portability)</div> | |
| <div class="cell"><div class="k">Starter five</div>pandas · sklearn · seaborn · matplotlib · joblib</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">LIBRARY CHOOSER<button onclick="cp(this)">Copy</button></div><pre>My task: [describe]. My data: [size, format, where it lives]. From the standard Python data ecosystem, tell me the MINIMUM set of libraries needed, why each one, and explicitly which popular libraries I should NOT use yet and why.</pre></div> | |
| <div class="prompt"><div class="ph">PIPELINE SKELETON<button onclick="cp(this)">Copy</button></div><pre>Write a commented Colab notebook skeleton with these sections: 1) load data from [source], 2) quick quality check, 3) one EDA chart with seaborn, 4) placeholder for a sklearn model, 5) save outputs. Add a markdown cell before each section explaining its purpose in simple English.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>Official documentation: numpy.org · pandas.pydata.org · scikit-learn.org · matplotlib.org · seaborn.pydata.org · plotly.com/python · scipy.org · statsmodels.org · pola.rs · duckdb.org · arrow.apache.org · joblib.readthedocs.io · mlflow.org · dvc.org · onnx.ai.</li> | |
| <li>Harris et al. (2020). "Array programming with NumPy." <em>Nature</em> 585.</li> | |
| <li>Pedregosa et al. (2011). "Scikit-learn: Machine Learning in Python." <em>JMLR</em> 12.</li> | |
| <li>McKinney, W. (2010). "Data Structures for Statistical Computing in Python." <em>SciPy Conf.</em> (pandas).</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(2)">← Ch. 2</button><button class="btn done" onclick="done(3)">Mark complete → Ch. 4</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 4 ================= --> | |
| <article id="ch4" class="chx"> | |
| <div class="plate"><span class="tag">CH-04</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Data — The 80% of Every Project</h2> | |
| <p class="ch-intro">Practitioners spend most of their time here, not on models. Master this chapter and you're ahead of most people who "know ML".</p> | |
| <h3><span class="sn">4.1</span>Three shapes of data</h3> | |
| <table class="tb"> | |
| <tr><th>Shape</th><th>Looks like</th><th>Plant examples</th><th>ML readiness</th></tr> | |
| <tr><td><strong>Structured</strong></td><td>Rows × columns</td><td>Filling logs, OPEX tables, SCADA exports</td><td><span class="pill g">Ready — start here</span></td></tr> | |
| <tr><td><strong>Semi-structured</strong></td><td>Nested tags/keys (JSON, XML, emails)</td><td>API responses, maintenance emails, SAP exports</td><td><span class="pill o">Needs flattening into a table</span></td></tr> | |
| <tr><td><strong>Unstructured</strong></td><td>Free text, images, audio</td><td>Shift reports, cylinder photos, spec PDFs on SharePoint</td><td><span class="pill r">Needs deep learning / LLMs (Ch. 8, 12)</span></td></tr> | |
| </table> | |
| <h3><span class="sn">4.2</span>Data quality: the six killers</h3> | |
| <table class="tb"> | |
| <tr><th>Problem</th><th>How it hides</th><th>Standard treatment</th></tr> | |
| <tr><td><strong>Missing values</strong></td><td>Blanks, zeros-that-mean-blank, "N/A", -999</td><td>Drop rows (if few) · fill with median (numeric) or mode (category) · add a "was_missing" flag column — missingness itself is often a signal</td></tr> | |
| <tr><td><strong>Duplicates</strong></td><td>Same record entered twice; system re-exports</td><td><code>df.duplicated().sum()</code> then <code>drop_duplicates()</code> — always check before counting anything</td></tr> | |
| <tr><td><strong>Wrong types</strong></td><td>Numbers stored as text ("4,200"), dates as strings</td><td><code>pd.to_numeric</code>, <code>pd.to_datetime</code> immediately after loading</td></tr> | |
| <tr><td><strong>Inconsistent categories</strong></td><td>"GFB-U", "GFBU", "Umm Al-Aish" all meaning one plant</td><td>Standardize with a mapping dictionary; lowercase and strip spaces</td></tr> | |
| <tr><td><strong>Outliers</strong></td><td>A 500kg "cylinder", a negative fill time</td><td>Investigate FIRST — is it an error or a real rare event? Errors get fixed; real extremes often carry the most information (they may be the failures you want to predict)</td></tr> | |
| <tr><td><strong>Unit chaos</strong></td><td>kg vs tonnes, KWD vs fils, mixed fiscal/calendar years</td><td>One unit per column, documented. Your planning background makes you the natural enforcer</td></tr> | |
| </table> | |
| <h3><span class="sn">4.3</span>Preparing features: encoding & scaling</h3> | |
| <p>Models eat numbers only. Two conversions handle almost everything:</p> | |
| <ul> | |
| <li><strong>Encoding (categories → numbers).</strong> <em>One-hot encoding</em> turns "Plant = GFB-U/GFB-S" into two 0/1 columns. Never encode categories as 1, 2, 3 when no order exists — the model will invent a fake ranking.</li> | |
| <li><strong>Scaling (numbers → comparable ranges).</strong> <em>Standardization</em> rescales each column to mean 0, spread 1; <em>normalization</em> squeezes into 0–1. Needed for distance-based models (KNN, SVM, neural nets); NOT needed for tree models (Random Forest, XGBoost). This one fact resolves 90% of "should I scale?" confusion.</li> | |
| </ul> | |
| <div class="co analogy"><div class="co-t">⬡ Plant Analogy</div><p>Scaling is unit conversion before comparison. Comparing pressure in bar against temperature in °C on raw magnitude is meaningless — you first bring both to a common reference, exactly like standardization does for features.</p></div> | |
| <h3><span class="sn">4.4</span>Feature engineering — your superpower</h3> | |
| <p><strong>Feature engineering</strong> = creating new input columns from raw data using domain knowledge. It is the highest-ROI activity in classical ML, and it's where a Planning Engineer beats any outside data scientist:</p> | |
| <ul> | |
| <li>From a date → <code>day_of_week</code>, <code>is_ramadan</code>, <code>is_summer_peak</code>, <code>days_to_holiday</code></li> | |
| <li>From maintenance logs → <code>hours_since_last_service</code>, <code>failures_last_90d</code></li> | |
| <li>From production → <code>rolling_7d_average</code>, <code>qty_vs_same_month_last_year</code></li> | |
| </ul> | |
| <p><strong>Feature selection</strong> is the reverse: removing columns that add noise. Fewer, stronger features → simpler, more robust, more explainable models.</p> | |
| <h3><span class="sn">4.5</span>Data leakage — the silent project killer</h3> | |
| <div class="co warn"><div class="co-t">⚠ The most expensive mistake in applied ML</div><p><strong>Leakage</strong> = the model accidentally sees information during training that won't exist at prediction time. Example: predicting cylinder QC failure using a column filled in <em>after</em> inspection. Training accuracy looks miraculous (98%!); production accuracy collapses. <strong>Symptom: results too good to be true. Diagnosis: for every feature ask, "would I know this value BEFORE the moment of prediction?"</strong> If not — remove it.</p></div> | |
| <h3><span class="sn">4.6</span>Splitting data honestly</h3> | |
| <figure> | |
| <svg viewBox="0 0 720 190" width="100%" role="img" aria-label="Train validation test split"> | |
| <g font-family="IBM Plex Sans" font-size="12.5"> | |
| <rect x="10" y="30" width="420" height="54" rx="8" fill="#E3F1EB" stroke="#1E7A5A" stroke-width="2"/><text x="220" y="52" text-anchor="middle" font-weight="700" fill="#1E7A5A">TRAIN ~70%</text><text x="220" y="70" text-anchor="middle" font-size="11">the model learns from this</text> | |
| <rect x="440" y="30" width="120" height="54" rx="8" fill="#E7F0F7" stroke="#2C5F8A" stroke-width="2"/><text x="500" y="52" text-anchor="middle" font-weight="700" fill="#2C5F8A">VALIDATION</text><text x="500" y="70" text-anchor="middle" font-size="11">tune & compare</text> | |
| <rect x="570" y="30" width="140" height="54" rx="8" fill="#FBE9DF" stroke="#E8551A" stroke-width="2"/><text x="640" y="52" text-anchor="middle" font-weight="700" fill="#E8551A">TEST ~15%</text><text x="640" y="70" text-anchor="middle" font-size="11">touched ONCE, at the end</text> | |
| <text x="15" y="120" font-weight="700" fill="#16262E" font-family="Archivo">Time-series rule:</text> | |
| <rect x="10" y="132" width="500" height="34" rx="7" fill="#E3F1EB" stroke="#1E7A5A"/><text x="260" y="153" text-anchor="middle" font-size="11.5">TRAIN = the PAST (e.g., 2021–2024)</text> | |
| <rect x="520" y="132" width="190" height="34" rx="7" fill="#FBE9DF" stroke="#E8551A"/><text x="615" y="153" text-anchor="middle" font-size="11.5">TEST = the FUTURE (2025)</text> | |
| </g></svg> | |
| <figcaption><span class="fn">FIG 4.1</span> — The test set is a sealed sample kept for final QC. For time series, always split by time — random splits leak the future into training.</figcaption> | |
| </figure> | |
| <p><strong>Cross-validation</strong> improves on a single split: divide training data into K folds (usually 5), train on 4, validate on the 5th, rotate, and average. The result is a stable performance estimate not dependent on one lucky split. In sklearn: <code>cross_val_score(model, X, y, cv=5)</code>.</p> | |
| <h3><span class="sn">4.7</span>Pipelines, versioning, labeling</h3> | |
| <ul> | |
| <li><strong>Pipeline</strong> — chaining all preprocessing + the model into one object (<code>sklearn.pipeline.Pipeline</code>), so the exact same transformations apply in training and production. Prevents an entire class of leakage bugs; from Chapter 14 onward every project uses one.</li> | |
| <li><strong>Data versioning</strong> — "which dataset produced this result?" At minimum: dated, immutable snapshot files (<code>filling_log_2026-07-01.parquet</code>); DVC when it grows serious.</li> | |
| <li><strong>Labeling / annotation</strong> — manually attaching correct answers (drawing boxes on defect photos, tagging stoppage causes). Budget it honestly: labeling is often the true cost of an ML project, and consistent labeling guidelines matter more than volume.</li> | |
| </ul> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>Data preparation is most of the job; model training is the short final step.</li> | |
| <li>One-hot encode unordered categories; scale features for distance-based models, skip scaling for trees.</li> | |
| <li>Feature engineering from domain knowledge is the highest-leverage skill you own.</li> | |
| <li>Leakage check for every feature: "known before prediction time?" Too-good results = investigate, don't celebrate.</li> | |
| <li>Test set is sealed until the end; time series split by time; cross-validation for stable estimates.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"Cleaning means deleting anything strange"</summary><div class="dbody">Outliers are investigated, not auto-deleted. In predictive maintenance, the outliers ARE the events you're trying to predict.</div></details> | |
| <details><summary>"98% training accuracy — we're done!"</summary><div class="dbody">Training accuracy is nearly meaningless; models can memorize. Only performance on held-out (or future) data counts.</div></details> | |
| <details><summary>"Filling missing values with 0 is neutral"</summary><div class="dbody">Zero is a real value with meaning (zero production ≠ unrecorded production). Use median/mode + a missing-flag column.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>A reusable "GFB data quality report" notebook: missing %, duplicates, type issues, category chaos — run on any new export.</li> | |
| <li>An engineered feature library for LPG demand: Ramadan/Eid flags, summer index, days-since-holiday.</li> | |
| <li>Dated Parquet snapshots of monthly planning tables — instant reproducibility for any past analysis.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="1"><div class="qt">1. Your QC-failure model scores 99% in training using a column recorded during final inspection. This is:</div> | |
| <label><input type="radio" name="c4q1" value="0">Excellent — deploy it</label><label><input type="radio" name="c4q1" value="1">Data leakage — that column won't exist at prediction time</label><label><input type="radio" name="c4q1" value="2">Underfitting</label> | |
| <div class="exp">Classic leakage: the feature is a consequence of the answer. Production performance will collapse.</div></div> | |
| <div class="quiz-q" data-a="2"><div class="qt">2. Forecasting 2026 demand — the correct split is:</div> | |
| <label><input type="radio" name="c4q2" value="0">Random 80/20</label><label><input type="radio" name="c4q2" value="1">Test on 2021, train on 2022–2025</label><label><input type="radio" name="c4q2" value="2">Train on the past, test on the most recent period</label> | |
| <div class="exp">Time flows one way. Random splits let the model peek at the future.</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">3. Encoding plants GFB-U/GFB-S as 1 and 2 for a linear model is risky because:</div> | |
| <label><input type="radio" name="c4q3" value="0">It invents a fake order/magnitude — use one-hot instead</label><label><input type="radio" name="c4q3" value="1">Numbers are slower than text</label><label><input type="radio" name="c4q3" value="2">Two categories is too few</label> | |
| <div class="exp">1 vs 2 implies GFB-S is "twice" GFB-U. One-hot creates neutral 0/1 columns.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Run a quality audit on one real work table: % missing per column, duplicate count, and every distinct spelling of one category.</li> | |
| <li>Design (on paper) 8 engineered features for monthly LPG demand. Mark each: available before prediction time? ✓/✗</li> | |
| <li>Ask Claude for a leakage review: paste your feature list + prediction moment, ask it to flag suspects.</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Which dataset at KNPC do you trust least, and is the cause missing values, inconsistent entry, or duplicates?</li> | |
| <li>What do you know about demand drivers that is written in NO database — and how could it become a column?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Scaling before splitting (statistics computed on test data = subtle leakage; pipelines prevent this).</li> | |
| <li>Dropping every row with any missing value and silently losing 40% of the data.</li> | |
| <li>Treating -999 / "N/A" sentinel codes as real numbers.</li> | |
| <li>Touching the test set repeatedly until results look good — that's just slow training on test.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Audit trio</div><code>df.info()</code> · <code>df.isna().sum()</code> · <code>df.duplicated().sum()</code></div> | |
| <div class="cell"><div class="k">Missing</div>median/mode fill + was_missing flag</div> | |
| <div class="cell"><div class="k">Categories</div>one-hot (<code>pd.get_dummies</code>) if unordered</div> | |
| <div class="cell"><div class="k">Scaling</div>KNN/SVM/NN: yes · trees: no</div> | |
| <div class="cell"><div class="k">Leakage test</div>"Known before prediction time?"</div> | |
| <div class="cell"><div class="k">Splits</div>70/15/15 · time series → split by time · CV=5</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">QUALITY AUDIT<button onclick="cp(this)">Copy</button></div><pre>Here is df.info() and df.head(20) from my dataset: [paste]. Write a data-quality audit: missing values per column with a recommended treatment and reasoning, duplicate check, type corrections, and category standardization. Explain every decision in simple English. Do NOT drop anything without stating what % of data is lost.</pre></div> | |
| <div class="prompt"><div class="ph">LEAKAGE REVIEW<button onclick="cp(this)">Copy</button></div><pre>I predict [target] at [exact moment of prediction]. My candidate features: [list]. For each feature, state whether its value is fully known BEFORE that moment. Flag any leakage risks, explain the mechanism, and suggest a safe replacement (e.g., lagged version).</pre></div> | |
| <div class="prompt"><div class="ph">FEATURE BRAINSTORM<button onclick="cp(this)">Copy</button></div><pre>Business problem: [describe]. Raw columns available: [list]. Acting as a senior feature engineer for industrial plant data, propose 15 engineered features grouped by: time-based, rolling statistics, domain flags (Kuwait context: Ramadan, Eid, summer peak), and interactions. For each: one-line rationale.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>scikit-learn User Guide: Preprocessing, Pipelines, Cross-validation, Common pitfalls & data leakage — scikit-learn.org.</li> | |
| <li>pandas documentation: Working with missing data — pandas.pydata.org.</li> | |
| <li>Kaufman et al. (2012). "Leakage in Data Mining." <em>ACM TKDD</em> — the standard leakage taxonomy.</li> | |
| <li>Google Developers, ML Crash Course: "Splitting Data" & "Data Preparation and Feature Engineering" guides.</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(3)">← Ch. 3</button><button class="btn done" onclick="done(4)">Mark complete → Ch. 5</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 5 ================= --> | |
| <article id="ch5" class="chx"> | |
| <div class="plate"><span class="tag">CH-05</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Exploratory Data Analysis</h2> | |
| <p class="ch-intro">EDA is the structured conversation you have with a dataset before trusting it with predictions. Skip it and every later step inherits your blindness.</p> | |
| <h3><span class="sn">5.1</span>The EDA protocol — same 6 steps, every dataset</h3> | |
| <table class="tb"> | |
| <tr><th>#</th><th>Step</th><th>Core question</th><th>Tools</th></tr> | |
| <tr><td>1</td><td>Shape & types</td><td>How many rows/columns? Right types?</td><td><code>df.shape</code>, <code>df.info()</code></td></tr> | |
| <tr><td>2</td><td>Summary statistics</td><td>Do min/max/mean pass the sanity check?</td><td><code>df.describe()</code></td></tr> | |
| <tr><td>3</td><td>Each variable alone</td><td>What does its distribution look like? Skewed? Two humps?</td><td>Histogram, boxplot, <code>value_counts()</code></td></tr> | |
| <tr><td>4</td><td>Pairs of variables</td><td>What moves with what?</td><td>Scatter plots, correlation heatmap</td></tr> | |
| <tr><td>5</td><td>Time behavior</td><td>Trend? Seasonality? Level shifts?</td><td>Line plots, rolling means</td></tr> | |
| <tr><td>6</td><td>The weird stuff</td><td>What surprised me, and is it error or signal?</td><td>Filtering, sorting, asking colleagues</td></tr> | |
| </table> | |
| <h3><span class="sn">5.2</span>Choosing the right chart</h3> | |
| <table class="tb"> | |
| <tr><th>You want to see…</th><th>Chart</th><th>Seaborn one-liner</th></tr> | |
| <tr><td>Distribution of one number</td><td>Histogram / KDE</td><td><code>sns.histplot(df, x="qty")</code></td></tr> | |
| <tr><td>Distribution across groups</td><td>Boxplot</td><td><code>sns.boxplot(df, x="plant", y="fill_time")</code></td></tr> | |
| <tr><td>Relationship of two numbers</td><td>Scatter</td><td><code>sns.scatterplot(df, x="temp", y="demand")</code></td></tr> | |
| <tr><td>Change over time</td><td>Line</td><td><code>sns.lineplot(df, x="date", y="qty")</code></td></tr> | |
| <tr><td>All correlations at once</td><td>Heatmap</td><td><code>sns.heatmap(df.corr(numeric_only=True), annot=True)</code></td></tr> | |
| <tr><td>Category counts</td><td>Bar</td><td><code>sns.countplot(df, x="stoppage_cause")</code></td></tr> | |
| </table> | |
| <div class="co warn"><div class="co-t">⚠ Correlation ≠ causation</div><p>Ice-cream sales correlate with drowning (both follow summer). Your energy use may correlate with a variable that merely shadows production volume. EDA finds <em>candidates</em> for relationships; domain reasoning — yours — decides which are real.</p></div> | |
| <h3><span class="sn">5.3</span>Thinking like a data scientist</h3> | |
| <p>The mindset, condensed to five habits:</p> | |
| <ol> | |
| <li><strong>Hypothesis first.</strong> Before plotting, write what you expect ("demand peaks in July"). Surprises are only visible against expectations.</li> | |
| <li><strong>Distrust aggregates.</strong> A healthy average can hide two sick groups (one line over-performing, one failing). Always disaggregate by plant, shift, season.</li> | |
| <li><strong>Chase anomalies to their source.</strong> The weird spike in March? That investigation usually produces the project's most valuable insight.</li> | |
| <li><strong>Quantify, don't adjective.</strong> Replace "demand is much higher in summer" with "July demand runs 38% above the annual mean."</li> | |
| <li><strong>End with decisions.</strong> An EDA that doesn't change what you do next was decoration.</li> | |
| </ol> | |
| <div class="co plant"><div class="co-t">▲ Deliverable habit</div><p>Every EDA ends in a one-page summary: 3 findings, 3 charts, 3 decisions. That page is what management reads — and it's the specification for the model you build next.</p></div> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>EDA is a fixed 6-step protocol, not artistic wandering: shape → stats → single → pairs → time → anomalies.</li> | |
| <li>One table maps every question to its chart; six chart types cover 95% of EDA.</li> | |
| <li>Correlation nominates suspects; your domain knowledge convicts.</li> | |
| <li>Averages hide; disaggregate by plant, line, shift, season.</li> | |
| <li>EDA output = findings that change decisions, captured in one page.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"EDA is optional — the model will figure it out"</summary><div class="dbody">Models amplify whatever the data contains, including its errors. EDA is where you catch the leakage column, the duplicate export, the broken sensor.</div></details> | |
| <details><summary>"More charts = better EDA"</summary><div class="dbody">Twenty aimless plots < five plots each answering a written hypothesis.</div></details> | |
| <details><summary>"A 0.9 correlation proves causation"</summary><div class="dbody">It proves co-movement. A third factor (season, production volume) may drive both.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>EDA on 3 years of stoppage logs before any downtime model — the cluster of causes usually surprises everyone.</li> | |
| <li>Seasonal decomposition of cylinder demand: quantify the Ramadan and summer effects you know qualitatively.</li> | |
| <li>Correlation heatmap of OPEX line items vs production volume — instant candidates for cost drivers.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="2"><div class="qt">1. To compare fill-time distributions between GFB-U and GFB-S, best chart:</div> | |
| <label><input type="radio" name="c5q1" value="0">Pie chart</label><label><input type="radio" name="c5q1" value="1">Line plot</label><label><input type="radio" name="c5q1" value="2">Boxplot by plant</label> | |
| <div class="exp">Distribution across groups = boxplot. Shows median, spread, and outliers per plant at once.</div></div> | |
| <div class="quiz-q" data-a="1"><div class="qt">2. df.describe() shows max fill_time = 9,600 minutes. First move:</div> | |
| <label><input type="radio" name="c5q2" value="0">Delete it</label><label><input type="radio" name="c5q2" value="1">Investigate the row — data-entry error or a real extreme event?</label><label><input type="radio" name="c5q2" value="2">Ignore, the model will handle it</label> | |
| <div class="exp">Anomalies are investigated first. Error → fix; real → possibly your most informative data point.</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">3. Energy cost correlates 0.85 with cylinder output. Correct statement:</div> | |
| <label><input type="radio" name="c5q3" value="0">They move together; whether output drives cost needs domain reasoning</label><label><input type="radio" name="c5q3" value="1">Output causes 85% of cost</label><label><input type="radio" name="c5q3" value="2">Reducing output cuts cost by 0.85%</label> | |
| <div class="exp">Correlation measures co-movement only, and its value is not a percentage of anything.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Write 5 hypotheses about your filling data BEFORE looking. Then run the 6-step protocol and score yourself.</li> | |
| <li>Build the correlation heatmap for one plant dataset; pick the strongest surprising pair and investigate it to the source.</li> | |
| <li>Produce the one-page output: 3 findings, 3 charts, 3 decisions. Show it to a colleague — do they reach the same decisions?</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Which "known fact" about plant operations have you never actually verified in data?</li> | |
| <li>When did an average last mislead your team, and what disaggregation would have caught it?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Plotting before cleaning — beautiful charts of broken data.</li> | |
| <li>Pie charts for anything with more than 3 categories.</li> | |
| <li>Correlation on categories encoded as arbitrary numbers.</li> | |
| <li>Never plotting the target variable itself over time — the single most informative chart.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Protocol</div>shape → describe → single vars → pairs → time → anomalies</div> | |
| <div class="cell"><div class="k">One number</div>histplot · groups: boxplot</div> | |
| <div class="cell"><div class="k">Two numbers</div>scatterplot · all: heatmap</div> | |
| <div class="cell"><div class="k">Over time</div>lineplot + rolling mean</div> | |
| <div class="cell"><div class="k">Golden rule</div>correlation ≠ causation</div> | |
| <div class="cell"><div class="k">Output</div>3 findings · 3 charts · 3 decisions</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">FULL EDA<button onclick="cp(this)">Copy</button></div><pre>Here is df.info() and df.describe() for my dataset about [context]: [paste]. Generate a complete EDA notebook following this protocol: 1) quality checks, 2) distribution of every numeric column, 3) counts of every category, 4) correlation heatmap, 5) target variable over time, 6) an "anomalies to investigate" list. Use seaborn. After the code, write the 3 most important questions I should ask about this data as the domain expert.</pre></div> | |
| <div class="prompt"><div class="ph">CHART CRITIC<button onclick="cp(this)">Copy</button></div><pre>I attach a chart from my analysis. Critique it: is this the right chart type for the question "[question]"? What could mislead a manager viewing it? Rewrite the code for a clearer version and suggest one alternative view of the same data.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>Tukey, J. W. (1977). <em>Exploratory Data Analysis.</em> Addison-Wesley — the founding text of the field.</li> | |
| <li>seaborn official tutorial — seaborn.pydata.org/tutorial.</li> | |
| <li>pandas documentation: Descriptive statistics, GroupBy — pandas.pydata.org.</li> | |
| <li>NIST/SEMATECH e-Handbook of Statistical Methods, ch. 1 (EDA) — itl.nist.gov.</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(4)">← Ch. 4</button><button class="btn done" onclick="done(5)">Mark complete → Ch. 6</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 6 ================= --> | |
| <article id="ch6" class="chx"> | |
| <div class="plate"><span class="tag">CH-06</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Statistics for Machine Learning</h2> | |
| <p class="ch-intro">Only the statistics you'll actually use — as intuition, not derivation. Every concept here earns its place by appearing later in evaluation, tuning, or interpretation.</p> | |
| <h3><span class="sn">6.1</span>Spread: variance & standard deviation</h3> | |
| <p><strong>Variance</strong> measures how far values scatter from their mean; <strong>standard deviation (σ)</strong> is its square root, expressed in the original units — which is why you'll quote σ, not variance.</p> | |
| <div class="co analogy"><div class="co-t">⬡ Plant Analogy</div><p>Two filling lines both average 14.9 kg per cylinder. Line A: σ = 0.05 kg (tight, consistent). Line B: σ = 0.4 kg (all over the place). Same mean, completely different quality story — the mean without σ is half a report.</p></div> | |
| <h3><span class="sn">6.2</span>Distributions — the shapes data takes</h3> | |
| <figure> | |
| <svg viewBox="0 0 720 210" width="100%" role="img" aria-label="Normal distribution with sigma bands"> | |
| <g font-family="IBM Plex Sans" font-size="11.5"> | |
| <path d="M 60 180 C 160 180 190 40 300 40 C 410 40 440 180 540 180" fill="none" stroke="#16262E" stroke-width="2.5"/> | |
| <path d="M 190 180 C 230 100 260 45 300 45 C 340 45 370 100 410 180 Z" fill="#E3F1EB" opacity="0.8"/> | |
| <line x1="300" y1="40" x2="300" y2="185" stroke="#E8551A" stroke-width="2" stroke-dasharray="5 4"/> | |
| <text x="300" y="200" text-anchor="middle" font-family="IBM Plex Mono" fill="#E8551A" font-weight="600">mean</text> | |
| <text x="300" y="130" text-anchor="middle" fill="#1E7A5A" font-weight="700">±1σ ≈ 68%</text> | |
| <line x1="190" y1="180" x2="190" y2="170" stroke="#33566B" stroke-width="2"/><line x1="410" y1="180" x2="410" y2="170" stroke="#33566B" stroke-width="2"/> | |
| <line x1="120" y1="180" x2="120" y2="170" stroke="#33566B" stroke-width="2"/><line x1="480" y1="180" x2="480" y2="170" stroke="#33566B" stroke-width="2"/> | |
| <text x="120" y="165" text-anchor="middle" font-family="IBM Plex Mono" fill="#33566B">-2σ</text><text x="480" y="165" text-anchor="middle" font-family="IBM Plex Mono" fill="#33566B">+2σ</text> | |
| <text x="595" y="70" fill="#16262E" font-weight="700">Normal (bell) curve:</text> | |
| <text x="595" y="90" fill="#33566B">±1σ → ~68%</text> | |
| <text x="595" y="107" fill="#33566B">±2σ → ~95%</text> | |
| <text x="595" y="124" fill="#33566B">±3σ → ~99.7%</text> | |
| <text x="595" y="150" fill="#E8551A" font-size="10.5">basis of control charts</text> | |
| <text x="595" y="164" fill="#E8551A" font-size="10.5">& anomaly thresholds</text> | |
| </g></svg> | |
| <figcaption><span class="fn">FIG 6.1</span> — The normal distribution and the 68–95–99.7 rule. A reading beyond ±3σ is a 1-in-370 event — exactly the logic of SPC control charts and simple anomaly detectors.</figcaption> | |
| </figure> | |
| <p>You'll meet three shapes constantly: <strong>normal</strong> (measurement noise, fill weights), <strong>skewed</strong> (repair costs, downtime durations — many small, few huge; the mean gets dragged, so report the <em>median</em>), and <strong>bimodal</strong> (two humps = two hidden populations, like two shifts or two machines mixed in one column — split them).</p> | |
| <h3><span class="sn">6.3</span>Probability & Bayes in working form</h3> | |
| <p>Probability quantifies uncertainty from 0 to 1. The one theorem worth internalizing is <strong>Bayes' rule</strong>: update your belief when evidence arrives. Its practical sting is the <em>base-rate trap</em>:</p> | |
| <div class="co warn"><div class="co-t">⚠ The base-rate trap (real numbers)</div><p>A defect detector is 95% accurate. Defects occur in 1% of cylinders. A cylinder is flagged — probability it's truly defective? Not 95%. Out of 10,000 cylinders: 100 defective → 95 true flags; 9,900 good → 495 false flags. Truly defective among flags = 95/(95+495) ≈ <strong>16%</strong>. When the event is rare, even good models produce mostly false alarms. This single calculation explains why Chapter 9 refuses to accept "accuracy" for rare events, and why precision/recall exist.</p></div> | |
| <h3><span class="sn">6.4</span>Hypothesis tests & confidence intervals</h3> | |
| <ul> | |
| <li><strong>Hypothesis test</strong> — "is this difference real or luck?" You assume no difference (null hypothesis), then compute how surprising your data would be if that were true (the <strong>p-value</strong>). Convention: p < 0.05 → the difference is unlikely to be luck. Use it when comparing line A vs line B, before/after a modification.</li> | |
| <li><strong>Confidence interval (CI)</strong> — a range, not a point: "mean fill time = 42.3 s, 95% CI [41.8, 42.8]". Rule of thumb: if two CIs don't overlap, the difference is real. Quote CIs in reports; single numbers overstate certainty.</li> | |
| <li><strong>Correlation vs covariance</strong> — both measure co-movement; correlation is covariance standardized to [-1, +1] so it's comparable across variable pairs. You'll use correlation; covariance lives inside the math.</li> | |
| </ul> | |
| <h3><span class="sn">6.5</span>The math intuitions (no equations required)</h3> | |
| <table class="tb"> | |
| <tr><th>Field</th><th>The one intuition you need</th><th>Where it surfaces</th></tr> | |
| <tr><td><strong>Linear algebra</strong></td><td>Your data table IS a matrix; each row is a point in a space with one dimension per feature. Models draw boundaries or surfaces in that space</td><td>Why features must be numeric; embeddings (Ch. 8, 12); PCA</td></tr> | |
| <tr><td><strong>Calculus</strong></td><td>The derivative is a slope — it tells the model which direction reduces error</td><td>"Gradient" in gradient descent / gradient boosting</td></tr> | |
| <tr><td><strong>Optimization</strong></td><td>Training = walking downhill on an error landscape, step by step, until a low point. The step size is the "learning rate"</td><td>Why training iterates; why it can stall in a local dip; hyperparameters (Ch. 10)</td></tr> | |
| </table> | |
| <div class="co analogy"><div class="co-t">⬡ Gradient descent, plant edition</div><p>You're on dunes at night with a flashlight, seeking the lowest point. You feel the slope under your feet (the gradient) and step downhill (learning rate = stride length). Too-long strides overshoot the valley; too-short takes forever. That is — genuinely — how every neural network trains.</p></div> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>Always report spread (σ) with the mean; for skewed data (costs, durations) prefer the median.</li> | |
| <li>68–95–99.7: the ±σ rule powers control charts and threshold-based anomaly detection.</li> | |
| <li>Bayes' base-rate trap: with rare events, most alarms are false even from good models — the reason precision/recall exist.</li> | |
| <li>p-values test "real or luck"; confidence intervals replace false-precision single numbers.</li> | |
| <li>Training = gradient descent = walking downhill on an error landscape. That intuition unlocks Chapters 8 and 10.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"p < 0.05 proves my hypothesis"</summary><div class="dbody">It says the data would be surprising if nothing were going on. With enough comparisons, 1-in-20 flukes appear by design — beware testing many things and reporting the one that "worked".</div></details> | |
| <details><summary>"95% accurate means 95% of alarms are true"</summary><div class="dbody">Only if the event is common. For 1% events, see §6.3 — it can be 16%.</div></details> | |
| <details><summary>"I need to master the math first"</summary><div class="dbody">Practitioners need the three intuitions in §6.5. Depth can come later, on demand, per topic.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>±3σ control limits on daily energy per cylinder — a statistically honest anomaly alarm before any ML.</li> | |
| <li>A proper hypothesis test on "did the carousel upgrade actually reduce fill time?" instead of eyeballing two averages.</li> | |
| <li>Confidence intervals on your demand forecast in the annual plan — ranges managers can trust.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="1"><div class="qt">1. Downtime durations: many short, a few multi-day monsters. Report the typical duration using:</div> | |
| <label><input type="radio" name="c6q1" value="0">Mean</label><label><input type="radio" name="c6q1" value="1">Median</label><label><input type="radio" name="c6q1" value="2">Maximum</label> | |
| <div class="exp">Skewed data drags the mean toward the monsters; the median resists.</div></div> | |
| <div class="quiz-q" data-a="2"><div class="qt">2. A rare defect (0.5%) is flagged by a 95%-accurate model. The flag is:</div> | |
| <label><input type="radio" name="c6q2" value="0">95% certainly a defect</label><label><input type="radio" name="c6q2" value="1">Certain — the model is accurate</label><label><input type="radio" name="c6q2" value="2">Probably a false alarm — the base-rate trap</label> | |
| <div class="exp">Rare event + imperfect model = false flags outnumber true ones. Compute it Bayes-style.</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">3. In gradient descent, the "learning rate" is:</div> | |
| <label><input type="radio" name="c6q3" value="0">The size of each downhill step on the error landscape</label><label><input type="radio" name="c6q3" value="1">How fast the human learns ML</label><label><input type="radio" name="c6q3" value="2">The GPU clock speed</label> | |
| <div class="exp">Stride length in the night-dunes analogy — too big overshoots, too small crawls.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Compute mean, median, σ for one real KPI. Which two numbers tell the honest story?</li> | |
| <li>Redo the base-rate calculation of §6.3 with your own numbers: pick a rare plant event, assume a 90%-accurate detector, compute the true-alarm rate.</li> | |
| <li>Ask Claude to run a t-test comparing a KPI before/after a real change at the plant, and to explain the p-value in one sentence a manager accepts.</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Which number in your monthly report is presented with false precision and deserves a confidence interval?</li> | |
| <li>Where has a "difference" between two periods been celebrated that a hypothesis test might expose as noise?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Comparing two averages without asking whether the gap exceeds normal variation.</li> | |
| <li>Using the mean on skewed cost/duration data.</li> | |
| <li>Testing 20 relationships and reporting the one with p < 0.05.</li> | |
| <li>Reading correlation strength as effect size or causation.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Spread</div>σ with every mean; median for skew</div> | |
| <div class="cell"><div class="k">Sigma rule</div>68 / 95 / 99.7 at ±1/2/3σ</div> | |
| <div class="cell"><div class="k">Bayes trap</div>rare event → most alarms false</div> | |
| <div class="cell"><div class="k">p-value</div><0.05 ⇒ unlikely to be luck (not proof)</div> | |
| <div class="cell"><div class="k">CI habit</div>report ranges, not points</div> | |
| <div class="cell"><div class="k">Training</div>= gradient descent downhill; step = learning rate</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">STAT TEST RUNNER<button onclick="cp(this)">Copy</button></div><pre>I have two groups of measurements: [context]. Data: [paste or describe columns]. Choose the appropriate statistical test (explain WHY that one), run it in Python with scipy, and interpret the result in two sentences for a non-statistician manager, including a clear statement of what it does NOT prove.</pre></div> | |
| <div class="prompt"><div class="ph">BASE-RATE CHECK<button onclick="cp(this)">Copy</button></div><pre>Event base rate: [X]% . Detector accuracy: assume [Y]% sensitivity and [Z]% specificity (suggest defaults if unknown). Build the 10,000-case table showing true/false alarms, compute the probability a flagged case is real, and state the operational implication for alarm handling.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>OpenIntro Statistics (open textbook, openintro.org) — distributions, tests, CIs.</li> | |
| <li>NIST/SEMATECH e-Handbook — control charts & the 3σ convention.</li> | |
| <li>SciPy stats documentation — docs.scipy.org (tests used in exercises).</li> | |
| <li>3Blue1Brown, "Essence of Linear Algebra" & "Essence of Calculus" video series — the visual intuitions of §6.5.</li> | |
| <li>Ng, A., Machine Learning Specialization (DeepLearning.AI) — gradient descent framing.</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(5)">← Ch. 5</button><button class="btn done" onclick="done(6)">Mark complete → Ch. 7</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 7 ================= --> | |
| <article id="ch7" class="chx"> | |
| <div class="plate"><span class="tag">CH-07</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Machine Learning Algorithms</h2> | |
| <p class="ch-intro">The full toolbox, organized so you never guess. One selection flowchart, then a register of every major algorithm with its honest strengths, weaknesses, and plant use cases.</p> | |
| <h3><span class="sn">7.1</span>The selection flowchart</h3> | |
| <figure> | |
| <svg viewBox="0 0 720 330" width="100%" role="img" aria-label="Algorithm selection decision tree"> | |
| <defs><marker id="ar7" markerWidth="8" markerHeight="8" refX="6" refY="4" orient="auto"><path d="M0,0 L8,4 L0,8 z" fill="#33566B"/></marker></defs> | |
| <g font-family="IBM Plex Sans" font-size="11.5"> | |
| <rect x="270" y="10" width="180" height="40" rx="8" fill="#16262E"/><text x="360" y="34" text-anchor="middle" fill="#fff" font-weight="700">Do you have labels?</text> | |
| <line x1="310" y1="50" x2="180" y2="85" stroke="#33566B" stroke-width="2" marker-end="url(#ar7)"/><text x="215" y="65" font-family="IBM Plex Mono" font-size="10" fill="#1E7A5A">YES</text> | |
| <line x1="410" y1="50" x2="545" y2="85" stroke="#33566B" stroke-width="2" marker-end="url(#ar7)"/><text x="500" y="65" font-family="IBM Plex Mono" font-size="10" fill="#B3541E">NO</text> | |
| <rect x="80" y="90" width="200" height="40" rx="8" fill="#E3F1EB" stroke="#1E7A5A"/><text x="180" y="114" text-anchor="middle" font-weight="700" fill="#1E7A5A">Predicting number or category?</text> | |
| <rect x="450" y="90" width="200" height="40" rx="8" fill="#FCEFE3" stroke="#B3541E"/><text x="550" y="114" text-anchor="middle" font-weight="700" fill="#B3541E">Groups or oddballs?</text> | |
| <line x1="130" y1="130" x2="90" y2="165" stroke="#33566B" stroke-width="2" marker-end="url(#ar7)"/><text x="88" y="150" font-family="IBM Plex Mono" font-size="10" fill="#33566B">number</text> | |
| <line x1="230" y1="130" x2="270" y2="165" stroke="#33566B" stroke-width="2" marker-end="url(#ar7)"/><text x="262" y="150" font-family="IBM Plex Mono" font-size="10" fill="#33566B">category</text> | |
| <line x1="500" y1="130" x2="460" y2="165" stroke="#33566B" stroke-width="2" marker-end="url(#ar7)"/><text x="450" y="150" font-family="IBM Plex Mono" font-size="10" fill="#33566B">groups</text> | |
| <line x1="600" y1="130" x2="640" y2="165" stroke="#33566B" stroke-width="2" marker-end="url(#ar7)"/><text x="632" y="150" font-family="IBM Plex Mono" font-size="10" fill="#33566B">oddballs</text> | |
| <rect x="15" y="170" width="150" height="66" rx="8" fill="#fff" stroke="#1E7A5A" stroke-width="2"/><text x="90" y="190" text-anchor="middle" font-weight="700">REGRESSION</text><text x="90" y="207" text-anchor="middle" font-size="10.5">Linear → RF →</text><text x="90" y="221" text-anchor="middle" font-size="10.5">Gradient Boosting</text> | |
| <rect x="195" y="170" width="150" height="66" rx="8" fill="#fff" stroke="#1E7A5A" stroke-width="2"/><text x="270" y="190" text-anchor="middle" font-weight="700">CLASSIFICATION</text><text x="270" y="207" text-anchor="middle" font-size="10.5">Logistic → RF →</text><text x="270" y="221" text-anchor="middle" font-size="10.5">Gradient Boosting</text> | |
| <rect x="385" y="170" width="150" height="66" rx="8" fill="#fff" stroke="#B3541E" stroke-width="2"/><text x="460" y="190" text-anchor="middle" font-weight="700">CLUSTERING</text><text x="460" y="207" text-anchor="middle" font-size="10.5">K-Means →</text><text x="460" y="221" text-anchor="middle" font-size="10.5">DBSCAN</text> | |
| <rect x="565" y="170" width="150" height="66" rx="8" fill="#fff" stroke="#B3541E" stroke-width="2"/><text x="640" y="190" text-anchor="middle" font-weight="700">ANOMALY</text><text x="640" y="207" text-anchor="middle" font-size="10.5">3σ rule →</text><text x="640" y="221" text-anchor="middle" font-size="10.5">Isolation Forest</text> | |
| <rect x="15" y="262" width="695" height="52" rx="9" fill="#F8FAFB" stroke="#C9D3D8"/> | |
| <text x="362" y="283" text-anchor="middle" fill="#16262E" font-weight="700">Special data → special families:</text> | |
| <text x="362" y="301" text-anchor="middle" fill="#33566B" font-size="11">values ordered in time → TIME SERIES · too many columns → DIMENSIONALITY REDUCTION (PCA) · "what should I suggest?" → RECOMMENDERS · "what order?" → RANKING</text> | |
| </g></svg> | |
| <figcaption><span class="fn">FIG 7.1</span> — The arrows inside each box are an escalation ladder: start simple, escalate only when the simple model underperforms the baseline meaningfully.</figcaption> | |
| </figure> | |
| <h3><span class="sn">7.2</span>The algorithm register — supervised</h3> | |
| <table class="tb"> | |
| <tr><th>Algorithm</th><th>Idea in one line</th><th>Strengths</th><th>Weaknesses</th><th>Interpretability / Speed</th><th>Plant use</th></tr> | |
| <tr><td><strong>Linear Regression</strong></td><td>Best straight-line fit through the data</td><td>Instant, fully explainable coefficients ("each °C adds 12 cylinders")</td><td>Only straight-line relationships</td><td><span class="pill g">High / Fast</span></td><td>First model for demand vs drivers; the explainability benchmark</td></tr> | |
| <tr><td><strong>Logistic Regression</strong></td><td>Linear model squeezed into a 0–1 probability</td><td>Calibrated probabilities, explainable, robust</td><td>Linear boundaries only</td><td><span class="pill g">High / Fast</span></td><td>Probability a cylinder fails QC; baseline for any yes/no</td></tr> | |
| <tr><td><strong>Decision Tree</strong></td><td>Learned flowchart of if/then splits</td><td>Reads like a procedure; handles mixed data; no scaling needed</td><td>A single tree overfits easily</td><td><span class="pill g">High / Fast</span></td><td>Communicating rules to operators; root-cause style views</td></tr> | |
| <tr><td><strong>Random Forest</strong></td><td>Hundreds of varied trees voting</td><td>Strong default accuracy, hard to break, feature importances</td><td>Slower; explains less than one tree</td><td><span class="pill o">Medium / Medium</span></td><td>The workhorse: downtime prediction, demand, failure classification</td></tr> | |
| <tr><td><strong>Gradient Boosting (XGBoost / LightGBM)</strong></td><td>Trees built one at a time, each fixing the last one's errors</td><td>Usually the top accuracy on tabular data; wins most Kaggle tables</td><td>Sensitive to tuning; easier to overfit</td><td><span class="pill o">Medium / Medium</span></td><td>Squeezing the last accuracy from forecasts once RF plateaus</td></tr> | |
| <tr><td><strong>SVM</strong></td><td>The widest possible margin between classes</td><td>Strong in high-dimensional, small-sample settings</td><td>Slow on large data; needs scaling; opaque</td><td><span class="pill r">Low / Slow</span></td><td>Niche today — tree ensembles usually win on plant tabular data</td></tr> | |
| <tr><td><strong>KNN</strong></td><td>Copy the answer of the k most similar past cases</td><td>Zero training; intuitive ("cases like this one")</td><td>Slow predictions; needs scaling; struggles with many features</td><td><span class="pill o">Medium / Fast-train, slow-predict</span></td><td>"Find the 5 most similar historical stoppages" — great as a lookup tool</td></tr> | |
| <tr><td><strong>Naive Bayes</strong></td><td>Bayes' rule assuming features are independent</td><td>Extremely fast; shines on text/word counts</td><td>The independence assumption is usually false</td><td><span class="pill g">High / Fast</span></td><td>Quick classifier for maintenance-log text categories</td></tr> | |
| </table> | |
| <h3><span class="sn">7.3</span>The register — unsupervised & special families</h3> | |
| <table class="tb"> | |
| <tr><th>Algorithm / family</th><th>Idea</th><th>Watch out</th><th>Plant use</th></tr> | |
| <tr><td><strong>K-Means</strong> (clustering)</td><td>Split data into K round-ish groups around centers</td><td>You choose K (use the elbow method); needs scaling; assumes blob shapes</td><td>Segmenting stoppage events, customer ordering patterns, operating regimes</td></tr> | |
| <tr><td><strong>DBSCAN</strong> (clustering)</td><td>Groups = dense regions; sparse points = noise</td><td>Two sensitive parameters; struggles with mixed densities</td><td>Finds odd-shaped clusters AND flags outliers in one pass</td></tr> | |
| <tr><td><strong>PCA</strong> (dimensionality reduction)</td><td>Compress many correlated columns into a few "summary directions"</td><td>Components lose physical meaning; scale first</td><td>Compressing 40 correlated sensor channels before modeling or plotting</td></tr> | |
| <tr><td><strong>Isolation Forest</strong> (anomaly)</td><td>Anomalies are easier to isolate with random splits — few cuts needed = suspicious</td><td>Set expected contamination %; validate flags with experts</td><td>Multivariate anomaly detection on energy + production + downtime jointly</td></tr> | |
| <tr><td><strong>Time series: ARIMA / SARIMA / Prophet / gradient boosting with lags</strong></td><td>Model trend + seasonality + autocorrelation of a value over time</td><td>NEVER random-split; watch for regime changes (drift)</td><td>The LPG demand forecast — your flagship use case. Prophet handles Ramadan-style moving holidays via custom events</td></tr> | |
| <tr><td><strong>Recommenders</strong></td><td>"Users like you also…" (collaborative) or "items similar to this" (content-based)</td><td>Cold start: new users/items have no history</td><td>Suggesting relevant spec documents to engineers; spare-part co-occurrence</td></tr> | |
| <tr><td><strong>Ranking</strong></td><td>Learn the best ORDER of items, not their values</td><td>Needs preference/relevance data</td><td>Ordering maintenance backlog by urgency; prioritizing CAPEX candidates</td></tr> | |
| </table> | |
| <div class="co plant"><div class="co-t">▲ The escalation ladder (memorize this)</div><p>For any tabular problem: <strong>Baseline → Linear/Logistic → Random Forest → Gradient Boosting.</strong> Stop at the first rung whose improvement over the previous is too small to matter operationally. Deep learning enters only for images, audio, and free text — Chapter 8.</p></div> | |
| <h3><span class="sn">7.4</span>Decision boundaries — seeing how models think</h3> | |
| <figure> | |
| <svg viewBox="0 0 720 220" width="100%" role="img" aria-label="Decision boundaries comparison"> | |
| <g font-family="IBM Plex Sans" font-size="11"> | |
| <rect x="20" y="20" width="200" height="150" rx="8" fill="#fff" stroke="#C9D3D8"/> | |
| <line x1="45" y1="150" x2="195" y2="45" stroke="#E8551A" stroke-width="2.5"/> | |
| <circle cx="70" cy="60" r="5" fill="#1E7A5A"/><circle cx="95" cy="80" r="5" fill="#1E7A5A"/><circle cx="60" cy="105" r="5" fill="#1E7A5A"/><circle cx="110" cy="55" r="5" fill="#1E7A5A"/> | |
| <circle cx="160" cy="120" r="5" fill="#2C5F8A"/><circle cx="180" cy="95" r="5" fill="#2C5F8A"/><circle cx="140" cy="140" r="5" fill="#2C5F8A"/><circle cx="190" cy="140" r="5" fill="#2C5F8A"/> | |
| <text x="120" y="192" text-anchor="middle" font-weight="700">LOGISTIC: one straight cut</text> | |
| <rect x="260" y="20" width="200" height="150" rx="8" fill="#fff" stroke="#C9D3D8"/> | |
| <path d="M 285 150 L 285 110 L 340 110 L 340 60 L 395 60 L 395 130 L 435 130" fill="none" stroke="#E8551A" stroke-width="2.5"/> | |
| <circle cx="305" cy="70" r="5" fill="#1E7A5A"/><circle cx="320" cy="90" r="5" fill="#1E7A5A"/><circle cx="300" cy="130" r="5" fill="#1E7A5A"/><circle cx="370" cy="45" r="5" fill="#1E7A5A"/> | |
| <circle cx="420" cy="90" r="5" fill="#2C5F8A"/><circle cx="380" cy="150" r="5" fill="#2C5F8A"/><circle cx="430" cy="50" r="5" fill="#2C5F8A"/><circle cx="350" cy="140" r="5" fill="#2C5F8A"/> | |
| <text x="360" y="192" text-anchor="middle" font-weight="700">TREE: axis-aligned steps</text> | |
| <rect x="500" y="20" width="200" height="150" rx="8" fill="#fff" stroke="#C9D3D8"/> | |
| <path d="M 520 140 C 560 60, 600 160, 640 70 C 660 40, 680 90, 690 60" fill="none" stroke="#E8551A" stroke-width="2.5"/> | |
| <circle cx="545" cy="60" r="5" fill="#1E7A5A"/><circle cx="590" cy="100" r="5" fill="#1E7A5A"/><circle cx="530" cy="110" r="5" fill="#1E7A5A"/><circle cx="640" cy="40" r="5" fill="#1E7A5A"/> | |
| <circle cx="620" cy="130" r="5" fill="#2C5F8A"/><circle cx="660" cy="100" r="5" fill="#2C5F8A"/><circle cx="570" cy="150" r="5" fill="#2C5F8A"/><circle cx="680" cy="140" r="5" fill="#2C5F8A"/> | |
| <text x="600" y="192" text-anchor="middle" font-weight="700">ENSEMBLE/NN: flexible curves</text> | |
| <text x="360" y="215" text-anchor="middle" font-size="10.5" fill="#33566B">More flexible boundary = more power = more risk of memorizing noise (overfitting)</text> | |
| </g></svg> | |
| <figcaption><span class="fn">FIG 7.2</span> — Same data, three ways of cutting it. Flexibility is bought with overfitting risk — the central trade-off Chapter 10 manages.</figcaption> | |
| </figure> | |
| <p><strong>Overfitting</strong> = memorizing training noise instead of learning the pattern (great in training, poor on new data). <strong>Underfitting</strong> = model too simple to capture the real pattern (poor everywhere). The escalation ladder walks you from underfit toward the sweet spot, and validation data tells you when to stop.</p> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>Two questions place any problem: labels? number-or-category? Then FIG 7.1 hands you the family.</li> | |
| <li>Escalation ladder for tables: Baseline → Linear → Random Forest → Gradient Boosting. Stop when gains stop mattering.</li> | |
| <li>Tree ensembles dominate industrial tabular data; SVM/KNN are niche; deep learning is for images/text/audio.</li> | |
| <li>Time series is its own discipline: time-based splits, seasonality, moving holidays (Ramadan).</li> | |
| <li>Flexibility vs overfitting is THE trade-off; validation data is the referee.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"There's a single best algorithm"</summary><div class="dbody">Performance depends on the data (formalized as the "no free lunch" principle). That's why practitioners compare 2–3 candidates — cheap to do with sklearn's uniform API.</div></details> | |
| <details><summary>"Neural networks beat everything"</summary><div class="dbody">On tabular business data, gradient-boosted trees routinely match or beat deep learning at a fraction of the cost — a repeated benchmark finding.</div></details> | |
| <details><summary>"K-Means told me there are 4 clusters, so 4 groups exist"</summary><div class="dbody">K-Means produces K clusters because you asked for K. Whether they're meaningful requires the elbow method + your domain review.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>Random Forest on downtime events: predict which line stops next week + feature importances showing why.</li> | |
| <li>Prophet with Ramadan/Eid custom events for the monthly cylinder demand forecast feeding your Power BI plan.</li> | |
| <li>Isolation Forest across energy + volume + downtime jointly — anomalies invisible in any single metric.</li> | |
| <li>KNN "similar past incidents" lookup for new stoppages — instant institutional memory.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="1"><div class="qt">1. Labeled data, predicting monthly demand (a number), first proper model after the baseline:</div> | |
| <label><input type="radio" name="c7q1" value="0">K-Means</label><label><input type="radio" name="c7q1" value="1">Linear Regression</label><label><input type="radio" name="c7q1" value="2">DBSCAN</label> | |
| <div class="exp">Supervised + number = regression; the ladder starts at linear.</div></div> | |
| <div class="quiz-q" data-a="2"><div class="qt">2. Model: 99% on training, 71% on validation. Diagnosis:</div> | |
| <label><input type="radio" name="c7q2" value="0">Underfitting</label><label><input type="radio" name="c7q2" value="1">Perfect fit</label><label><input type="radio" name="c7q2" value="2">Overfitting — memorized training noise</label> | |
| <div class="exp">Large train-validation gap = memorization. Simplify, regularize, or get more data (Ch. 10).</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">3. No labels, and you want to flag strange combinations of sensor readings:</div> | |
| <label><input type="radio" name="c7q3" value="0">Isolation Forest</label><label><input type="radio" name="c7q3" value="1">Logistic Regression</label><label><input type="radio" name="c7q3" value="2">Gradient Boosting</label> | |
| <div class="exp">No labels + oddballs = anomaly detection. Isolation Forest is the standard multivariate choice.</div></div> | |
| <div class="quiz-q" data-a="1"><div class="qt">4. Why does Random Forest usually beat a single decision tree?</div> | |
| <label><input type="radio" name="c7q4" value="0">It trains faster</label><label><input type="radio" name="c7q4" value="1">Many varied trees voting average out each tree's overfitting</label><label><input type="radio" name="c7q4" value="2">It uses deep learning internally</label> | |
| <div class="exp">Ensemble wisdom: individual errors cancel when the trees are diverse.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Route 5 real plant problems through FIG 7.1 on paper; name the family and the first algorithm for each.</li> | |
| <li>In Colab, train LinearRegression and RandomForestRegressor on the same small dataset; compare validation errors and the improvement per rung.</li> | |
| <li>Ask Claude to plot Random Forest feature importances for one model — do the top features match your engineering intuition?</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Which decision at the plant would benefit more from an explainable model (linear/tree) than a slightly more accurate black box?</li> | |
| <li>Where would a "similar past cases" KNN lookup preserve knowledge that retires when senior operators do?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Starting with XGBoost before establishing the baseline and linear rungs — you can't measure what complexity bought.</li> | |
| <li>Forgetting to scale for KNN/SVM/K-Means (distance-based), or wasting effort scaling for trees.</li> | |
| <li>Random-splitting time series (Ch. 4's cardinal sin, repeated because it's that common).</li> | |
| <li>Reporting training accuracy in a management deck.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Number + labels</div>Linear → RF → GBM</div> | |
| <div class="cell"><div class="k">Category + labels</div>Logistic → RF → GBM</div> | |
| <div class="cell"><div class="k">No labels, groups</div>K-Means (elbow for K) → DBSCAN</div> | |
| <div class="cell"><div class="k">No labels, oddballs</div>3σ → Isolation Forest</div> | |
| <div class="cell"><div class="k">Over time</div>Prophet / SARIMA / GBM+lags, split by time</div> | |
| <div class="cell"><div class="k">Diagnosis</div>train≫validation = overfit · both poor = underfit</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">ALGORITHM SHOOTOUT<button onclick="cp(this)">Copy</button></div><pre>Dataset: [describe columns, rows, target]. Problem type: [regression/classification]. Write Colab code that: 1) computes a naive baseline, 2) trains Linear/Logistic, Random Forest, and Gradient Boosting with 5-fold cross-validation, 3) presents results in one comparison table including the baseline, 4) states which model to choose and whether the accuracy gain justifies the interpretability loss.</pre></div> | |
| <div class="prompt"><div class="ph">EXPLAIN THE WINNER<button onclick="cp(this)">Copy</button></div><pre>For the winning model above, produce: feature importance chart, 3 example predictions explained in plain English, and a one-paragraph summary I can put in a management report describing what drives the predictions — no jargon.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>scikit-learn User Guide & algorithm cheat-sheet — scikit-learn.org/stable/machine_learning_map.</li> | |
| <li>Breiman, L. (2001). "Random Forests." <em>Machine Learning</em> 45.</li> | |
| <li>Chen & Guestrin (2016). "XGBoost: A Scalable Tree Boosting System." <em>KDD</em>.</li> | |
| <li>Taylor & Letham (2018). "Forecasting at Scale." <em>The American Statistician</em> (Prophet).</li> | |
| <li>Liu et al. (2008). "Isolation Forest." <em>ICDM</em>.</li> | |
| <li>Wolpert (1996). "The Lack of A Priori Distinctions Between Learning Algorithms." <em>Neural Computation</em> (no free lunch).</li> | |
| <li>Grinsztajn et al. (2022). "Why do tree-based models still outperform deep learning on tabular data?" <em>NeurIPS</em>.</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(6)">← Ch. 6</button><button class="btn done" onclick="done(7)">Mark complete → Ch. 8</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 8 ================= --> | |
| <article id="ch8" class="chx"> | |
| <div class="plate"><span class="tag">CH-08</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Deep Learning</h2> | |
| <p class="ch-intro">Deep learning is what unlocked images, speech, and language for machines — and it's the engine inside Claude. You need working intuition, not implementation.</p> | |
| <h3><span class="sn">8.1</span>The neuron and the network</h3> | |
| <figure> | |
| <svg viewBox="0 0 720 240" width="100%" role="img" aria-label="Neural network layers"> | |
| <g font-family="IBM Plex Sans" font-size="11"> | |
| <text x="70" y="25" text-anchor="middle" font-weight="700" fill="#2C5F8A">INPUT LAYER</text> | |
| <text x="70" y="40" text-anchor="middle" font-size="10" fill="#33566B">(your features)</text> | |
| <circle cx="70" cy="70" r="16" fill="#E7F0F7" stroke="#2C5F8A" stroke-width="2"/><text x="70" y="74" text-anchor="middle" font-size="9">temp</text> | |
| <circle cx="70" cy="120" r="16" fill="#E7F0F7" stroke="#2C5F8A" stroke-width="2"/><text x="70" y="124" text-anchor="middle" font-size="9">vibr</text> | |
| <circle cx="70" cy="170" r="16" fill="#E7F0F7" stroke="#2C5F8A" stroke-width="2"/><text x="70" y="174" text-anchor="middle" font-size="9">hours</text> | |
| <text x="300" y="25" text-anchor="middle" font-weight="700" fill="#1E7A5A">HIDDEN LAYERS ("deep")</text> | |
| <text x="300" y="40" text-anchor="middle" font-size="10" fill="#33566B">learn combinations of combinations</text> | |
| <g fill="#E3F1EB" stroke="#1E7A5A" stroke-width="2"> | |
| <circle cx="230" cy="60" r="14"/><circle cx="230" cy="110" r="14"/><circle cx="230" cy="160" r="14"/><circle cx="230" cy="205" r="14"/> | |
| <circle cx="370" cy="80" r="14"/><circle cx="370" cy="135" r="14"/><circle cx="370" cy="185" r="14"/> | |
| </g> | |
| <g stroke="#C9D3D8" stroke-width="1"> | |
| <line x1="86" y1="70" x2="216" y2="60"/><line x1="86" y1="70" x2="216" y2="110"/><line x1="86" y1="120" x2="216" y2="110"/><line x1="86" y1="120" x2="216" y2="160"/><line x1="86" y1="170" x2="216" y2="160"/><line x1="86" y1="170" x2="216" y2="205"/><line x1="86" y1="70" x2="216" y2="160"/><line x1="86" y1="170" x2="216" y2="110"/> | |
| <line x1="244" y1="60" x2="356" y2="80"/><line x1="244" y1="110" x2="356" y2="80"/><line x1="244" y1="110" x2="356" y2="135"/><line x1="244" y1="160" x2="356" y2="135"/><line x1="244" y1="160" x2="356" y2="185"/><line x1="244" y1="205" x2="356" y2="185"/> | |
| </g> | |
| <text x="560" y="25" text-anchor="middle" font-weight="700" fill="#E8551A">OUTPUT</text> | |
| <circle cx="530" cy="130" r="20" fill="#FBE9DF" stroke="#E8551A" stroke-width="2"/><text x="530" y="134" text-anchor="middle" font-size="9">fail?</text> | |
| <line x1="384" y1="80" x2="510" y2="125" stroke="#C9D3D8"/><line x1="384" y1="135" x2="510" y2="130" stroke="#C9D3D8"/><line x1="384" y1="185" x2="510" y2="138" stroke="#C9D3D8"/> | |
| <text x="620" y="120" font-size="10.5" fill="#33566B">Every line = a weight,</text> | |
| <text x="620" y="135" font-size="10.5" fill="#33566B">a number tuned by</text> | |
| <text x="620" y="150" font-size="10.5" fill="#33566B">gradient descent (§6.5)</text> | |
| </g></svg> | |
| <figcaption><span class="fn">FIG 8.1</span> — A neural network: layers of simple units, each computing a weighted sum + a squash. "Deep" just means many hidden layers. Training adjusts the millions of line-weights downhill on the error landscape.</figcaption> | |
| </figure> | |
| <p>Why layers matter: each layer learns features OF the previous layer's features. For images: edges → shapes → parts → objects. This automatic feature engineering is exactly why deep learning wins on raw data (pixels, audio, text) where nobody can hand-craft columns — and why it's usually unnecessary for your tabular data, where good columns already exist.</p> | |
| <h3><span class="sn">8.2</span>The architecture zoo</h3> | |
| <table class="tb"> | |
| <tr><th>Architecture</th><th>Built for</th><th>Core trick</th><th>Status 2026</th><th>Plant relevance</th></tr> | |
| <tr><td><strong>CNN</strong> (convolutional)</td><td>Images</td><td>Small filters slide across the image detecting local patterns, position-independent</td><td>Standard for vision</td><td>Cylinder defect detection, gauge reading, PPE compliance cameras</td></tr> | |
| <tr><td><strong>RNN / LSTM</strong></td><td>Sequences</td><td>A memory state carried step to step; LSTM gates decide what to keep/forget</td><td>Largely superseded by Transformers, still in lightweight sensor models</td><td>Legacy sequence models on sensor streams</td></tr> | |
| <tr><td><strong>Transformer</strong></td><td>Originally language; now everything</td><td><strong>Attention</strong>: every element looks at every other and weighs relevance ("In 'the valve failed because IT froze', attention links IT→valve")</td><td>The dominant architecture; powers Claude, ChatGPT, Qwen</td><td>Your on-prem Qwen3 deployment IS a transformer</td></tr> | |
| <tr><td><strong>Vision / Multimodal models</strong></td><td>Images+text together</td><td>Transformers applied to image patches; joint text-image training</td><td>Mainstream (Claude reads your photos)</td><td>"Describe the defect in this cylinder photo" without training anything</td></tr> | |
| </table> | |
| <h3><span class="sn">8.3</span>Embeddings — the bridge concept</h3> | |
| <div class="jd"> | |
| <div class="jd-head"><span class="w">Embedding</span><span class="ar">التمثيل الشعاعي</span></div> | |
| <table> | |
| <tr><td>Technical</td><td>A learned list of numbers (vector) representing an item, where geometric distance ≈ semantic similarity.</td></tr> | |
| <tr><td>Simple</td><td>Meaning turned into coordinates. Similar meanings land close together.</td></tr> | |
| <tr><td>Why it matters</td><td>Embeddings power search-by-meaning: "pump cavitation" finds documents saying "impeller vapor damage" with zero shared words. This is the engine of RAG (Ch. 12) — your SharePoint corpus project depends on it.</td></tr> | |
| <tr><td>Analogy</td><td>Plot equipment on a map by behavior instead of location: pumps that fail similarly become neighbors, regardless of which plant they're in.</td></tr> | |
| <tr><td>You'll use it</td><td>An embedding model + vector database over the LPG spec corpus = semantic search for the whole division.</td></tr> | |
| </table> | |
| </div> | |
| <h3><span class="sn">8.4</span>LLMs in three sentences</h3> | |
| <p>A <strong>Large Language Model</strong> is a giant transformer trained on enormous text to predict the next token (word piece), then refined with human feedback to be helpful and follow instructions. That simple objective, at extreme scale, produces translation, reasoning, and code as side effects. Two consequences you must carry: LLMs are <em>probabilistic</em>, not databases — hence hallucination (§1.7) — and their knowledge freezes at training time — hence RAG to feed them your current documents (Ch. 12).</p> | |
| <div class="co plant"><div class="co-t">▲ The practitioner's split</div><p>Numbers in tables → classical ML (Ch. 7). Images → CNN/vision models, usually fine-tuned, not built from scratch. Text/documents/questions → LLMs via prompting + RAG, almost never trained by you. This one routing rule prevents the most common architecture mistakes in industry.</p></div> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>Deep learning = automatic feature learning through stacked layers; essential for raw data, usually overkill for tables.</li> | |
| <li>CNNs own images; Transformers (attention) own language and increasingly everything else.</li> | |
| <li>Embeddings turn meaning into geometry — the foundation of semantic search and RAG.</li> | |
| <li>LLMs predict next tokens at scale; probabilistic by nature → verify, and feed them fresh context via RAG.</li> | |
| <li>You will USE deep models (fine-tune, prompt, API), not build them — that's the practitioner position.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"I should build a neural network for my forecast"</summary><div class="dbody">Tabular demand data → gradient boosting almost always wins on cost, speed, and explainability. Deep learning earns its complexity only on raw data.</div></details> | |
| <details><summary>"The LLM understands like a human"</summary><div class="dbody">It models statistical patterns of language extraordinarily well. Fluency ≠ comprehension ≠ factual reliability.</div></details> | |
| <details><summary>"Training a vision model needs millions of images"</summary><div class="dbody">Transfer learning: start from a pre-trained model, fine-tune on hundreds of your defect photos. This is what makes cylinder defect detection feasible.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>Fine-tuned CNN on cylinder photos: refurbish / scrap / OK — starting from a pre-trained backbone.</li> | |
| <li>Multimodal Claude reading gauge photos from inspection rounds into structured readings.</li> | |
| <li>Embedding-powered semantic search over specs — the underrated first LLM win you already identified.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="0"><div class="qt">1. Detecting valve defects from photos calls for:</div> | |
| <label><input type="radio" name="c8q1" value="0">A CNN / vision model (fine-tuned)</label><label><input type="radio" name="c8q1" value="1">Linear regression</label><label><input type="radio" name="c8q1" value="2">K-Means</label> | |
| <div class="exp">Raw images = deep learning territory; convolution detects local visual patterns.</div></div> | |
| <div class="quiz-q" data-a="2"><div class="qt">2. "Attention" in a transformer means:</div> | |
| <label><input type="radio" name="c8q2" value="0">The model asks the user to pay attention</label><label><input type="radio" name="c8q2" value="1">Training with louder alarms</label><label><input type="radio" name="c8q2" value="2">Each element weighs its relevance to every other element in the input</label> | |
| <div class="exp">Attention lets "it" find "the valve" across a sentence — context-dependent meaning.</div></div> | |
| <div class="quiz-q" data-a="1"><div class="qt">3. Two maintenance reports use completely different words for the same failure. What finds them as similar?</div> | |
| <label><input type="radio" name="c8q3" value="0">Keyword search</label><label><input type="radio" name="c8q3" value="1">Embeddings — nearby vectors in meaning-space</label><label><input type="radio" name="c8q3" value="2">A histogram</label> | |
| <div class="exp">Semantic similarity = geometric closeness. Keywords fail; embeddings don't.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Send Claude a photo of any equipment nameplate and ask for structured JSON output — you just used a multimodal transformer.</li> | |
| <li>List 5 documents-or-images problems at the plant; route each: prompt an LLM / fine-tune vision / classical ML.</li> | |
| <li>Ask Claude to compute embeddings for 10 short maintenance phrases and show their similarity matrix — watch synonyms cluster.</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Which unstructured data at GFB (photos, reports, PDFs) holds value that tabular systems can't reach?</li> | |
| <li>Where would a wrong-but-confident model output be dangerous enough to require a human gate?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Reaching for deep learning on tabular data as a first move.</li> | |
| <li>Training vision models from scratch instead of fine-tuning pre-trained ones.</li> | |
| <li>Treating LLM output as a database lookup rather than a probabilistic draft.</li> | |
| <li>Ignoring embeddings — the cheapest, highest-ROI deep-learning product for document-heavy teams.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Routing</div>tables→GBM · images→CNN · text→LLM+RAG</div> | |
| <div class="cell"><div class="k">Deep =</div>many layers learning features of features</div> | |
| <div class="cell"><div class="k">Transformer</div>attention: everything weighs everything</div> | |
| <div class="cell"><div class="k">Embedding</div>meaning → coordinates; close = similar</div> | |
| <div class="cell"><div class="k">LLM</div>next-token predictor at scale; verify outputs</div> | |
| <div class="cell"><div class="k">Vision shortcut</div>fine-tune pre-trained, hundreds of images suffice</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">VISION FEASIBILITY<button onclick="cp(this)">Copy</button></div><pre>I want to detect [defect type] on [object] using a camera at [location/conditions]. Assess feasibility: what image quantity/quality/labeling would I need, whether transfer learning applies, expected pitfalls (lighting, night shift, class imbalance), and a phased plan starting with a manual-photo pilot before any hardware purchase.</pre></div> | |
| <div class="prompt"><div class="ph">EMBEDDING DEMO<button onclick="cp(this)">Copy</button></div><pre>Write Colab code using sentence-transformers to embed these 15 maintenance phrases [paste], compute cosine similarity, and visualize as a heatmap. Then explain in simple English why phrase pairs with no shared words still score high.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>LeCun, Bengio & Hinton (2015). "Deep learning." <em>Nature</em> 521.</li> | |
| <li>Vaswani et al. (2017). "Attention Is All You Need." <em>NeurIPS</em>.</li> | |
| <li>Hochreiter & Schmidhuber (1997). "Long Short-Term Memory." <em>Neural Computation</em>.</li> | |
| <li>Dosovitskiy et al. (2021). "An Image is Worth 16×16 Words" (ViT). <em>ICLR</em>.</li> | |
| <li>DeepLearning.AI — Deep Learning Specialization (Andrew Ng); PyTorch & TensorFlow official tutorials; Hugging Face course — huggingface.co/learn.</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(7)">← Ch. 7</button><button class="btn done" onclick="done(8)">Mark complete → Ch. 9</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 9 ================= --> | |
| <article id="ch9" class="chx"> | |
| <div class="plate"><span class="tag">CH-09</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Model Evaluation</h2> | |
| <p class="ch-intro">The chapter that keeps you honest. A model is worth exactly what its evaluation proves — measured on data it never saw, with the right metric for the business cost.</p> | |
| <h3><span class="sn">9.1</span>The confusion matrix — read it once, use it forever</h3> | |
| <figure> | |
| <svg viewBox="0 0 720 270" width="100%" role="img" aria-label="Confusion matrix"> | |
| <g font-family="IBM Plex Sans" font-size="12"> | |
| <text x="245" y="22" text-anchor="middle" font-weight="700" font-family="Archivo">PREDICTED</text> | |
| <text x="150" y="52" text-anchor="middle" font-size="11" fill="#33566B">FAIL</text><text x="340" y="52" text-anchor="middle" font-size="11" fill="#33566B">PASS</text> | |
| <text x="28" y="140" text-anchor="middle" font-weight="700" font-family="Archivo" transform="rotate(-90 28 140)">ACTUAL</text> | |
| <text x="52" y="115" text-anchor="middle" font-size="11" fill="#33566B" transform="rotate(-90 52 115)">FAIL</text><text x="52" y="215" text-anchor="middle" font-size="11" fill="#33566B" transform="rotate(-90 52 215)">PASS</text> | |
| <rect x="65" y="60" width="170" height="95" rx="8" fill="#E3F1EB" stroke="#1E7A5A" stroke-width="2"/><text x="150" y="98" text-anchor="middle" font-weight="700" fill="#1E7A5A">TRUE POSITIVE</text><text x="150" y="118" text-anchor="middle" font-size="10.5">caught a real defect ✓</text><text x="150" y="136" text-anchor="middle" font-family="IBM Plex Mono" font-size="13" font-weight="600">90</text> | |
| <rect x="255" y="60" width="170" height="95" rx="8" fill="#F7E1E1" stroke="#A33" stroke-width="2"/><text x="340" y="98" text-anchor="middle" font-weight="700" fill="#A33">FALSE NEGATIVE</text><text x="340" y="118" text-anchor="middle" font-size="10.5">missed defect → reaches customer ✗✗</text><text x="340" y="136" text-anchor="middle" font-family="IBM Plex Mono" font-size="13" font-weight="600">10</text> | |
| <rect x="65" y="165" width="170" height="95" rx="8" fill="#FCEFE3" stroke="#B3541E" stroke-width="2"/><text x="150" y="203" text-anchor="middle" font-weight="700" fill="#B3541E">FALSE POSITIVE</text><text x="150" y="223" text-anchor="middle" font-size="10.5">false alarm → needless recheck ✗</text><text x="150" y="241" text-anchor="middle" font-family="IBM Plex Mono" font-size="13" font-weight="600">200</text> | |
| <rect x="255" y="165" width="170" height="95" rx="8" fill="#E7F0F7" stroke="#2C5F8A" stroke-width="2"/><text x="340" y="203" text-anchor="middle" font-weight="700" fill="#2C5F8A">TRUE NEGATIVE</text><text x="340" y="223" text-anchor="middle" font-size="10.5">good cylinder passed ✓</text><text x="340" y="241" text-anchor="middle" font-family="IBM Plex Mono" font-size="13" font-weight="600">9,700</text> | |
| <text x="575" y="80" font-weight="700" font-family="Archivo" fill="#16262E">From these 4 cells:</text> | |
| <text x="575" y="108" font-size="11.5" fill="#1E7A5A" font-weight="700">Precision = TP/(TP+FP)</text> | |
| <text x="575" y="124" font-size="10.5" fill="#33566B">= 90/290 ≈ 31%</text> | |
| <text x="575" y="140" font-size="10.5" fill="#33566B">"when it flags, is it right?"</text> | |
| <text x="575" y="170" font-size="11.5" fill="#E8551A" font-weight="700">Recall = TP/(TP+FN)</text> | |
| <text x="575" y="186" font-size="10.5" fill="#33566B">= 90/100 = 90%</text> | |
| <text x="575" y="202" font-size="10.5" fill="#33566B">"of real defects, how many caught?"</text> | |
| <text x="575" y="232" font-size="11.5" fill="#2C5F8A" font-weight="700">Accuracy = 97.9%</text> | |
| <text x="575" y="248" font-size="10.5" fill="#A33">…and still 200 false alarms/day</text> | |
| </g></svg> | |
| <figcaption><span class="fn">FIG 9.1</span> — One realistic day of cylinder QC. Accuracy looks superb while precision is terrible — this is the base-rate trap from §6.3 wearing its evaluation costume.</figcaption> | |
| </figure> | |
| <div class="co warn"><div class="co-t">⚠ The accuracy trap</div><p>With 1% defects, a model that flags NOTHING scores 99% accuracy while catching zero defects. <strong>Never accept accuracy alone for imbalanced problems</strong> — which describes almost every failure/defect/anomaly problem at a plant.</p></div> | |
| <h3><span class="sn">9.2</span>Choosing the classification metric</h3> | |
| <table class="tb"> | |
| <tr><th>Metric</th><th>Answers</th><th>Optimize it when the costly error is…</th><th>Plant framing</th></tr> | |
| <tr><td><strong>Precision</strong></td><td>Of flagged, how many real?</td><td>False alarms (wasted rechecks, alarm fatigue)</td><td>Inspection team drowning in false flags → raise precision</td></tr> | |
| <tr><td><strong>Recall</strong></td><td>Of real cases, how many caught?</td><td>Misses (defect reaches a customer, failure unpredicted)</td><td>Safety-critical detection → recall first, always</td></tr> | |
| <tr><td><strong>F1</strong></td><td>Harmonic balance of both</td><td>You need one number and costs are comparable</td><td>Model comparison shorthand</td></tr> | |
| <tr><td><strong>ROC-AUC</strong></td><td>Across ALL thresholds, how well does the model separate classes? (1.0 perfect, 0.5 coin-flip)</td><td>Comparing models before choosing an operating threshold</td><td>Model A AUC 0.91 vs B 0.84 → A separates better, then set the threshold by business cost</td></tr> | |
| <tr><td><strong>Calibration</strong></td><td>When it says "70%", does it happen ~70% of the time?</td><td>Probabilities feed decisions (maintenance scheduling by risk)</td><td>Risk-ranked maintenance lists need calibrated probabilities, not just rankings</td></tr> | |
| </table> | |
| <p>The <strong>threshold</strong> is the dial: a classifier outputs a probability, and YOU choose the cutoff. Lower it → more recall, more false alarms; raise it → cleaner flags, more misses. Setting this dial is a business decision about error costs — yours, not the algorithm's.</p> | |
| <h3><span class="sn">9.3</span>Regression metrics</h3> | |
| <table class="tb"> | |
| <tr><th>Metric</th><th>Meaning</th><th>Character</th><th>Use</th></tr> | |
| <tr><td><strong>MAE</strong></td><td>Average absolute miss, in real units</td><td>Robust, honest, explainable: "off by 210 cylinders on average"</td><td>Default for management reporting</td></tr> | |
| <tr><td><strong>RMSE</strong></td><td>Root of mean squared miss</td><td>Punishes big misses extra — sensitive to catastrophic errors</td><td>When one huge miss hurts far more than many small ones</td></tr> | |
| <tr><td><strong>MAPE</strong></td><td>Average % miss</td><td>Comparable across products/scales; explodes near zero actuals</td><td>"Forecast within 6.5%" — planning language; avoid for low-volume items</td></tr> | |
| </table> | |
| <div class="co plant"><div class="co-t">▲ Reporting pattern</div><p>Quote MAE in units + MAPE in % + the naive-baseline comparison: <em>"MAE 210 cylinders (4.8% MAPE) vs 9.1% for the seasonal-naive baseline — error roughly halved."</em> One sentence, complete evaluation, management-ready.</p></div> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>Every classification metric derives from four cells: TP, FP, FN, TN. Master the matrix, own them all.</li> | |
| <li>Accuracy lies on imbalanced data; precision vs recall encodes which error costs more.</li> | |
| <li>AUC compares models; the threshold operationalizes one — and setting it is a business call.</li> | |
| <li>MAE for honesty, RMSE when big misses are catastrophic, MAPE for planning language.</li> | |
| <li>Always evaluate on unseen data, always against the baseline.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"97% accuracy = excellent model"</summary><div class="dbody">Not with 1% positives — the do-nothing model scores 99%. Ask for the confusion matrix before applauding.</div></details> | |
| <details><summary>"Maximize precision AND recall"</summary><div class="dbody">They trade off via the threshold. You choose the balance the business needs; you can't max both.</div></details> | |
| <details><summary>"A 70% predicted probability is a fact"</summary><div class="dbody">Only if calibrated. Uncalibrated models rank well but their probability numbers can be fantasy.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>Set the failure-prediction threshold from real costs: false alarm = 2 tech-hours vs miss = 8h line-down. The math gives the threshold.</li> | |
| <li>Monthly forecast scorecard: MAE, MAPE, vs baseline — the honesty layer under the planning dashboard.</li> | |
| <li>Confusion-matrix review meeting per quarter: are FN (misses) drifting up? That's your retraining trigger.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="1"><div class="qt">1. Missing a defective cylinder is far costlier than a false alarm. Prioritize:</div> | |
| <label><input type="radio" name="c9q1" value="0">Precision</label><label><input type="radio" name="c9q1" value="1">Recall</label><label><input type="radio" name="c9q1" value="2">Accuracy</label> | |
| <div class="exp">Recall counts the fraction of real defects caught. Safety-critical = recall first.</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">2. Forecast A: MAE 200. Forecast B: MAE 220 but far fewer huge misses. Metric that would favor B:</div> | |
| <label><input type="radio" name="c9q2" value="0">RMSE</label><label><input type="radio" name="c9q2" value="1">MAPE</label><label><input type="radio" name="c9q2" value="2">Accuracy</label> | |
| <div class="exp">RMSE squares errors, so it punishes catastrophic misses — B's strength.</div></div> | |
| <div class="quiz-q" data-a="2"><div class="qt">3. Lowering the classification threshold generally:</div> | |
| <label><input type="radio" name="c9q3" value="0">Raises precision, lowers recall</label><label><input type="radio" name="c9q3" value="1">Improves both</label><label><input type="radio" name="c9q3" value="2">Raises recall, lowers precision</label> | |
| <div class="exp">A looser trigger flags more: catches more real cases AND more false alarms.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Build FIG 9.1 for a real or imagined plant detector with your own numbers; compute precision, recall, F1 by hand once.</li> | |
| <li>Write the cost of a false alarm and of a miss for one plant use case in KWD/hours — then state which metric leads.</li> | |
| <li>Ask Claude for a threshold-tuning plot (precision & recall vs threshold) on any classifier and pick the operating point.</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Which report at KNPC currently quotes a single accuracy-style number that hides its error structure?</li> | |
| <li>Who should own threshold decisions — engineering, operations, or HSE — for each of your candidate models?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Reporting accuracy on a 1%-positive problem.</li> | |
| <li>Evaluating on training data (Ch. 4's sealed-test rule exists for this).</li> | |
| <li>Leaving the threshold at the 0.5 default without a cost analysis.</li> | |
| <li>Using MAPE on series with near-zero months.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Precision</div>TP/(TP+FP) — trust in flags</div> | |
| <div class="cell"><div class="k">Recall</div>TP/(TP+FN) — coverage of real cases</div> | |
| <div class="cell"><div class="k">F1</div>balance · AUC: separation across thresholds</div> | |
| <div class="cell"><div class="k">MAE</div>avg miss in units · RMSE: punishes big misses</div> | |
| <div class="cell"><div class="k">MAPE</div>% language; beware near-zero actuals</div> | |
| <div class="cell"><div class="k">Golden rule</div>unseen data + baseline comparison, always</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">FULL EVALUATION<button onclick="cp(this)">Copy</button></div><pre>Evaluate this trained classifier properly: confusion matrix (labeled with my domain terms: [terms]), precision, recall, F1, ROC-AUC, and a calibration curve. Then a plain-English paragraph: what the model is good at, its failure mode, and whether it beats the baseline of [describe]. Flag anything suspiciously good.</pre></div> | |
| <div class="prompt"><div class="ph">THRESHOLD BY COST<button onclick="cp(this)">Copy</button></div><pre>False alarm cost: [X]. Missed case cost: [Y]. Positives are [Z]% of cases. Compute the expected cost per prediction across thresholds 0.05–0.95 for my model, plot it, recommend the operating threshold, and show the resulting daily alarm volume for [N] predictions/day.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>scikit-learn User Guide: Model evaluation — scikit-learn.org/stable/modules/model_evaluation.</li> | |
| <li>Fawcett, T. (2006). "An introduction to ROC analysis." <em>Pattern Recognition Letters</em>.</li> | |
| <li>Saito & Rehmsmeier (2015). "The Precision-Recall Plot Is More Informative than ROC on Imbalanced Data." <em>PLOS ONE</em>.</li> | |
| <li>Hyndman & Athanasopoulos, <em>Forecasting: Principles and Practice</em> — otexts.com/fpp3 (forecast accuracy metrics).</li> | |
| <li>Niculescu-Mizil & Caruana (2005). "Predicting Good Probabilities with Supervised Learning." <em>ICML</em> (calibration).</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(8)">← Ch. 8</button><button class="btn done" onclick="done(9)">Mark complete → Ch. 10</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 10 ================= --> | |
| <article id="ch10" class="chx"> | |
| <div class="plate"><span class="tag">CH-10</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Model Improvement</h2> | |
| <p class="ch-intro">Your first model works. Now the disciplined art of making it better — in the right order, because most improvement effort is spent in the wrong place.</p> | |
| <h3><span class="sn">10.1</span>The improvement hierarchy (spend effort top-down)</h3> | |
| <table class="tb"> | |
| <tr><th>Rank</th><th>Lever</th><th>Typical gain</th><th>Why</th></tr> | |
| <tr><td>1</td><td><strong>Better data</strong> — more rows, fixed errors, corrected labels</td><td><span class="pill g">Large</span></td><td>Garbage-in ceiling: no tuning outruns bad data</td></tr> | |
| <tr><td>2</td><td><strong>Better features</strong> — domain-driven engineering (Ch. 4)</td><td><span class="pill g">Large</span></td><td>The model can only combine what you give it</td></tr> | |
| <tr><td>3</td><td><strong>Better algorithm</strong> — next rung on the ladder (Ch. 7)</td><td><span class="pill o">Moderate</span></td><td>RF → GBM often helps; beyond that, diminishing</td></tr> | |
| <tr><td>4</td><td><strong>Hyperparameter tuning</strong></td><td><span class="pill o">Small–moderate</span></td><td>Polish, not transformation — despite its glamour</td></tr> | |
| <tr><td>5</td><td><strong>Ensembling</strong> — combine models</td><td><span class="pill r">Small</span></td><td>Last percent; adds complexity to maintain</td></tr> | |
| </table> | |
| <div class="co warn"><div class="co-t">⚠ Beginner gravity</div><p>Beginners are pulled toward levers 4–5 (they feel technical) while levers 1–2 sit ignored. When a model disappoints, your first question is never "which hyperparameters?" — it's "what data or feature is missing?"</p></div> | |
| <h3><span class="sn">10.2</span>Hyperparameter tuning</h3> | |
| <p><strong>Parameters</strong> are learned by training (the weights). <strong>Hyperparameters</strong> are the settings YOU choose before training: tree depth, number of trees, learning rate, regularization strength. Three search strategies:</p> | |
| <table class="tb"> | |
| <tr><th>Method</th><th>How</th><th>Verdict</th></tr> | |
| <tr><td><strong>Grid search</strong></td><td>Try every combination in a grid</td><td>Exhaustive but explodes combinatorially; fine for 2–3 knobs</td></tr> | |
| <tr><td><strong>Random search</strong></td><td>Sample random combinations</td><td>Surprisingly effective — usually beats grid at equal budget; your default</td></tr> | |
| <tr><td><strong>Bayesian optimization (e.g., Optuna)</strong></td><td>Each trial informs where to look next</td><td>Most sample-efficient; use when training is slow/expensive</td></tr> | |
| </table> | |
| <p>Non-negotiable rule: tune against <strong>cross-validation</strong> scores, never against the test set — otherwise you've quietly trained on your exam.</p> | |
| <h3><span class="sn">10.3</span>Regularization — the overfitting brake</h3> | |
| <p><strong>Regularization</strong> penalizes model complexity during training, forcing it to prefer simpler explanations. In linear models: L1/Lasso (can zero-out useless features — built-in feature selection) and L2/Ridge (shrinks all coefficients smoothly). In trees: limiting depth and requiring minimum samples per leaf. In neural nets: dropout (randomly silencing neurons during training) and early stopping (halt when validation error turns upward).</p> | |
| <div class="co analogy"><div class="co-t">⬡ Plant Analogy</div><p>Regularization is a design margin against over-optimization. A procedure tuned to perfection for last July's exact conditions fails in August; you deliberately keep it slightly general so it survives conditions it hasn't seen.</p></div> | |
| <h3><span class="sn">10.4</span>Data augmentation & imbalance handling</h3> | |
| <ul> | |
| <li><strong>Augmentation</strong> — synthetically expanding training data: for images, rotate/flip/brighten defect photos (one photo becomes twenty); for tabular data, use with caution (SMOTE creates synthetic minority samples — helpful sometimes, dangerous if it invents impossible cases).</li> | |
| <li><strong>Class weights</strong> — the cleaner first move for imbalance: tell the model that missing a defect costs 50× a false alarm (<code>class_weight</code> in sklearn). Try weights before synthetic data.</li> | |
| </ul> | |
| <h3><span class="sn">10.5</span>Ensembling</h3> | |
| <p>Three classical flavors: <strong>bagging</strong> (parallel diverse models voting — Random Forest is bagging built-in), <strong>boosting</strong> (sequential error-correction — XGBoost/LightGBM), and <strong>stacking</strong> (a meta-model learns to combine base models' outputs). Practical guidance: you already use ensembling every time you pick RF or GBM; explicit stacking is competition territory, rarely worth its maintenance cost at work.</p> | |
| <h3><span class="sn">10.6</span>The improvement loop, operationalized</h3> | |
| <pre class="code"><span class="c"># The disciplined loop — one change at a time, always vs CV score</span> | |
| <span class="k">1.</span> Baseline + current model score (CV) <span class="c"># where are we?</span> | |
| <span class="k">2.</span> Error analysis: WHERE does it fail? <span class="c"># worst months? one plant? rare class?</span> | |
| <span class="k">3.</span> Hypothesis: one change likely to fix it <span class="c"># new feature? more data? deeper trees?</span> | |
| <span class="k">4.</span> Apply ONE change → re-score <span class="c"># never bundle changes</span> | |
| <span class="k">5.</span> Keep if better, revert if not, log it <span class="c"># notebook or MLflow</span></pre> | |
| <p><strong>Error analysis</strong> (step 2) is the professional's secret: look at the actual worst predictions. Ten minutes reading the 20 biggest forecast misses teaches more than a day of tuning — you'll spot the missing holiday flag, the sensor outage, the unrecorded line shutdown.</p> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>Improvement hierarchy: data → features → algorithm → tuning → ensembling. Effort flows top-down.</li> | |
| <li>Random search is your tuning default; Bayesian (Optuna) when training is expensive; always against CV, never test.</li> | |
| <li>Regularization trades a little training fit for generalization — the deliberate design margin.</li> | |
| <li>For imbalance: class weights first, synthetic data (SMOTE) second, with skepticism.</li> | |
| <li>One change at a time, driven by error analysis of actual worst cases.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"Tuning will rescue my weak model"</summary><div class="dbody">Tuning polishes; it doesn't transform. A model failing by 30% has a data or feature problem, not a hyperparameter problem.</div></details> | |
| <details><summary>"I tuned on the test set and improved!"</summary><div class="dbody">You leaked. Every peek at test during development converts it into slow training data. Tune on CV; open test once, at the end.</div></details> | |
| <details><summary>"More complex always wins"</summary><div class="dbody">Complexity must pay rent in validation gains AND be worth its maintenance cost. A 0.5% gain from stacking five models is a bad trade at a plant.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>Error analysis ritual on the demand forecast: every quarter, read the 10 worst months — feed findings into the feature library.</li> | |
| <li>Class weights on the QC model reflecting the true KWD cost ratio of miss vs false alarm.</li> | |
| <li>An Optuna tuning run logged with MLflow once the forecast model matures — polish captured, reproducibly.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="2"><div class="qt">1. Your model is 25% worse than needed. Highest-leverage first move:</div> | |
| <label><input type="radio" name="c10q1" value="0">Bayesian hyperparameter search</label><label><input type="radio" name="c10q1" value="1">Stack five models</label><label><input type="radio" name="c10q1" value="2">Error analysis → better data/features</label> | |
| <div class="exp">Large gaps live at the top of the hierarchy. Tuning buys single digits, not 25%.</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">2. Tuning must be scored against:</div> | |
| <label><input type="radio" name="c10q2" value="0">Cross-validation on training data</label><label><input type="radio" name="c10q2" value="1">The test set, repeatedly</label><label><input type="radio" name="c10q2" value="2">Training accuracy</label> | |
| <div class="exp">The test set stays sealed. Repeated test peeks = leakage into your choices.</div></div> | |
| <div class="quiz-q" data-a="1"><div class="qt">3. L1 (Lasso) regularization is special because it:</div> | |
| <label><input type="radio" name="c10q3" value="0">Makes models deeper</label><label><input type="radio" name="c10q3" value="1">Can shrink useless feature weights to exactly zero — automatic selection</label><label><input type="radio" name="c10q3" value="2">Only works on images</label> | |
| <div class="exp">Lasso's zeroing property doubles as built-in feature selection.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Take any Ch. 7 exercise model; run error analysis on its 10 worst predictions and write one hypothesis per finding.</li> | |
| <li>Run RandomizedSearchCV on a Random Forest (ask Claude for the code); record gain vs default settings — calibrate your expectations.</li> | |
| <li>Add class_weight to an imbalanced classifier and watch the precision/recall trade shift.</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>In your engineering work, where do you already apply the "one change at a time" discipline — and where is it violated?</li> | |
| <li>What plant knowledge would explain your forecast's worst month better than any hyperparameter?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Tuning before establishing a clean baseline comparison.</li> | |
| <li>Changing features AND algorithm AND hyperparameters at once — gains unattributable.</li> | |
| <li>SMOTE-ing physically impossible synthetic samples into safety data.</li> | |
| <li>Not logging experiments; two weeks later nobody can reproduce "the good run".</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Hierarchy</div>data → features → algo → tune → ensemble</div> | |
| <div class="cell"><div class="k">Search</div>random default · Optuna when slow · vs CV only</div> | |
| <div class="cell"><div class="k">Regularize</div>L1 zeros · L2 shrinks · trees: depth/leaf · NN: dropout, early stop</div> | |
| <div class="cell"><div class="k">Imbalance</div>class_weight first · SMOTE with care</div> | |
| <div class="cell"><div class="k">Loop</div>score → error analysis → 1 change → re-score → log</div> | |
| <div class="cell"><div class="k">Secret weapon</div>read the 20 worst predictions</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">ERROR ANALYST<button onclick="cp(this)">Copy</button></div><pre>Here are my model's 20 worst predictions with their feature values: [paste]. Find patterns: do errors cluster by time, category, or range? For each pattern, hypothesize a cause and propose ONE specific fix (feature, data, or model change), ranked by expected impact. Domain context: [describe].</pre></div> | |
| <div class="prompt"><div class="ph">TUNING RUN<button onclick="cp(this)">Copy</button></div><pre>Write a RandomizedSearchCV (or Optuna if training is slow) for my [model] with sensible search ranges you justify, 5-fold CV, and a final table: default score vs tuned score vs baseline. End with an honest one-liner: was the tuning worth it?</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>Bergstra & Bengio (2012). "Random Search for Hyper-Parameter Optimization." <em>JMLR</em>.</li> | |
| <li>Akiba et al. (2019). "Optuna: A Next-generation Hyperparameter Optimization Framework." <em>KDD</em>.</li> | |
| <li>Tibshirani (1996). "Regression Shrinkage and Selection via the Lasso." <em>JRSS-B</em>.</li> | |
| <li>Srivastava et al. (2014). "Dropout." <em>JMLR</em>; Chawla et al. (2002). "SMOTE." <em>JAIR</em>.</li> | |
| <li>scikit-learn User Guide: Tuning the hyper-parameters, Ensemble methods — scikit-learn.org.</li> | |
| <li>Ng, A. — "Machine Learning Yearning" (error analysis methodology).</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(9)">← Ch. 9</button><button class="btn done" onclick="done(10)">Mark complete → Ch. 11</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 11 ================= --> | |
| <article id="ch11" class="chx"> | |
| <div class="plate"><span class="tag">CH-11</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Production ML — Models That Keep Working</h2> | |
| <p class="ch-intro">A notebook that predicted well once is a demo. Production means the model runs on schedule, survives change, and someone knows when it breaks. This is where most real-world ML fails.</p> | |
| <h3><span class="sn">11.1</span>Deployment patterns — pick the simplest that works</h3> | |
| <table class="tb"> | |
| <tr><th>Pattern</th><th>How it runs</th><th>Latency</th><th>Fits when</th><th>Your likely uses</th></tr> | |
| <tr><td><strong>Batch</strong></td><td>Scheduled script: load model → score new data → write results to a table/file</td><td>Hours–daily</td><td>Decisions aren't second-by-second</td><td><span class="pill g">90% of plant use cases:</span> nightly demand forecast → Power BI, weekly failure-risk list</td></tr> | |
| <tr><td><strong>Real-time API</strong></td><td>Model behind a web endpoint (FastAPI); systems POST features, get predictions back</td><td>Milliseconds</td><td>A system needs answers on demand</td><td>QC station querying defect probability per cylinder</td></tr> | |
| <tr><td><strong>Edge</strong></td><td>Model runs on the device itself (camera, PLC-adjacent box)</td><td>Instant, offline-capable</td><td>No connectivity / privacy / speed at source</td><td>Vision inspection camera on the filling carousel</td></tr> | |
| </table> | |
| <div class="co plant"><div class="co-t">▲ The boring truth</div><p>A cron-scheduled Python script writing to a database table is a deployment. Start there. FastAPI + Docker enter only when another <em>system</em> (not a person) needs predictions on demand.</p></div> | |
| <h3><span class="sn">11.2</span>The minimum production stack</h3> | |
| <ul> | |
| <li><strong>Serialization</strong> — <code>joblib.dump</code> the entire <em>pipeline</em> (preprocessing + model), never the model alone, or production data won't be transformed identically.</li> | |
| <li><strong>FastAPI</strong> — the standard Python web framework for serving models: ~20 lines wraps your pipeline in an HTTP endpoint.</li> | |
| <li><strong>Docker</strong> — packages code + libraries + Python version into a <em>container</em> that runs identically anywhere. Solves "works on my machine" permanently; it's also how your Ollama/vLLM stack ships.</li> | |
| <li><strong>CI/CD</strong> — automation that tests and deploys on every code change (e.g., GitHub Actions). For you initially: automated tests that run before any model update goes live.</li> | |
| </ul> | |
| <h3><span class="sn">11.3</span>Drift — why every model dies</h3> | |
| <div class="jd"> | |
| <div class="jd-head"><span class="w">Model Drift</span><span class="ar">انحراف النموذج</span></div> | |
| <table> | |
| <tr><td>Technical</td><td><strong>Data drift</strong>: input distributions shift from training-time. <strong>Concept drift</strong>: the input→output relationship itself changes.</td></tr> | |
| <tr><td>Simple</td><td>The world changes; the model's frozen worldview doesn't. Accuracy decays silently.</td></tr> | |
| <tr><td>Why it matters</td><td>The #1 killer of deployed models. New subsidy policy, new customer segment, replaced equipment — your 2024-trained forecast quietly loses touch with 2026.</td></tr> | |
| <tr><td>Analogy</td><td>Instrument calibration drift. Nobody asks IF the gauge drifts — only how often to recalibrate. Models are gauges pointed at a moving process.</td></tr> | |
| <tr><td>You'll use it</td><td>Monthly drift check on the demand model: compare recent MAE vs commissioning MAE; alert at +25%.</td></tr> | |
| </table> | |
| </div> | |
| <h3><span class="sn">11.4</span>Monitoring & retraining — the operating procedure</h3> | |
| <table class="tb"> | |
| <tr><th>Monitor</th><th>What to track</th><th>Trigger</th></tr> | |
| <tr><td><strong>System health</strong></td><td>Did the job run? On time? Errors?</td><td>Any failure → alert same day</td></tr> | |
| <tr><td><strong>Input data</strong></td><td>Volumes, missing %, ranges vs training profile</td><td>Distribution shift → investigate before trusting outputs</td></tr> | |
| <tr><td><strong>Prediction quality</strong></td><td>Rolling MAE/recall once actuals arrive</td><td>Sustained degradation past threshold → retrain</td></tr> | |
| <tr><td><strong>Business impact</strong></td><td>Is the decision the model feeds still improving?</td><td>No → question the model's existence, not just its tuning</td></tr> | |
| </table> | |
| <p><strong>Retraining policy</strong> — decide at deployment, not during the crisis: scheduled (e.g., quarterly with latest data) or triggered (when monitoring crosses thresholds). Either way: retrain → evaluate against the incumbent on recent data → promote only if better → keep the old version for rollback. Version everything: model file, training data snapshot, code commit.</p> | |
| <h3><span class="sn">11.5</span>MLOps — the name for all of this</h3> | |
| <p><strong>MLOps</strong> = the discipline of running ML reliably: versioning (Git + DVC + model registry), automation (CI/CD), monitoring, and governance. Your maturity path: <em>Level 0</em> — manual notebook runs (fine for pilots) → <em>Level 1</em> — scheduled scripts, saved pipelines, basic monitoring (your target for year one) → <em>Level 2</em> — automated retraining pipelines, registries, full observability (justified only when several models run in production). For your on-prem LLM stack, this same layer is exactly litellm (routing) + langfuse (observability) + ragas (evaluation) — MLOps with LLM-flavored names.</p> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>Batch scheduling covers ~90% of plant ML; APIs/edge only when a system needs on-demand answers.</li> | |
| <li>Serialize the full pipeline; Docker guarantees "runs the same everywhere"; CI/CD tests before deploy.</li> | |
| <li>All models drift. Plan recalibration at commissioning, like any instrument.</li> | |
| <li>Monitor four layers: system ran → inputs sane → predictions accurate → business impact real.</li> | |
| <li>Retrain-evaluate-promote-rollback, all versioned. MLOps is just operational discipline you already respect.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"Deployed = done"</summary><div class="dbody">Deployment is commissioning day. The operating life — monitoring, drift, retraining — is where value is won or silently lost.</div></details> | |
| <details><summary>"I need Kubernetes"</summary><div class="dbody">You need cron + joblib + a database table. Industrial-scale orchestration enters with industrial-scale model counts.</div></details> | |
| <details><summary>"Accuracy was validated at launch, so it holds"</summary><div class="dbody">Validation certifies the past. Only monitoring certifies the present.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>Nightly batch job: score demand model → write to the table Power BI reads → your dashboard becomes model-fed.</li> | |
| <li>A one-page "Model Operating Procedure" per deployed model — owner, schedule, monitors, retrain triggers, rollback. Pure KNPC procedure culture, applied to ML.</li> | |
| <li>Drift dashboard: rolling forecast MAE with a control-limit line (Ch. 6's 3σ thinking, recycled).</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="1"><div class="qt">1. A weekly failure-risk list for maintenance planning should be served as:</div> | |
| <label><input type="radio" name="c11q1" value="0">Real-time API with Kubernetes</label><label><input type="radio" name="c11q1" value="1">A scheduled batch job writing to a table</label><label><input type="radio" name="c11q1" value="2">Edge deployment on each pump</label> | |
| <div class="exp">Weekly cadence = batch. Simplest pattern that meets the latency requirement wins.</div></div> | |
| <div class="quiz-q" data-a="2"><div class="qt">2. Forecast MAE has crept from 210 to 340 over four months. Most likely:</div> | |
| <label><input type="radio" name="c11q2" value="0">The GPU is aging</label><label><input type="radio" name="c11q2" value="1">joblib corrupted the file</label><label><input type="radio" name="c11q2" value="2">Drift — the demand pattern has shifted since training</label> | |
| <div class="exp">Gradual decay is drift's signature. Investigate what changed, then retrain on recent data.</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">3. Why serialize the whole pipeline rather than just the model?</div> | |
| <label><input type="radio" name="c11q3" value="0">So production data receives the exact same preprocessing as training data</label><label><input type="radio" name="c11q3" value="1">Smaller file size</label><label><input type="radio" name="c11q3" value="2">Docker requires it</label> | |
| <div class="exp">Mismatched preprocessing is a classic silent production failure — the pipeline object prevents it.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Draft the one-page Model Operating Procedure for your future demand model: owner, schedule, 4 monitors, retrain trigger, rollback step.</li> | |
| <li>Ask Claude to wrap a trained pipeline in a 20-line FastAPI app and explain every line — just to see how small "deployment" is.</li> | |
| <li>Define the drift alert for one KPI model: which metric, computed how often, alarm threshold, and who gets notified.</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Your plant has recalibration schedules for instruments. What's the organizational equivalent for models — and who owns it?</li> | |
| <li>Which upcoming change at KNPC (pricing, subsidy, equipment) would silently break a model trained today?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>No baseline snapshot of input distributions at deployment — drift becomes undetectable.</li> | |
| <li>Retraining on data that includes the degraded period's corrupted inputs without review.</li> | |
| <li>Deploying a new version with no rollback path.</li> | |
| <li>Monitoring accuracy but not whether the job even ran (silent 3-week outages are real).</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Pattern</div>batch (default) · API (on-demand) · edge (at source)</div> | |
| <div class="cell"><div class="k">Stack</div>pipeline + joblib → FastAPI → Docker → CI/CD</div> | |
| <div class="cell"><div class="k">Drift</div>data drift (inputs) · concept drift (relationship)</div> | |
| <div class="cell"><div class="k">Monitor</div>ran? inputs? accuracy? business impact?</div> | |
| <div class="cell"><div class="k">Retrain</div>schedule or trigger → beat incumbent → promote → keep rollback</div> | |
| <div class="cell"><div class="k">Version</div>model + data snapshot + code commit, together</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">DEPLOYMENT PLANNER<button onclick="cp(this)">Copy</button></div><pre>My model: [describe]. Consumers of predictions: [who/what systems]. Required freshness: [latency]. Infrastructure available: [describe — e.g., one on-prem server, no cloud]. Recommend the simplest deployment pattern, the exact minimal stack, and a monitoring plan with 4 specific checks and thresholds. Explicitly list what I should NOT build yet.</pre></div> | |
| <div class="prompt"><div class="ph">BATCH JOB BUILDER<button onclick="cp(this)">Copy</button></div><pre>Write a production-quality Python batch script that: loads pipeline.pkl, reads new rows from [source], validates inputs against expected ranges (fail loudly with a clear log message), writes predictions to [destination] with a timestamp and model version column, and logs a run summary. Include error handling that never writes partial results.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>Sculley et al. (2015). "Hidden Technical Debt in Machine Learning Systems." <em>NeurIPS</em> — the founding MLOps paper.</li> | |
| <li>Google Cloud Architecture Center: "MLOps: Continuous delivery and automation pipelines in ML" (maturity levels).</li> | |
| <li>Gama et al. (2014). "A Survey on Concept Drift Adaptation." <em>ACM Computing Surveys</em>.</li> | |
| <li>FastAPI documentation — fastapi.tiangolo.com; Docker documentation — docs.docker.com.</li> | |
| <li>Huyen, C. (2022). <em>Designing Machine Learning Systems.</em> O'Reilly.</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(10)">← Ch. 10</button><button class="btn done" onclick="done(11)">Mark complete → Ch. 12</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 12 ================= --> | |
| <article id="ch12" class="chx"> | |
| <div class="plate"><span class="tag">CH-12</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Generative AI & LLMs in Practice</h2> | |
| <p class="ch-intro">The chapter closest to your day-to-day: you're already deploying Qwen on-prem and planning RAG over SharePoint. Here's the complete conceptual map behind those decisions.</p> | |
| <h3><span class="sn">12.1</span>Prompt engineering — the six patterns that matter</h3> | |
| <table class="tb"> | |
| <tr><th>Pattern</th><th>What it is</th><th>Example fragment</th></tr> | |
| <tr><td><strong>Role</strong></td><td>Assign expertise + audience</td><td>"You are a senior reliability engineer writing for plant operators…"</td></tr> | |
| <tr><td><strong>Context loading</strong></td><td>Give the situation, constraints, data — the model can't read your mind</td><td>"Two filling plants, 12kg focus, Kuwaiti summer peak, fiscal year Apr–Mar…"</td></tr> | |
| <tr><td><strong>Output specification</strong></td><td>Dictate format, length, structure</td><td>"Return a markdown table with columns X, Y, Z; max 10 rows"</td></tr> | |
| <tr><td><strong>Few-shot examples</strong></td><td>Show 1–3 examples of input→desired output; the strongest single lever for consistency</td><td>"Here are two correctly classified logs: …"</td></tr> | |
| <tr><td><strong>Step-by-step reasoning</strong></td><td>Ask it to reason before answering — measurably improves multi-step accuracy</td><td>"Think through the calculation first, then state the answer"</td></tr> | |
| <tr><td><strong>Self-critique</strong></td><td>Second pass: "review your answer for errors/omissions"</td><td>Catches a surprising share of first-draft mistakes</td></tr> | |
| </table> | |
| <h3><span class="sn">12.2</span>RAG — your most important architecture</h3> | |
| <div class="jd"> | |
| <div class="jd-head"><span class="w">RAG — Retrieval-Augmented Generation</span><span class="ar">التوليد المعزز بالاسترجاع</span></div> | |
| <table> | |
| <tr><td>Technical</td><td>Pipeline: documents → chunks → embeddings → vector DB; at query time, retrieve the most relevant chunks and inject them into the LLM's prompt as grounding context.</td></tr> | |
| <tr><td>Simple</td><td>Give the model an open-book exam with YOUR books, instead of asking it to answer from memory.</td></tr> | |
| <tr><td>Why it matters</td><td>Solves the three LLM killers at once: stale knowledge, no access to private documents, hallucination (answers now cite retrieved text). It's the standard enterprise pattern — and cheaper/safer than fine-tuning for knowledge.</td></tr> | |
| <tr><td>Analogy</td><td>A brilliant new engineer who hasn't read KNPC's archives. RAG hands them the right 5 pages from the archive before every question — instead of sending them to a 6-month training course (fine-tuning).</td></tr> | |
| <tr><td>You'll use it</td><td>Your SharePoint spec corpus → markitdown/MinerU for conversion → chunking → embeddings → vector DB → Qwen answers with citations. Quality measured with ragas.</td></tr> | |
| </table> | |
| </div> | |
| <figure> | |
| <svg viewBox="0 0 720 200" width="100%" role="img" aria-label="RAG pipeline"> | |
| <defs><marker id="ar12" markerWidth="8" markerHeight="8" refX="6" refY="4" orient="auto"><path d="M0,0 L8,4 L0,8 z" fill="#33566B"/></marker></defs> | |
| <g font-family="IBM Plex Sans" font-size="11"> | |
| <rect x="10" y="20" width="120" height="52" rx="8" fill="#E7F0F7" stroke="#2C5F8A"/><text x="70" y="42" text-anchor="middle" font-weight="700">SPEC PDFs</text><text x="70" y="58" text-anchor="middle" font-size="10">SharePoint corpus</text> | |
| <line x1="130" y1="46" x2="160" y2="46" stroke="#33566B" stroke-width="2" marker-end="url(#ar12)"/> | |
| <rect x="165" y="20" width="110" height="52" rx="8" fill="#E3F1EB" stroke="#1E7A5A"/><text x="220" y="42" text-anchor="middle" font-weight="700">CHUNK</text><text x="220" y="58" text-anchor="middle" font-size="10">markitdown / MinerU</text> | |
| <line x1="275" y1="46" x2="305" y2="46" stroke="#33566B" stroke-width="2" marker-end="url(#ar12)"/> | |
| <rect x="310" y="20" width="110" height="52" rx="8" fill="#E3F1EB" stroke="#1E7A5A"/><text x="365" y="42" text-anchor="middle" font-weight="700">EMBED</text><text x="365" y="58" text-anchor="middle" font-size="10">meaning → vectors</text> | |
| <line x1="420" y1="46" x2="450" y2="46" stroke="#33566B" stroke-width="2" marker-end="url(#ar12)"/> | |
| <rect x="455" y="20" width="120" height="52" rx="8" fill="#FCEFE3" stroke="#B3541E"/><text x="515" y="42" text-anchor="middle" font-weight="700">VECTOR DB</text><text x="515" y="58" text-anchor="middle" font-size="10">Chroma / Qdrant / pgvector</text> | |
| <rect x="10" y="120" width="130" height="52" rx="8" fill="#fff" stroke="#16262E" stroke-width="2"/><text x="75" y="142" text-anchor="middle" font-weight="700">USER QUESTION</text><text x="75" y="158" text-anchor="middle" font-size="10">"min wall thickness?"</text> | |
| <line x1="140" y1="146" x2="450" y2="146" stroke="#33566B" stroke-width="2" marker-end="url(#ar12)"/><text x="295" y="138" text-anchor="middle" font-size="10" font-family="IBM Plex Mono" fill="#33566B">embed query → find top-5 similar chunks</text> | |
| <line x1="515" y1="72" x2="515" y2="115" stroke="#33566B" stroke-width="2" marker-end="url(#ar12)"/> | |
| <rect x="455" y="120" width="120" height="52" rx="8" fill="#FBE9DF" stroke="#E8551A" stroke-width="2"/><text x="515" y="142" text-anchor="middle" font-weight="700">LLM (Qwen)</text><text x="515" y="158" text-anchor="middle" font-size="10">answers FROM chunks</text> | |
| <line x1="575" y1="146" x2="605" y2="146" stroke="#33566B" stroke-width="2" marker-end="url(#ar12)"/> | |
| <rect x="610" y="120" width="100" height="52" rx="8" fill="#16262E"/><text x="660" y="142" text-anchor="middle" fill="#fff" font-weight="700">ANSWER</text><text x="660" y="158" text-anchor="middle" fill="#9FB3BD" font-size="10">+ citations</text> | |
| </g></svg> | |
| <figcaption><span class="fn">FIG 12.1</span> — Your SharePoint RAG pipeline end to end. The top row runs once (and on document updates); the bottom row runs per question.</figcaption> | |
| </figure> | |
| <p><strong>What actually determines RAG quality</strong> (in order): document conversion fidelity (garbage extraction = garbage answers — why MinerU matters for engineering PDFs with tables), chunking strategy (split by section, not blindly by character count), retrieval quality (embedding model choice; hybrid keyword+vector search for spec numbers), and only then the LLM. Evaluate with <strong>ragas</strong>-style metrics: faithfulness (answer grounded in retrieved text?), relevance, and retrieval hit rate on a golden set of ~50 real engineer questions.</p> | |
| <h3><span class="sn">12.3</span>Fine-tuning vs RAG vs prompting</h3> | |
| <table class="tb"> | |
| <tr><th>Need</th><th>Right tool</th><th>Why</th></tr> | |
| <tr><td>Model lacks YOUR documents/knowledge</td><td><span class="pill g">RAG</span></td><td>Knowledge stays fresh, citable, updatable without retraining</td></tr> | |
| <tr><td>Model output format/style/behavior wrong</td><td><span class="pill g">Prompting first</span>, fine-tuning if truly persistent</td><td>Few-shot examples fix most format problems free</td></tr> | |
| <tr><td>Specialized vocabulary/task at high volume</td><td><span class="pill o">Fine-tuning (LoRA)</span></td><td>Teaches behavior patterns; does NOT reliably add facts</td></tr> | |
| <tr><td>General capability shortfall</td><td><span class="pill r">Bigger/better model</span></td><td>No amount of tuning turns a 3B model into a 30B one</td></tr> | |
| </table> | |
| <div class="co warn"><div class="co-t">⚠ The classic enterprise mistake</div><p>Fine-tuning to inject company knowledge. It's expensive, freezes knowledge at tuning time, and hallucinates confidently about the gaps. Knowledge → RAG. Behavior → fine-tune. Tattoo this distinction.</p></div> | |
| <h3><span class="sn">12.4</span>Agents, tools, and MCP</h3> | |
| <p>An <strong>agent</strong> is an LLM in a loop: it plans, calls <strong>tools</strong> (run SQL, search documents, call an API), observes results, and iterates until the task completes. <strong>MCP (Model Context Protocol)</strong> is the open standard for plugging tools into LLMs — you've already used it connecting Power BI to Claude. Your text-to-SQL pilot is exactly a single-tool agent: question → generated SQL → execute → summarize results. The engineering discipline: constrain tools (read-only DB access!), log every step (langfuse), and keep a human gate on anything that writes.</p> | |
| <h3><span class="sn">12.5</span>Your on-prem serving stack, mapped</h3> | |
| <table class="tb"> | |
| <tr><th>Layer</th><th>Your component</th><th>Role</th></tr> | |
| <tr><td>Inference engine</td><td><strong>Ollama</strong> (pilot) → <strong>vLLM</strong> (production)</td><td>Runs Qwen3-30B; vLLM adds high-throughput serving for ~30 users</td></tr> | |
| <tr><td>Gateway / routing</td><td><strong>litellm</strong></td><td>One API for all models; per-user keys, quotas, fallbacks</td></tr> | |
| <tr><td>Observability</td><td><strong>langfuse</strong></td><td>Logs every prompt/response/cost/latency — your COGNOS for LLM usage</td></tr> | |
| <tr><td>Evaluation</td><td><strong>ragas</strong> + golden question set</td><td>Regression-tests RAG quality after every change</td></tr> | |
| <tr><td>Document ingestion</td><td><strong>markitdown / MinerU</strong></td><td>PDF/Office → clean text for the RAG index</td></tr> | |
| </table> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>Six prompt patterns (role, context, output spec, few-shot, reasoning, self-critique) cover professional prompting.</li> | |
| <li>RAG = open-book exam with your books; quality lives in conversion + chunking + retrieval, not just the LLM.</li> | |
| <li>Knowledge → RAG; behavior → fine-tune; format → prompt. Never fine-tune to inject facts.</li> | |
| <li>Agents = LLM + tools in a loop; MCP standardizes the plugs; constrain, log, human-gate writes.</li> | |
| <li>Your stack is coherent: Ollama/vLLM serve, litellm routes, langfuse watches, ragas grades.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"RAG eliminates hallucination"</summary><div class="dbody">It dramatically reduces it and adds citations, but bad retrieval or over-creative generation still fabricates. That's why ragas faithfulness scoring exists.</div></details> | |
| <details><summary>"We should fine-tune Qwen on all our documents"</summary><div class="dbody">Fine-tuning teaches style, not a reliable document memory. Your corpus belongs in a vector DB, retrieved per-question.</div></details> | |
| <details><summary>"Better prompts are about magic words"</summary><div class="dbody">They're about information: context, examples, and explicit output contracts. The six patterns are engineering, not incantation.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>Spec-RAG assistant: "What's the test pressure for 12kg cylinders per our spec?" → answer + clause citation.</li> | |
| <li>Text-to-SQL on planning tables (your live pilot) — with read-only credentials and langfuse audit trail.</li> | |
| <li>Bilingual report drafting: Arabic/English management summaries from structured monthly data.</li> | |
| <li>Golden-question regression suite: 50 real engineer questions, scored monthly with ragas.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="1"><div class="qt">1. Engineers need current spec answers with citations. The right architecture:</div> | |
| <label><input type="radio" name="c12q1" value="0">Fine-tune Qwen on the PDFs</label><label><input type="radio" name="c12q1" value="1">RAG over the spec corpus</label><label><input type="radio" name="c12q1" value="2">A bigger base model</label> | |
| <div class="exp">Private, updatable, citable knowledge = RAG. Fine-tuning won't reliably memorize documents.</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">2. The single strongest prompt lever for consistent output format:</div> | |
| <label><input type="radio" name="c12q2" value="0">Few-shot examples of input→desired output</label><label><input type="radio" name="c12q2" value="1">Typing in capital letters</label><label><input type="radio" name="c12q2" value="2">Longer prompts of any kind</label> | |
| <div class="exp">Showing beats telling. 1–3 examples anchor structure better than paragraphs of instructions.</div></div> | |
| <div class="quiz-q" data-a="2"><div class="qt">3. In your stack, the component that answers "which users asked what, at what cost, with what latency":</div> | |
| <label><input type="radio" name="c12q3" value="0">vLLM</label><label><input type="radio" name="c12q3" value="1">ragas</label><label><input type="radio" name="c12q3" value="2">langfuse</label> | |
| <div class="exp">langfuse = observability. vLLM serves; ragas evaluates quality.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Rewrite one of your recent Claude prompts applying all six patterns; compare outputs side by side.</li> | |
| <li>Draft the 50-question golden set for your spec-RAG from real questions engineers actually ask (start with 10).</li> | |
| <li>Design the guardrails table for your text-to-SQL agent: allowed schemas, read-only enforcement, row limits, logging, human review triggers.</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Which division questions are asked repeatedly whose answers already exist in documents nobody re-reads?</li> | |
| <li>Where must a human stay in the loop permanently, regardless of how good the models get?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Skipping document-conversion quality checks — the #1 silent RAG killer for engineering PDFs.</li> | |
| <li>Chunking by fixed character count through the middle of specification tables.</li> | |
| <li>Granting an SQL agent write permissions "for convenience".</li> | |
| <li>Deploying RAG with no golden-set evaluation — quality regressions go unnoticed for months.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Prompt six</div>role · context · output spec · few-shot · reasoning · critique</div> | |
| <div class="cell"><div class="k">RAG chain</div>convert → chunk → embed → store → retrieve → generate</div> | |
| <div class="cell"><div class="k">Decision</div>knowledge→RAG · behavior→fine-tune · format→prompt</div> | |
| <div class="cell"><div class="k">Agent rules</div>constrain tools · log all · gate writes</div> | |
| <div class="cell"><div class="k">Your stack</div>Ollama/vLLM · litellm · langfuse · ragas</div> | |
| <div class="cell"><div class="k">RAG quality</div>conversion > chunking > retrieval > model</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">RAG ARCHITECT<button onclick="cp(this)">Copy</button></div><pre>Design a RAG pipeline for [N] documents of type [engineering specs with tables/scanned PDFs/mixed], serving [M] users on-prem with [hardware]. Specify: conversion tool and quality checks, chunking strategy with sizes and WHY, embedding model (multilingual — Arabic+English), vector DB choice, retrieval approach (hybrid?), and an evaluation plan with a golden question set. Flag the top 3 failure risks for my document type.</pre></div> | |
| <div class="prompt"><div class="ph">PROMPT UPGRADER<button onclick="cp(this)">Copy</button></div><pre>Here is a prompt I use often: [paste]. Rewrite it applying: role, context loading, explicit output specification, one few-shot example, and a self-critique instruction. Then explain each change's purpose in one line.</pre></div> | |
| <div class="prompt"><div class="ph">AGENT GUARDRAILS<button onclick="cp(this)">Copy</button></div><pre>I'm building a text-to-SQL agent over [describe tables] for [users]. Write its system prompt including: allowed schemas only, read-only enforcement, automatic LIMIT clauses, refusal rules for destructive requests, and a response format that always shows the SQL before results. Then list 5 test prompts a malicious or careless user might try.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>Lewis et al. (2020). "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks." <em>NeurIPS</em> — the RAG paper.</li> | |
| <li>Brown et al. (2020). "Language Models are Few-Shot Learners." <em>NeurIPS</em> (GPT-3, few-shot).</li> | |
| <li>Wei et al. (2022). "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models." <em>NeurIPS</em>.</li> | |
| <li>Hu et al. (2021). "LoRA: Low-Rank Adaptation of Large Language Models." <em>ICLR</em>.</li> | |
| <li>Anthropic prompt engineering documentation — docs.claude.com; Model Context Protocol — modelcontextprotocol.io.</li> | |
| <li>Es et al. (2023). "RAGAS: Automated Evaluation of Retrieval Augmented Generation." <em>arXiv:2309.15217</em>.</li> | |
| <li>Kwon et al. (2023). "Efficient Memory Management for LLM Serving with PagedAttention." <em>SOSP</em> (vLLM).</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(11)">← Ch. 11</button><button class="btn done" onclick="done(12)">Mark complete → Ch. 13</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 13 ================= --> | |
| <article id="ch13" class="chx"> | |
| <div class="plate"><span class="tag">CH-13</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">ML at the LPG Filling Plant</h2> | |
| <p class="ch-intro">Everything so far, aimed at GFB-U and GFB-S. A ranked application portfolio, honest difficulty ratings, and the sequencing logic of which to build first.</p> | |
| <h3><span class="sn">13.1</span>The application portfolio</h3> | |
| <table class="tb"> | |
| <tr><th>Application</th><th>Technique (Ch.)</th><th>Data readiness</th><th>Difficulty</th><th>Impact</th></tr> | |
| <tr><td><strong>Cylinder demand forecasting</strong> (monthly/weekly, by size)</td><td>Time series: Prophet/GBM+lags (7)</td><td><span class="pill g">High — years of filling logs</span></td><td><span class="pill g">Low</span></td><td><span class="pill g">High: inventory, shifts, distribution planning</span></td></tr> | |
| <tr><td><strong>Energy anomaly detection</strong> (kWh per cylinder drifting)</td><td>3σ → Isolation Forest (6, 7)</td><td><span class="pill g">High — meter + production data</span></td><td><span class="pill g">Low</span></td><td><span class="pill o">Medium: OPEX + early fault signal</span></td></tr> | |
| <tr><td><strong>Spec-RAG assistant</strong> (SharePoint corpus)</td><td>RAG (12)</td><td><span class="pill g">High — corpus exists</span></td><td><span class="pill o">Medium</span></td><td><span class="pill g">High: division-wide time savings</span></td></tr> | |
| <tr><td><strong>Text-to-SQL on planning data</strong></td><td>LLM agent (12)</td><td><span class="pill g">High — your pilot</span></td><td><span class="pill o">Medium</span></td><td><span class="pill g">High: self-serve analytics for 30 users</span></td></tr> | |
| <tr><td><strong>Downtime/stoppage classification & prediction</strong></td><td>Classification: RF/GBM (7)</td><td><span class="pill o">Medium — depends on log discipline</span></td><td><span class="pill o">Medium</span></td><td><span class="pill g">High: maintenance planning</span></td></tr> | |
| <tr><td><strong>Maintenance-log mining</strong> (cluster failure texts)</td><td>Embeddings + clustering (8, 7)</td><td><span class="pill o">Medium — free-text quality varies</span></td><td><span class="pill o">Medium</span></td><td><span class="pill o">Medium: recurring-failure visibility</span></td></tr> | |
| <tr><td><strong>Cylinder visual defect detection</strong></td><td>Fine-tuned CNN (8)</td><td><span class="pill r">Low — needs labeled photos</span></td><td><span class="pill r">High</span></td><td><span class="pill g">High: QC consistency & safety</span></td></tr> | |
| <tr><td><strong>Filling-line predictive maintenance</strong> (sensor-based)</td><td>Anomaly + classification (7)</td><td><span class="pill r">Low — sensor history & failure labels scarce</span></td><td><span class="pill r">High</span></td><td><span class="pill g">High: unplanned downtime ↓</span></td></tr> | |
| <tr><td><strong>OPEX/CAPEX forecast & variance early warning</strong></td><td>Regression + control limits (7, 6)</td><td><span class="pill g">High — your own tables</span></td><td><span class="pill g">Low</span></td><td><span class="pill o">Medium: planning accuracy</span></td></tr> | |
| <tr><td><strong>Distribution/truck scheduling optimization</strong></td><td>Optimization (+ forecast input)</td><td><span class="pill o">Medium</span></td><td><span class="pill r">High</span></td><td><span class="pill o">Medium–high, shared with logistics</span></td></tr> | |
| </table> | |
| <h3><span class="sn">13.2</span>Sequencing — the dependency logic</h3> | |
| <figure> | |
| <svg viewBox="0 0 720 210" width="100%" role="img" aria-label="Project sequencing waves"> | |
| <g font-family="IBM Plex Sans" font-size="11"> | |
| <rect x="10" y="30" width="220" height="150" rx="10" fill="#E3F1EB" stroke="#1E7A5A" stroke-width="2"/> | |
| <text x="120" y="55" text-anchor="middle" font-weight="800" font-family="Archivo" fill="#1E7A5A">WAVE 1 · 0–4 mo</text> | |
| <text x="120" y="75" text-anchor="middle" font-size="10" font-family="IBM Plex Mono" fill="#1E7A5A">PROVE VALUE, BUILD SKILL</text> | |
| <text x="25" y="100" font-size="11">• Demand forecast (flagship)</text> | |
| <text x="25" y="120" font-size="11">• Energy anomaly (3σ → IsoForest)</text> | |
| <text x="25" y="140" font-size="11">• OPEX variance early warning</text> | |
| <text x="25" y="163" font-size="10" fill="#33566B">Tabular, existing data, batch jobs</text> | |
| <rect x="250" y="30" width="220" height="150" rx="10" fill="#FCEFE3" stroke="#B3541E" stroke-width="2"/> | |
| <text x="360" y="55" text-anchor="middle" font-weight="800" font-family="Archivo" fill="#B3541E">WAVE 2 · 4–10 mo</text> | |
| <text x="360" y="75" text-anchor="middle" font-size="10" font-family="IBM Plex Mono" fill="#B3541E">LEVERAGE THE LLM STACK</text> | |
| <text x="265" y="100" font-size="11">• Spec-RAG assistant</text> | |
| <text x="265" y="120" font-size="11">• Text-to-SQL to production</text> | |
| <text x="265" y="140" font-size="11">• Downtime classification</text> | |
| <text x="265" y="163" font-size="10" fill="#33566B">Rides your Qwen/vLLM investment</text> | |
| <rect x="490" y="30" width="220" height="150" rx="10" fill="#F7E1E1" stroke="#A33" stroke-width="2"/> | |
| <text x="600" y="55" text-anchor="middle" font-weight="800" font-family="Archivo" fill="#A33">WAVE 3 · 10 mo+</text> | |
| <text x="600" y="75" text-anchor="middle" font-size="10" font-family="IBM Plex Mono" fill="#A33">DATA-GATED AMBITIONS</text> | |
| <text x="505" y="100" font-size="11">• Vision QC (start photo collection NOW)</text> | |
| <text x="505" y="120" font-size="11">• Sensor predictive maintenance</text> | |
| <text x="505" y="140" font-size="11">• Scheduling optimization</text> | |
| <text x="505" y="163" font-size="10" fill="#33566B">Blocked on data you must begin logging today</text> | |
| </g></svg> | |
| <figcaption><span class="fn">FIG 13.1</span> — Waves ordered by data readiness × skill building. Wave 3's gate is data collection that must START during Wave 1 — the classic sequencing insight.</figcaption> | |
| </figure> | |
| <div class="co plant"><div class="co-t">▲ The Wave-3 unlock</div><p>Vision QC and predictive maintenance aren't blocked by algorithms — they're blocked by <em>data you aren't collecting yet</em>. The highest-leverage action this quarter costs almost nothing: standardized defect photo capture at QC stations, and disciplined failure-cause codes in the maintenance log. Wave 1 delivers value; its quiet job is seeding Wave 3.</p></div> | |
| <h3><span class="sn">13.3</span>Fitting your existing landscape</h3> | |
| <table class="tb"> | |
| <tr><th>Existing asset</th><th>Role in the ML architecture</th></tr> | |
| <tr><td><strong>COGNOS / planning tables</strong></td><td>Source of record → SQL extracts feed pandas & the text-to-SQL agent</td></tr> | |
| <tr><td><strong>Power BI + DAX layer</strong></td><td>Delivery surface: model outputs land in tables Power BI already reads — predictions appear inside dashboards management already trusts</td></tr> | |
| <tr><td><strong>SharePoint spec system</strong></td><td>RAG corpus + the change-request workflow keeps the index fresh</td></tr> | |
| <tr><td><strong>On-prem server (Ollama→vLLM)</strong></td><td>LLM inference for RAG, text-to-SQL, and report drafting — data never leaves KNPC</td></tr> | |
| <tr><td><strong>Your HTML dashboards</strong></td><td>Rapid UI layer for pilot tools before formal IT integration</td></tr> | |
| </table> | |
| <p><strong>The selling framework</strong> for each proposal, in planning language: baseline cost of current practice → pilot scope (one plant, one product, 3 months) → success metric fixed in advance (e.g., "forecast MAPE under 7% vs 11% manual") → decision gate. Pilots framed as reversible experiments pass approval chains that "AI transformation programs" never survive.</p> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>Ten credible applications exist today; three are Wave-1 ready with data you already control.</li> | |
| <li>Sequence by data readiness × skill building, not by impressiveness.</li> | |
| <li>Wave 3 is gated by data collection that must start now — photos and failure codes.</li> | |
| <li>Deliver predictions through Power BI surfaces management already trusts.</li> | |
| <li>Sell pilots with baselines, pre-agreed metrics, and decision gates — planning language, not AI hype.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"Start with the most impressive project"</summary><div class="dbody">Vision QC demos well and fails first — no labeled photos. Credibility compounds from boring wins; start where the data is.</div></details> | |
| <details><summary>"We need new sensors/systems before any ML"</summary><div class="dbody">Wave 1 runs entirely on data already in COGNOS, meters, and your Excel exports.</div></details> | |
| <details><summary>"One big integrated AI platform"</summary><div class="dbody">Portfolio of small, independently valuable tools > monolith. Each survives on its own ROI.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>This chapter IS the application catalog — take FIG 13.1 into your 5-year plan as the digital initiatives roadmap.</li> | |
| <li>Quarterly portfolio review: re-score data readiness as logging discipline improves.</li> | |
| <li>Pair each wave with a procedure doc (your LPG-ENG-PROC pattern) so tools outlive their builder.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="0"><div class="qt">1. The correct first flagship project is demand forecasting because:</div> | |
| <label><input type="radio" name="c13q1" value="0">Data exists, difficulty is low, impact is high, and it builds the full skill loop</label><label><input type="radio" name="c13q1" value="1">It uses the most advanced algorithms</label><label><input type="radio" name="c13q1" value="2">It requires no evaluation</label> | |
| <div class="exp">Readiness × impact ÷ difficulty. The flagship also exercises every chapter of this manual once.</div></div> | |
| <div class="quiz-q" data-a="2"><div class="qt">2. Vision QC's real blocker is:</div> | |
| <label><input type="radio" name="c13q2" value="0">CNNs aren't accurate enough</label><label><input type="radio" name="c13q2" value="1">GPU cost</label><label><input type="radio" name="c13q2" value="2">No labeled defect photo dataset exists yet</label> | |
| <div class="exp">Algorithms are commoditized; labeled data isn't. Hence: start collecting now.</div></div> | |
| <div class="quiz-q" data-a="1"><div class="qt">3. The smartest delivery surface for model predictions at KNPC:</div> | |
| <label><input type="radio" name="c13q3" value="0">A new standalone AI portal</label><label><input type="radio" name="c13q3" value="1">Tables feeding the existing Power BI dashboards</label><label><input type="radio" name="c13q3" value="2">Email attachments</label> | |
| <div class="exp">Meet users where trust already exists. Batch job → table → dashboard.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Score each portfolio row for YOUR reality: adjust data readiness based on what you know about log quality at both plants.</li> | |
| <li>Draft the one-page pilot proposal for the demand forecast: baseline, scope, metric, gate — in your annual-plan format.</li> | |
| <li>Design the defect-photo collection SOP: angle, lighting, naming convention, label categories, storage location. One page, deployable this month.</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Which stakeholder loses convenience if these tools succeed, and how do you bring them along early?</li> | |
| <li>If you could only ship ONE application this year, which maximizes both plant value and your own capability growth?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Leading with Wave-3 ambitions in management presentations — then owning the failure.</li> | |
| <li>Building tools without a named operational owner for the post-you era.</li> | |
| <li>Ignoring the data-collection SOPs that unlock future waves.</li> | |
| <li>Measuring pilots by enthusiasm instead of the pre-agreed metric.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Wave 1</div>demand forecast · energy anomaly · OPEX variance</div> | |
| <div class="cell"><div class="k">Wave 2</div>spec-RAG · text-to-SQL · downtime classification</div> | |
| <div class="cell"><div class="k">Wave 3</div>vision QC · predictive maintenance · scheduling</div> | |
| <div class="cell"><div class="k">Unlock now</div>photo SOP + failure-cause codes</div> | |
| <div class="cell"><div class="k">Delivery</div>batch → table → Power BI</div> | |
| <div class="cell"><div class="k">Selling</div>baseline → pilot → metric → gate</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">PILOT PROPOSAL<button onclick="cp(this)">Copy</button></div><pre>Write a one-page pilot proposal for [application] at an LPG filling plant, in formal planning language: current-practice baseline and its cost, pilot scope (one plant, 3 months), data sources, success metric with target number, resources needed, risks with mitigations, and a go/no-go decision gate. Audience: division management. No AI hype vocabulary.</pre></div> | |
| <div class="prompt"><div class="ph">DATA COLLECTION SOP<button onclick="cp(this)">Copy</button></div><pre>Draft a standard operating procedure for collecting [defect photos / failure-cause codes] at a filling plant QC station: step-by-step capture instructions, naming and labeling convention, quality requirements, storage path, roles and responsibilities, and a weekly compliance check. Keep it to one page; operators are the audience.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>Hyndman & Athanasopoulos, <em>Forecasting: Principles and Practice</em> (3rd ed.) — otexts.com/fpp3 (demand forecasting methodology).</li> | |
| <li>Lee, Davari et al. (2018). "Industrial AI and predictive analytics in manufacturing" — <em>Manufacturing Letters</em> (industrial ML application patterns).</li> | |
| <li>Google, "Rules of Machine Learning" (M. Zinkevich) — developers.google.com (start-simple sequencing doctrine).</li> | |
| <li>McKinsey Global Institute reports on AI in operations — application economics framing (directional, not gospel).</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(12)">← Ch. 12</button><button class="btn done" onclick="done(13)">Mark complete → Ch. 14</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 14 ================= --> | |
| <article id="ch14" class="chx"> | |
| <div class="plate"><span class="tag">CH-14</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Your First Projects — Three Guided Builds</h2> | |
| <p class="ch-intro">Theory ends here. Three complete project walkthroughs, each with the exact structure, the Claude prompts to use, and the verification steps that keep you the engineer of record.</p> | |
| <h3><span class="sn">14.1</span>The universal project skeleton</h3> | |
| <table class="tb"> | |
| <tr><th>Phase</th><th>Deliverable</th><th>Chapters used</th><th>Share of effort</th></tr> | |
| <tr><td>1. Frame</td><td>One paragraph: decision served, prediction target, prediction moment, success metric vs baseline</td><td>1, 9</td><td>10%</td></tr> | |
| <tr><td>2. Data</td><td>Clean dataset + quality report + leakage check</td><td>4</td><td>35%</td></tr> | |
| <tr><td>3. EDA</td><td>3 findings, 3 charts, 3 decisions page</td><td>5</td><td>15%</td></tr> | |
| <tr><td>4. Model</td><td>Baseline + escalation ladder comparison table</td><td>7, 10</td><td>20%</td></tr> | |
| <tr><td>5. Evaluate</td><td>Right metric, unseen data, error analysis</td><td>9</td><td>10%</td></tr> | |
| <tr><td>6. Deliver</td><td>Saved pipeline + batch script + one-page summary</td><td>11</td><td>10%</td></tr> | |
| </table> | |
| <h3><span class="sn">14.2</span>Project A — Monthly cylinder demand forecast <span class="pill g">Wave 1 flagship</span></h3> | |
| <p><strong>Frame:</strong> Predict next-month 12kg cylinder demand per plant, at month-start, to set filling schedules and inventory. Success: MAPE < the seasonal-naive baseline ("same month last year") by at least 25% relative.</p> | |
| <ol> | |
| <li><strong>Data:</strong> 4–5 years of monthly filled quantities by plant and size + engineered features (Ch. 4): month, Ramadan overlap %, Eid flags, summer index, lags (1, 2, 12 months), rolling means. <em>Leakage check: every feature known on day 1 of the forecast month.</em></li> | |
| <li><strong>EDA:</strong> demand line per plant, seasonal decomposition, Ramadan-shift visualization (it moves ~11 days/year — your feature must track the Hijri calendar, not fixed months).</li> | |
| <li><strong>Model ladder:</strong> seasonal-naive baseline → linear with seasonality dummies → GBM with lags → (optional) Prophet with custom Ramadan/Eid events. Time-based split: train ≤2024, validate 2025.</li> | |
| <li><strong>Evaluate:</strong> MAE + MAPE per plant; error analysis on worst 5 months (expect: Ramadan boundary months and any subsidy/policy change).</li> | |
| <li><strong>Deliver:</strong> pipeline.pkl + monthly batch script writing to the Power BI table + one-page metric card.</li> | |
| </ol> | |
| <div class="prompt"><div class="ph">PROJECT A MASTER PROMPT<button onclick="cp(this)">Copy</button></div><pre>You are my ML pair programmer. Project: monthly LPG cylinder demand forecast for two plants, Kuwait (fiscal year Apr–Mar, Ramadan/Eid effects follow the Hijri calendar). I have monthly data: [describe columns/years]. Build it in phases and STOP after each phase for my review: 1) feature engineering incl. Hijri-aware Ramadan overlap fraction per Gregorian month, lags and rolling stats, with a leakage check table; 2) EDA with seasonal decomposition; 3) seasonal-naive baseline, then linear, then LightGBM — time-split train ≤2024 / test 2025, compare MAE and MAPE in one table; 4) error analysis of the 5 worst months; 5) save the full pipeline with joblib and write a monthly scoring script. Explain every step in simple English comments.</pre></div> | |
| <h3><span class="sn">14.3</span>Project B — Stoppage-cause classification <span class="pill o">Wave 2</span></h3> | |
| <p><strong>Frame:</strong> From stoppage log fields (line, duration, shift, preceding readings, free-text note), classify the cause category — building toward "which line stops next week". Success: F1 above the majority-class baseline; recall prioritized for the costliest cause.</p> | |
| <p><strong>The distinctive steps:</strong> standardize the cause labels first (the "GFB-U/GFBU" problem of Ch. 4 always lives here); use embeddings on the free-text notes as features (Ch. 8 meets Ch. 7); class weights for rare causes (Ch. 10); confusion matrix review with maintenance staff — their reading of the errors is the real evaluation (Ch. 9).</p> | |
| <div class="prompt"><div class="ph">PROJECT B MASTER PROMPT<button onclick="cp(this)">Copy</button></div><pre>Project: classify plant stoppage causes from a log with columns [list]. Free-text notes are mixed Arabic/English. Phase plan with stops for my review: 1) label audit — list all distinct cause spellings, propose a standardized mapping for my approval; 2) features: categorical encoding + multilingual sentence-embeddings of the notes; 3) Logistic → Random Forest with class weights, stratified 5-fold CV, report per-class precision/recall (NOT just accuracy — classes are imbalanced); 4) confusion matrix with my domain labels + the 10 most confident WRONG predictions for my expert review; 5) plain-English summary of which causes are predictable and which need better logging.</pre></div> | |
| <h3><span class="sn">14.4</span>Project C — Energy-per-cylinder anomaly monitor <span class="pill g">Wave 1</span></h3> | |
| <p><strong>Frame:</strong> Flag days where energy consumption per filled cylinder deviates abnormally — early warning for equipment degradation or metering issues. Success: operators confirm ≥50% of flags as "worth investigating" (precision proxy), with <3 flags/week (alarm-fatigue budget).</p> | |
| <p><strong>The distinctive steps:</strong> start with the 3σ control chart (Ch. 6) — deploy it, it's already valuable; then Isolation Forest on [energy/cylinder, volume, downtime, ambient temp] jointly to catch multivariate oddities the univariate chart misses; tune the contamination parameter to the alarm budget; every flag gets a one-line operator verdict — this feedback becomes labels for a future supervised model (the data flywheel).</p> | |
| <div class="prompt"><div class="ph">PROJECT C MASTER PROMPT<button onclick="cp(this)">Copy</button></div><pre>Project: daily energy-per-cylinder anomaly monitor. Data: daily [energy kWh, cylinders filled, downtime minutes, ambient temp] for [period]. Build: 1) 3-sigma control chart on energy/cylinder with clear matplotlib styling for a wall display; 2) Isolation Forest on all four variables, contamination tuned so total flags ≈ 2–3/week; 3) a comparison: which anomalies does each method catch, with dates listed; 4) a daily batch script that appends flags to a CSV with a blank "operator_verdict" column — explain how this column becomes training labels later. Simple English comments throughout.</pre></div> | |
| <div class="co plant"><div class="co-t">▲ The verification discipline (applies to all three)</div><p>For every AI-generated phase: (1) read the code and explain it back in one paragraph — if you can't, ask Claude to simplify until you can; (2) check one calculation by hand or in Excel; (3) run the leakage question on every feature; (4) confirm the metric beats the baseline on <em>unseen</em> data before telling anyone. You are the engineer of record; Claude is the contractor.</p></div> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>One skeleton fits all projects: frame → data → EDA → model ladder → evaluate → deliver.</li> | |
| <li>Framing fixes the prediction moment and the baseline BEFORE any code — everything downstream depends on it.</li> | |
| <li>Ramadan follows the Hijri calendar; your features must too. Domain detail beats model sophistication.</li> | |
| <li>Operator feedback loops (Project C's verdict column) turn unsupervised pilots into future supervised gold.</li> | |
| <li>Verification protocol: explain back, hand-check, leakage-check, baseline-check.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"I'll do all phases in one giant prompt"</summary><div class="dbody">Phase-gated prompting ("STOP after each phase") keeps you in control and catches errors before they compound.</div></details> | |
| <details><summary>"The project ends when the model works"</summary><div class="dbody">It ends when the batch script runs on schedule and someone uses the output. Phase 6 is the project.</div></details> | |
| <details><summary>"My first project should impress"</summary><div class="dbody">Your first project should FINISH. Completion teaches the full loop; ambition teaches abandonment.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>These ARE the workplace applications — Projects A and C are deployable within weeks on data you control.</li> | |
| <li>Each finished project becomes a template: swap the dataset, keep the skeleton.</li> | |
| <li>The one-page metric cards accumulate into your evidence file for the Wave-2 proposals.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="1"><div class="qt">1. Project A's Ramadan feature must be computed from:</div> | |
| <label><input type="radio" name="c14q1" value="0">Fixed Gregorian months</label><label><input type="radio" name="c14q1" value="1">The Hijri calendar overlap with each Gregorian month</label><label><input type="radio" name="c14q1" value="2">It doesn't matter</label> | |
| <div class="exp">Ramadan shifts ~11 days/year against the Gregorian calendar. A fixed-month flag decays into noise.</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">2. Project C limits flags to 2–3/week because:</div> | |
| <label><input type="radio" name="c14q2" value="0">Alarm fatigue kills adoption — a budget keeps flags credible</label><label><input type="radio" name="c14q2" value="1">The algorithm can't produce more</label><label><input type="radio" name="c14q2" value="2">Weekly reports look better</label> | |
| <div class="exp">An ignored monitor is a dead monitor. The alarm budget is a product decision (Ch. 9's threshold logic).</div></div> | |
| <div class="quiz-q" data-a="2"><div class="qt">3. "STOP after each phase for my review" exists so that:</div> | |
| <label><input type="radio" name="c14q3" value="0">Claude rests</label><label><input type="radio" name="c14q3" value="1">Prompts stay short</label><label><input type="radio" name="c14q3" value="2">You verify each stage before errors compound downstream</label> | |
| <div class="exp">Phase gates = your engineering hold points, applied to AI-assisted work.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Execute Project A end to end this month — it's the manual's graduation exam.</li> | |
| <li>Write the framing paragraph (decision, target, moment, metric, baseline) for a 4th project of your choosing.</li> | |
| <li>After Project A, write a half-page retrospective: where did you lose the most time, and which chapter would have saved it?</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Which project's output would YOUR manager notice first — and does that change your starting order?</li> | |
| <li>What does "engineer of record" mean to you when the code is 90% AI-written?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Skipping the framing paragraph and discovering mid-project that the prediction moment makes half the features illegal.</li> | |
| <li>Accepting phase outputs without the explain-back test.</li> | |
| <li>Random-splitting Project A (yes, again — it's the most repeated error in applied ML).</li> | |
| <li>Building all three at once; finish A before opening B.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Skeleton</div>frame → data → EDA → ladder → evaluate → deliver</div> | |
| <div class="cell"><div class="k">Project A</div>demand · Hijri features · time split · MAPE vs naive</div> | |
| <div class="cell"><div class="k">Project B</div>stoppage causes · label audit first · per-class metrics</div> | |
| <div class="cell"><div class="k">Project C</div>3σ chart → IsoForest · alarm budget · verdict column</div> | |
| <div class="cell"><div class="k">Verify</div>explain back · hand-check · leakage · baseline</div> | |
| <div class="cell"><div class="k">Rule</div>finish > impress</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">PHASE-GATE WRAPPER<button onclick="cp(this)">Copy</button></div><pre>For this entire project, operate in phase-gate mode: complete only the current phase, then STOP and give me: (a) what you did in 3 bullets, (b) what I should verify and how, (c) one risk you see. Do not proceed until I write "approved". Current phase: [X].</pre></div> | |
| <div class="prompt"><div class="ph">RETROSPECTIVE<button onclick="cp(this)">Copy</button></div><pre>Project finished. Here's what happened: [notes]. Write an honest retrospective: what consumed the most time vs its value, which decisions were wrong in hindsight, what to templatize for next time, and 3 specific skills to strengthen based on where I struggled.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>Hyndman & Athanasopoulos, <em>FPP3</em> — otexts.com/fpp3 (baseline & accuracy methodology for Project A).</li> | |
| <li>scikit-learn: TimeSeriesSplit, class_weight, IsolationForest docs — scikit-learn.org.</li> | |
| <li>Taylor & Letham (2018), Prophet — custom seasonality/holiday events.</li> | |
| <li>Reimers & Gurevych (2019). "Sentence-BERT." <em>EMNLP</em> (multilingual embeddings for Project B).</li> | |
| <li>NIST/SEMATECH e-Handbook — control chart construction for Project C.</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(13)">← Ch. 13</button><button class="btn done" onclick="done(14)">Mark complete → Ch. 15</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 15 ================= --> | |
| <article id="ch15" class="chx"> | |
| <div class="plate"><span class="tag">CH-15</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Working With AI Assistants</h2> | |
| <p class="ch-intro">Your force multiplier, used professionally. How to direct, verify, and debug with Claude/ChatGPT so that AI-written code carries your engineering signature.</p> | |
| <h3><span class="sn">15.1</span>The collaboration model</h3> | |
| <p>Treat the assistant as a <strong>brilliant, tireless contractor with no site knowledge and occasional confident errors</strong>. Contractors get: clear scopes (framing paragraphs), hold points (phase gates), inspections (verification), and documentation requirements (comments in simple English). The quality of AI output is a direct function of the quality of your direction — which is why Chapters 1–14 make you a better AI user: you now know what to ask for and what to distrust.</p> | |
| <h3><span class="sn">15.2</span>Hallucination detection in code & analysis</h3> | |
| <table class="tb"> | |
| <tr><th>Red flag</th><th>What it looks like</th><th>Your check</th></tr> | |
| <tr><td>Phantom functions/parameters</td><td>Code calls a library function that doesn't exist (plausible name, wrong reality)</td><td>Run it — <code>AttributeError</code>/<code>TypeError</code> exposes it instantly; or check the official docs</td></tr> | |
| <tr><td>Too-smooth results</td><td>Every step "works", metrics look great first try</td><td>Great first-try metrics = leakage until proven otherwise (Ch. 4)</td></tr> | |
| <tr><td>Invented facts/citations</td><td>Confident references, standards, or numbers you can't trace</td><td>Ask "give the exact source"; verify independently before it enters a KNPC document</td></tr> | |
| <tr><td>Silent assumption changes</td><td>You said fiscal year; the code quietly uses calendar year</td><td>Ask for an "assumptions list" with every deliverable; diff it against your framing</td></tr> | |
| <tr><td>Agreement drift</td><td>You push back; it capitulates even when it was right</td><td>Ask "steelman both options" instead of "am I right?"</td></tr> | |
| </table> | |
| <h3><span class="sn">15.3</span>The debugging protocol</h3> | |
| <pre class="code"><span class="c"># When AI code fails — in this order</span> | |
| <span class="k">1.</span> Paste the FULL traceback, not the last line | |
| <span class="k">2.</span> Include: what you ran, what you expected, what happened | |
| <span class="k">3.</span> Ask for the cause in ONE sentence before any fix <span class="c"># forces diagnosis over guessing</span> | |
| <span class="k">4.</span> If two fixes fail: "step back — question your approach, | |
| list 3 alternative root causes, rank by likelihood" <span class="c"># breaks the patch-loop</span> | |
| <span class="k">5.</span> After the fix works: "explain what was wrong so I | |
| recognize this class of error next time" <span class="c"># convert bugs into skill</span></pre> | |
| <h3><span class="sn">15.4</span>Testing & refactoring with AI</h3> | |
| <ul> | |
| <li><strong>Tests</strong> — ask for pytest tests with every non-trivial function: normal case, edge case (empty data, single row, missing month), and a deliberately wrong input that must raise an error. Run tests before trusting any refactor.</li> | |
| <li><strong>Refactoring</strong> — "same behavior, better structure": ask for it only AFTER tests exist, then run tests to prove behavior held. This is how notebooks graduate into the batch scripts of Ch. 11.</li> | |
| <li><strong>Code review</strong> — paste working code and ask for a review against a checklist: correctness risks, leakage, hard-coded values, missing error handling, unclear names. A second AI session reviewing the first one's code catches a surprising amount.</li> | |
| </ul> | |
| <h3><span class="sn">15.5</span>Session craft</h3> | |
| <ul> | |
| <li><strong>Context files beat re-explaining</strong> — maintain a project brief (your /handoff pattern is exactly this) and paste it at session start.</li> | |
| <li><strong>One session, one mission</strong> — mixing the forecast project with dashboard styling degrades both; long mixed sessions drift.</li> | |
| <li><strong>Checkpoint the state</strong> — at milestones, ask for a "current state summary + exact next step" and save it (your /save pattern). Sessions die; briefings persist.</li> | |
| <li><strong>You own the domain</strong> — never let generated text about LPG operations, Kuwait context, or KNPC procedure pass without your correction. The model writes; you author.</li> | |
| </ul> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>AI = brilliant contractor, no site knowledge: give scopes, hold points, inspections.</li> | |
| <li>Five hallucination red flags; the universal antidotes are "run it", "source it", and "list your assumptions".</li> | |
| <li>Debug protocol: full traceback → one-sentence diagnosis → escalate to root-cause listing after two failed patches.</li> | |
| <li>Tests before refactors; review sessions catch the builder session's blind spots.</li> | |
| <li>Session craft: briefs in, checkpoints out, one mission per session.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"AI code that runs is code that's right"</summary><div class="dbody">Running means syntactically valid. Right means correct logic, no leakage, sane assumptions — that's your inspection.</div></details> | |
| <details><summary>"Longer conversations = smarter assistant"</summary><div class="dbody">Long mixed sessions accumulate stale context and drift. Fresh session + good brief beats a marathon.</div></details> | |
| <details><summary>"Asking AI to check its own work is pointless"</summary><div class="dbody">Self-critique passes and fresh-session reviews measurably catch errors — cheap insurance you should always buy.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>A verification SOP for AI-assisted work in your division — the quality procedure nobody has written yet.</li> | |
| <li>Project briefs as standing artifacts (you already run /handoff — formalize it per KNPC project).</li> | |
| <li>Review-session ritual before any AI-built tool touches real planning data.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="2"><div class="qt">1. First-try model shows 99.2% accuracy. Professional reflex:</div> | |
| <label><input type="radio" name="c15q1" value="0">Celebrate and present it</label><label><input type="radio" name="c15q1" value="1">Ask for 99.5%</label><label><input type="radio" name="c15q1" value="2">Suspect leakage or evaluation error; audit before believing</label> | |
| <div class="exp">Too-good-too-fast is the signature of leakage (Ch. 4) or broken evaluation (Ch. 9).</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">2. Two AI fixes in a row failed. Best next prompt:</div> | |
| <label><input type="radio" name="c15q2" value="0">"Step back — list 3 alternative root causes ranked by likelihood"</label><label><input type="radio" name="c15q2" value="1">"Try again"</label><label><input type="radio" name="c15q2" value="2">"Fix it properly this time"</label> | |
| <div class="exp">Breaks the patch-loop by forcing re-diagnosis instead of a third guess.</div></div> | |
| <div class="quiz-q" data-a="1"><div class="qt">3. Refactoring is safe when:</div> | |
| <label><input type="radio" name="c15q3" value="0">The code looks cleaner</label><label><input type="radio" name="c15q3" value="1">Tests written beforehand still pass afterward</label><label><input type="radio" name="c15q3" value="2">The AI says it preserved behavior</label> | |
| <div class="exp">"Same behavior" is provable only by tests that existed before the change.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Take any AI code you accepted this month; run the review-checklist prompt on it in a fresh session. Count the findings.</li> | |
| <li>Practice the debugging protocol on a deliberately broken script — including step 4's root-cause escalation.</li> | |
| <li>Write your personal AI-verification SOP: 5 checks, one page, pinned next to your monitor.</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Where in your workflow do you currently accept AI output on trust — and what's the worst case if it's wrong there?</li> | |
| <li>Which of your engineering habits (hold points, checklists, sign-offs) transfer directly to AI collaboration?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Pasting the last error line instead of the full traceback.</li> | |
| <li>Letting the assistant "fix" by rewriting everything — diff discipline lost.</li> | |
| <li>Accepting apologetic capitulation as confirmation you were right.</li> | |
| <li>No project brief: re-explaining context badly every session, getting inconsistent architecture.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">Model</div>brilliant contractor, no site knowledge</div> | |
| <div class="cell"><div class="k">Red flags</div>phantom APIs · too-smooth · unsourced facts · silent assumptions</div> | |
| <div class="cell"><div class="k">Debug</div>full traceback → 1-sentence cause → root-cause list after 2 fails</div> | |
| <div class="cell"><div class="k">Safety</div>tests → refactor → tests again</div> | |
| <div class="cell"><div class="k">Sessions</div>brief in · checkpoint out · one mission</div> | |
| <div class="cell"><div class="k">Authority</div>the model writes; you author</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">CODE REVIEW CHECKLIST<button onclick="cp(this)">Copy</button></div><pre>Review this code as a skeptical senior engineer. Checklist: 1) correctness risks and edge cases, 2) data leakage anywhere in the pipeline, 3) hard-coded values that should be parameters, 4) missing error handling, 5) misleading names/comments. Rate each finding severity high/medium/low. Do NOT rewrite the code yet — findings only. [paste code]</pre></div> | |
| <div class="prompt"><div class="ph">ASSUMPTIONS AUDIT<button onclick="cp(this)">Copy</button></div><pre>List every assumption embedded in the work you just produced: about the data, the time ranges, calendars (fiscal vs calendar, Hijri effects), units, business rules, and defaults you chose silently. Format: assumption | where it appears | risk if wrong.</pre></div> | |
| <div class="prompt"><div class="ph">TEST GENERATOR<button onclick="cp(this)">Copy</button></div><pre>Write pytest tests for this function: one normal case with hand-computed expected values, edge cases (empty input, single row, missing values), and one invalid input that must raise a clear error. Explain what each test protects against. [paste function]</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>Anthropic — Claude documentation & prompt engineering guides — docs.claude.com.</li> | |
| <li>Chen et al. (2021). "Evaluating Large Language Models Trained on Code." <em>arXiv:2107.03374</em> (Codex — capabilities & error modes).</li> | |
| <li>Ji et al. (2023). "Survey of Hallucination in Natural Language Generation." <em>ACM Computing Surveys</em>.</li> | |
| <li>pytest documentation — docs.pytest.org.</li> | |
| <li>Fowler, M. <em>Refactoring</em> (2nd ed.) — behavior-preserving change discipline.</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(14)">← Ch. 14</button><button class="btn done" onclick="done(15)">Mark complete → Ch. 16</button></div> | |
| </article> | |
| <!-- ================= CHAPTER 16 ================= --> | |
| <article id="ch16" class="chx"> | |
| <div class="plate"><span class="tag">CH-16</span><span class="status">COMMISSIONED</span></div> | |
| <h2 class="ch">Your Learning Roadmap</h2> | |
| <p class="ch-intro">The manual ends; the practice begins. A time-phased plan calibrated to a working engineer's calendar — roughly 5 focused hours a week, compounding for two years.</p> | |
| <h3><span class="sn">16.1</span>The phased plan</h3> | |
| <table class="tb"> | |
| <tr><th>Horizon</th><th>Mission</th><th>Concrete outputs</th><th>Manual chapters in play</th></tr> | |
| <tr><td><strong>Days 1–30</strong></td><td>Environment + fluency</td><td>Colab running · Ch. 2–5 exercises done on one REAL work dataset · first EDA one-pager delivered to yourself</td><td>2, 3, 4, 5</td></tr> | |
| <tr><td><strong>Days 31–60</strong></td><td>First model, honestly evaluated</td><td>Project A (demand forecast) through phase 4 · baseline beaten and documented · error analysis written</td><td>6, 7, 9, 14</td></tr> | |
| <tr><td><strong>Days 61–90</strong></td><td>Ship something</td><td>Project A deployed as a scheduled batch job feeding Power BI · Model Operating Procedure page signed · Project C (3σ chart) live</td><td>10, 11, 14</td></tr> | |
| <tr><td><strong>Months 4–6</strong></td><td>The LLM layer</td><td>Spec-RAG pilot with golden-question evaluation · text-to-SQL hardened with guardrails · one Kaggle "Playground" competition entered (for calibration against the world)</td><td>8, 12, 13, 15</td></tr> | |
| <tr><td><strong>Year 1</strong></td><td>Portfolio + credibility</td><td>3 deployed tools with metric cards · Wave-3 data collection running 6+ months · one internal talk: "what ML actually did for GFB"</td><td>All</td></tr> | |
| <tr><td><strong>Year 2</strong></td><td>Depth + leadership</td><td>Chosen specialization (forecasting, industrial LLM systems, or vision QC) studied to depth · you review others' AI-assisted work · Wave 3 unblocked by the data you seeded</td><td>Beyond the manual</td></tr> | |
| </table> | |
| <div class="co warn"><div class="co-t">⚠ The honest failure mode</div><p>The plan above fails one way: courses without projects. Every horizon is defined by <em>outputs</em>, not by content consumed. If a week produced no artifact — code, page, chart, decision — it didn't count, whatever you watched.</p></div> | |
| <h3><span class="sn">16.2</span>Curated resources (deliberately short)</h3> | |
| <table class="tb"> | |
| <tr><th>Type</th><th>Resource</th><th>When</th></tr> | |
| <tr><td>Course</td><td><strong>Machine Learning Specialization</strong> — Andrew Ng, DeepLearning.AI/Coursera</td><td>Months 1–3, alongside projects (the theory backbone)</td></tr> | |
| <tr><td>Book</td><td><strong>Hands-On Machine Learning</strong> (3rd ed.) — Aurélien Géron, O'Reilly</td><td>The permanent desk reference; read chapters as projects demand</td></tr> | |
| <tr><td>Book (free)</td><td><strong>Forecasting: Principles and Practice</strong> — Hyndman & Athanasopoulos, otexts.com/fpp3</td><td>Before/during Project A — the forecasting bible</td></tr> | |
| <tr><td>Book (free)</td><td><strong>An Introduction to Statistical Learning</strong> — statlearning.com</td><td>Months 4+, when you want the "why" behind Ch. 7</td></tr> | |
| <tr><td>Practice</td><td><strong>Kaggle</strong> — Learn courses, then Playground competitions</td><td>Month 4+; calibrates your skill against reality</td></tr> | |
| <tr><td>Course</td><td><strong>DeepLearning.AI short courses</strong> (RAG, LLMOps, prompt engineering)</td><td>Months 4–6, feeding the Wave-2 builds</td></tr> | |
| <tr><td>Community</td><td>r/MachineLearning · Hugging Face forums · Kaggle discussions</td><td>Ongoing; lurk, then ask, then answer</td></tr> | |
| <tr><td>Reference</td><td>scikit-learn User Guide · Anthropic docs</td><td>Always open in a tab</td></tr> | |
| </table> | |
| <h3><span class="sn">16.3</span>Staying current without drowning</h3> | |
| <p>The field moves fast at the frontier and slowly at the foundations. Everything in Chapters 1–11 will still be true in five years; Chapter 12's tool names will churn. Strategy: <strong>foundations deep, frontier sampled</strong> — one weekly digest (your Scout agent already does this), one experiment per month with something new, and zero guilt about ignoring the rest. Depth on your three specialization candidates beats breadth across everything.</p> | |
| <div class="co plant"><div class="co-t">▲ Final word</div><p>You started this manual as a Planning Engineer who uses AI. You finish it knowing the full pipeline: what ML is, how data becomes models, how models are judged, shipped, and kept alive, and how to command AI assistants with an engineer's skepticism. The 16 valves on your manifold are open. Now go build Project A — the plant is waiting.</p></div> | |
| <div class="eoc"> | |
| <div class="eoc-block"><div class="hd"><span class="ic">01</span>Key Takeaways</div><div class="bd"><ul> | |
| <li>Outputs define progress: every horizon has artifacts, not watch-lists.</li> | |
| <li>90 days: fluency → honest model → shipped batch job. That's a complete practitioner loop.</li> | |
| <li>Eight resources suffice; the constraint is focused hours, not content supply.</li> | |
| <li>Foundations deep, frontier sampled — Ch. 1–11 age slowly; tool names churn.</li> | |
| <li>Year 2 is specialization + reviewing others — the transition from practitioner to leader.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">02</span>Beginner Misunderstandings</div><div class="bd"> | |
| <details><summary>"I'll start after finishing one more course"</summary><div class="dbody">The course queue is infinite by design. Project A requires only Chapters 1–9 of this manual — which you now have.</div></details> | |
| <details><summary>"Two years is too slow"</summary><div class="dbody">Two years of shipped tools at 5 hrs/week beats any bootcamp. You're compounding domain expertise WITH ML — the rare combination.</div></details> | |
| <details><summary>"I must keep up with every model release"</summary><div class="dbody">You must keep your tools working and your foundations sharp. Scout samples the frontier for you.</div></details> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">03</span>Workplace Applications</div><div class="bd"><ul> | |
| <li>Fold the 90-day plan into your personal objectives for the next review cycle — make the learning officially visible.</li> | |
| <li>The Year-1 internal talk doubles as the Wave-2 budget pitch.</li> | |
| <li>Reviewing colleagues' AI-assisted work (Year 2) is how the division's quality bar gets set — by you.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">04</span>Mini Quiz</div><div class="bd"> | |
| <div class="quiz-q" data-a="1"><div class="qt">1. A week of tutorials, zero artifacts. Per this roadmap, that week:</div> | |
| <label><input type="radio" name="c16q1" value="0">Counts — learning happened</label><label><input type="radio" name="c16q1" value="1">Didn't count — progress is measured in outputs</label><label><input type="radio" name="c16q1" value="2">Counts double if videos were 2×</label> | |
| <div class="exp">Course-consumption without artifacts is the plan's one failure mode.</div></div> | |
| <div class="quiz-q" data-a="2"><div class="qt">2. "Foundations deep, frontier sampled" means:</div> | |
| <label><input type="radio" name="c16q2" value="0">Ignore new developments entirely</label><label><input type="radio" name="c16q2" value="1">Chase every release</label><label><input type="radio" name="c16q2" value="2">Master the stable core; experiment monthly with the new; skip the rest guilt-free</label> | |
| <div class="exp">Ch. 1–11 barely age; tool churn is delegated to a weekly digest and one monthly experiment.</div></div> | |
| <div class="quiz-q" data-a="0"><div class="qt">3. The Day-90 milestone is:</div> | |
| <label><input type="radio" name="c16q3" value="0">A scheduled model feeding Power BI, with an operating procedure</label><label><input type="radio" name="c16q3" value="1">Finishing three courses</label><label><input type="radio" name="c16q3" value="2">Reading all 16 chapters twice</label> | |
| <div class="exp">Ship something that runs without you watching it — the complete loop, once.</div></div> | |
| <div class="q-actions"><button class="btn" onclick="grade(this)">Check answers</button><span class="score"></span></div> | |
| </div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">05</span>Exercises</div><div class="bd"><ol> | |
| <li>Block the recurring 5 hours in your calendar now — the plan is a calendar artifact or it's fiction.</li> | |
| <li>Write your Day-30, Day-60, Day-90 outputs as three one-line commitments; pin them where you work.</li> | |
| <li>Enroll in the Ng specialization and open Project A's Colab — today, both under 15 minutes.</li> | |
| </ol></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">06</span>Reflection</div><div class="bd"><ul> | |
| <li>Which of the three Year-2 specializations (forecasting, industrial LLM systems, vision QC) pulls you — and which does GFB need most?</li> | |
| <li>Who at KNPC should you teach first — because teaching is the fastest consolidation of everything in this manual?</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">07</span>Common Mistakes</div><div class="bd"><ul> | |
| <li>Course-collecting (the tutorial treadmill) — the plan's named enemy.</li> | |
| <li>Restarting from zero after a busy month instead of resuming mid-plan.</li> | |
| <li>Learning in private: no shipped tools, no talk, no visibility — compounding forfeited.</li> | |
| <li>Chasing frontier tools before Project A runs on schedule.</li> | |
| </ul></div></div> | |
| <div class="eoc-block"><div class="hd"><span class="ic">08</span>Cheat Sheet</div><div class="bd"><div class="cs"> | |
| <div class="cell"><div class="k">30d</div>Colab + Ch.2–5 on real data + EDA page</div> | |
| <div class="cell"><div class="k">60d</div>Project A modeled, baseline beaten</div> | |
| <div class="cell"><div class="k">90d</div>Project A shipped + 3σ monitor live</div> | |
| <div class="cell"><div class="k">6mo</div>RAG pilot + text-to-SQL hardened + 1 Kaggle</div> | |
| <div class="cell"><div class="k">1yr</div>3 deployed tools + internal talk</div> | |
| <div class="cell"><div class="k">2yr</div>specialization + reviewer role + Wave 3</div> | |
| </div></div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">09</span>AI Prompts</div><div class="bd"> | |
| <div class="prompt"><div class="ph">WEEKLY PLANNER<button onclick="cp(this)">Copy</button></div><pre>I'm in [phase] of my ML roadmap with 5 hours this week. Last week I produced: [artifacts]. Currently blocked on: [if any]. Plan this week's 5 hours into 2–3 sessions with a concrete artifact per session, and tell me what to explicitly SKIP this week.</pre></div> | |
| <div class="prompt"><div class="ph">MONTHLY REVIEW<button onclick="cp(this)">Copy</button></div><pre>Month review. Planned outputs: [list]. Actual outputs: [list]. Honestly assess: am I on the 90-day track? What's the single highest-leverage correction for next month? Be direct — no encouragement padding.</pre></div> | |
| </div></div> | |
| <div class="eoc-block clps"><div class="hd" onclick="tg(this)"><span class="ic">REF</span>Bibliography & Master Resource List</div><div class="bd" style="font-size:13.5px"><ul style="margin-left:18px"> | |
| <li>Ng, A. — Machine Learning Specialization, DeepLearning.AI / Coursera.</li> | |
| <li>Géron, A. (2022). <em>Hands-On Machine Learning with Scikit-Learn, Keras & TensorFlow</em>, 3rd ed. O'Reilly.</li> | |
| <li>James, Witten, Hastie, Tibshirani — <em>An Introduction to Statistical Learning</em> — statlearning.com (free).</li> | |
| <li>Hyndman & Athanasopoulos — <em>Forecasting: Principles and Practice</em>, 3rd ed. — otexts.com/fpp3 (free).</li> | |
| <li>Huyen, C. (2022). <em>Designing Machine Learning Systems.</em> O'Reilly.</li> | |
| <li>kaggle.com/learn · huggingface.co/learn · scikit-learn.org · docs.claude.com.</li> | |
| <li>Plus: every chapter's bibliography above — the full manual cites ~60 primary sources.</li> | |
| </ul></div></div> | |
| </div> | |
| <div class="chfoot"><button class="ghost" onclick="go(15)">← Ch. 15</button><button class="btn done" onclick="done(16)">Finish the manual ✓</button></div> | |
| </article> | |
| </main> | |
| </div> | |
| <script> | |
| var CH=16; | |
| function $(s,c){return (c||document).querySelector(s)} | |
| function $$(s,c){return Array.prototype.slice.call((c||document).querySelectorAll(s))} | |
| var completed={}; | |
| function go(n){ | |
| $$('.chx').forEach(function(a){a.classList.remove('active')}); | |
| var cover=$('.cover'); | |
| if(n<1){ if(cover)cover.classList.remove('hidden'); } | |
| else{ | |
| if(cover)cover.classList.add('hidden'); | |
| var art=$('#ch'+n); if(art)art.classList.add('active'); | |
| } | |
| $$('#toc a').forEach(function(a){a.classList.toggle('live', +a.dataset.ch===n)}); | |
| window.scrollTo({top:0,behavior:'instant'}); | |
| } | |
| function done(n){ | |
| completed[n]=true; | |
| var v=$('#v'+n); if(v)v.classList.add('open'); | |
| var t=$('#toc a[data-ch="'+n+'"]'); if(t)t.classList.add('done'); | |
| var mc=$('#mcount'); if(mc)mc.textContent=Object.keys(completed).length; | |
| if(n<CH){ go(n+1); } | |
| else{ | |
| go(0); | |
| var cover=$('.cover'); | |
| if(cover && !$('#donebanner')){ | |
| var b=document.createElement('div'); | |
| b.id='donebanner'; | |
| b.style.cssText='margin-top:26px;padding:18px 22px;border:2px solid var(--gauge);border-radius:12px;background:var(--gauge-soft);font-family:var(--disp);font-weight:800;color:var(--gauge);font-size:17px'; | |
| b.innerHTML='✓ ALL 16 CHAPTERS COMMISSIONED — Manual complete. Now go ship Project A.'; | |
| cover.appendChild(b); | |
| } | |
| } | |
| } | |
| function tg(hd){ hd.closest('.eoc-block').classList.toggle('open'); } | |
| function grade(btn){ | |
| var box = btn && btn.closest ? btn.closest('.bd') : null; | |
| if(!box) box = $('#quiz'); | |
| var qs=$$('.quiz-q',box), ok=0; | |
| qs.forEach(function(q){ | |
| q.classList.add('revealed'); | |
| var a=q.dataset.a, sel=$('input:checked',q); | |
| $$('label',q).forEach(function(l){l.classList.remove('right','picked')}); | |
| $$('label',q).forEach(function(l){ | |
| var inp=$('input',l); | |
| if(inp.value===a)l.classList.add('right'); | |
| if(inp.checked && inp.value!==a)l.classList.add('picked'); | |
| }); | |
| if(sel && sel.value===a){ok++; q.classList.add('correct'); q.classList.remove('wrong');} | |
| else{q.classList.add('wrong'); q.classList.remove('correct');} | |
| }); | |
| var sc=$('.score',box)||$('#score'); | |
| if(sc) sc.textContent=ok+' / '+qs.length+(ok===qs.length?' ✓ full marks':''); | |
| } | |
| function cp(b){ | |
| var pre=b.closest('.prompt').querySelector('pre'); | |
| var t=pre.textContent; | |
| if(navigator.clipboard&&navigator.clipboard.writeText){navigator.clipboard.writeText(t)} | |
| else{var ta=document.createElement('textarea');ta.value=t;document.body.appendChild(ta);ta.select();try{document.execCommand('copy')}catch(e){}document.body.removeChild(ta)} | |
| var old=b.textContent;b.textContent='Copied ✓';setTimeout(function(){b.textContent=old},1400); | |
| } | |
| function pick(i){ | |
| $$('.picker .ans').forEach(function(a,idx){a.classList.toggle('show', idx===i)}); | |
| } | |
| document.addEventListener('DOMContentLoaded',function(){ | |
| $$('#toc a').forEach(function(a){a.addEventListener('click',function(){go(+a.dataset.ch)})}); | |
| }); | |
| </script> | |
| </body> | |
| </html> | |