Spaces:
Running
Running
Download index.html from qliao/StreamingNorm: direct link, hf CLI and curl.
- Browser
- Download file 44.9 kB
-
https://huggingface.co/spaces/qliao/StreamingNorm/resolve/main/index.html
- Command line
-
hf download hf://spaces/qliao/StreamingNorm/index.html
-
curl -L -o index.html https://huggingface.co/spaces/qliao/StreamingNorm/resolve/main/index.html
44.9 kB
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>Streaming Normalization — Liao, Kawaguchi & Poggio (2016)</title> | |
| <meta name="description" content="A reader's guide to Streaming Normalization: normalization statistics collected online across all past samples and timesteps. arXiv:1610.06160 / CBMM Memo 057."> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> | |
| <link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=Newsreader:ital,opsz,wght@0,6..72,400;0,6..72,500;1,6..72,400&family=Space+Grotesk:wght@500;700&display=swap" rel="stylesheet"> | |
| <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.16.9/dist/katex.min.css" integrity="sha384-n8MVd4RsNIU0tAv4ct0nTaAbDJwPJzDEaqSD1odI+WdtXRGWt2kTvGFasHpSy3SV" crossorigin="anonymous"> | |
| <style> | |
| :root{ | |
| --paper:#E8EBE4; | |
| --card:#F3F5EF; | |
| --ink:#14191C; | |
| --ink-2:#4B565C; | |
| --rule:#C6CCC0; | |
| --teal:#14625E; | |
| --plum:#7B2E52; | |
| --gold:#A87D20; | |
| --maxw:720px; | |
| } | |
| *{box-sizing:border-box} | |
| html{-webkit-text-size-adjust:100%} | |
| body{ | |
| margin:0; | |
| background:var(--paper); | |
| color:var(--ink); | |
| font-family:"Newsreader",Georgia,serif; | |
| font-size:19px; | |
| line-height:1.62; | |
| font-optical-sizing:auto; | |
| } | |
| /* ---------- shared type ---------- */ | |
| .eyebrow{ | |
| font-family:"IBM Plex Mono",ui-monospace,monospace; | |
| font-size:.7rem; | |
| font-weight:500; | |
| letter-spacing:.16em; | |
| text-transform:uppercase; | |
| color:var(--ink-2); | |
| } | |
| h1,h2,h3{font-family:"Space Grotesk",system-ui,sans-serif;font-weight:700;letter-spacing:-.02em;line-height:1.1;margin:0} | |
| a{color:var(--teal);text-decoration:none;border-bottom:1px solid rgba(20,98,94,.32)} | |
| a:hover{border-bottom-color:var(--teal)} | |
| a:focus-visible,button:focus-visible,input:focus-visible{outline:2px solid var(--plum);outline-offset:3px} | |
| code,kbd{font-family:"IBM Plex Mono",ui-monospace,monospace;font-size:.86em} | |
| .wrap{max-width:var(--maxw);margin:0 auto;padding:0 24px} | |
| .wide{max-width:980px;margin:0 auto;padding:0 24px} | |
| /* ---------- top bar ---------- */ | |
| .topbar{ | |
| border-bottom:1px solid var(--rule); | |
| background:rgba(232,235,228,.86); | |
| backdrop-filter:blur(6px); | |
| position:sticky;top:0;z-index:10; | |
| } | |
| .topbar .wide{display:flex;align-items:center;justify-content:space-between;gap:16px;height:52px} | |
| .topbar .eyebrow{white-space:nowrap;overflow:hidden;text-overflow:ellipsis} | |
| .topbar nav{display:flex;gap:18px} | |
| .topbar nav a{border:0;font-family:"IBM Plex Mono",monospace;font-size:.72rem;letter-spacing:.08em;text-transform:uppercase;color:var(--ink-2)} | |
| .topbar nav a:hover{color:var(--ink)} | |
| /* ---------- disclaimer ---------- */ | |
| .disclaimer{ | |
| border-bottom:1px solid var(--rule); | |
| background:#DFE3D9; | |
| } | |
| .disclaimer .wide{ | |
| display:flex;gap:14px;align-items:flex-start; | |
| padding-top:14px;padding-bottom:14px; | |
| } | |
| .disclaimer .flag{ | |
| font-family:"IBM Plex Mono",monospace;font-size:.66rem;font-weight:500; | |
| letter-spacing:.14em;text-transform:uppercase;color:var(--plum); | |
| border:1px solid var(--plum);padding:3px 8px;border-radius:2px; | |
| white-space:nowrap;margin-top:2px; | |
| } | |
| .disclaimer p{ | |
| margin:0;font-size:.88rem;line-height:1.5;color:var(--ink-2);max-width:70ch; | |
| } | |
| .disclaimer a{color:var(--plum);border-bottom-color:rgba(123,46,82,.35)} | |
| @media (max-width:560px){ | |
| .disclaimer .wide{flex-direction:column;gap:10px} | |
| } | |
| /* ---------- hero ---------- */ | |
| header.hero{padding:64px 0 8px} | |
| .kicker{display:flex;flex-wrap:wrap;gap:10px 18px;margin-bottom:26px} | |
| h1{font-size:clamp(2.1rem,5.4vw,3.5rem);max-width:16ch} | |
| h1 .lede{display:block;font-size:.42em;font-weight:500;letter-spacing:-.005em;color:var(--ink-2);margin-top:18px;max-width:34ch;line-height:1.3} | |
| .byline{margin:26px 0 0;font-size:1.02rem} | |
| .byline .aff{color:var(--ink-2);font-size:.92rem;display:block;margin-top:4px} | |
| .links{display:flex;flex-wrap:wrap;gap:10px;margin:26px 0 0;padding:0;list-style:none} | |
| .links a{ | |
| border:1px solid var(--rule);background:var(--card); | |
| padding:7px 14px;border-radius:2px; | |
| font-family:"IBM Plex Mono",monospace;font-size:.74rem;letter-spacing:.05em;text-transform:uppercase;color:var(--ink) | |
| } | |
| .links a:hover{border-color:var(--ink);background:var(--paper)} | |
| .links a.code{border-color:var(--teal);color:var(--teal)} | |
| .links a.code:hover{background:var(--teal);color:var(--card);border-color:var(--teal)} | |
| /* ---------- signature: the stream ---------- */ | |
| .stream{margin:56px auto 0;max-width:980px;padding:0 24px} | |
| .stream-frame{ | |
| border:1px solid var(--rule); | |
| background:var(--card); | |
| padding:18px 18px 14px; | |
| } | |
| .stream-head{display:flex;flex-wrap:wrap;gap:12px 24px;align-items:baseline;justify-content:space-between;margin-bottom:14px} | |
| canvas{display:block;width:100%;height:250px;touch-action:pan-y;cursor:grab;border-radius:2px} | |
| canvas.grabbing{cursor:grabbing} | |
| canvas:focus-visible{outline:2px solid var(--plum);outline-offset:2px} | |
| .legend{display:flex;flex-wrap:wrap;gap:10px 22px;margin-top:14px;font-family:"IBM Plex Mono",monospace;font-size:.72rem;color:var(--ink-2)} | |
| .legend span{display:flex;align-items:center;gap:9px} | |
| .legend b{font-weight:500;color:var(--ink)} | |
| .sw{width:24px;border-top-style:solid;border-top-width:2px;display:inline-block;flex:none} | |
| .sw.dots{border-top-style:dotted;border-top-width:3px} | |
| .sw.dash{border-top-style:dashed;border-top-width:2px} | |
| .sw.thick{border-top-width:4px} | |
| .controls{display:flex;flex-wrap:wrap;gap:14px 22px;align-items:center;margin-top:16px;padding-top:14px;border-top:1px solid var(--rule)} | |
| .controls label{font-family:"IBM Plex Mono",monospace;font-size:.72rem;letter-spacing:.05em;color:var(--ink-2);display:flex;align-items:center;gap:9px} | |
| input[type=range]{width:120px;accent-color:var(--gold)} | |
| .controls .chk{cursor:pointer;text-transform:none;letter-spacing:.02em} | |
| .controls .chk input{accent-color:var(--teal);width:14px;height:14px;margin:0} | |
| #live{border-color:var(--plum);color:var(--plum)} | |
| #live[hidden]{display:none} | |
| button{ | |
| font-family:"IBM Plex Mono",monospace;font-size:.72rem;letter-spacing:.08em;text-transform:uppercase; | |
| border:1px solid var(--rule);background:var(--paper);color:var(--ink); | |
| padding:7px 14px;border-radius:2px;cursor:pointer | |
| } | |
| button:hover{border-color:var(--ink)} | |
| .caption{font-size:.86rem;color:var(--ink-2);margin:12px auto 0;max-width:820px;line-height:1.5} | |
| /* ---------- sections ---------- */ | |
| section{padding:60px 0 0} | |
| .sec-head{margin-bottom:22px} | |
| .sec-head h2{font-size:1.6rem;margin-top:8px} | |
| p{margin:0 0 1.05em} | |
| .wrap > p:last-child{margin-bottom:0} | |
| ul.plain{list-style:none;padding:0;margin:0 0 1.2em} | |
| ul.plain li{padding-left:1.3em;position:relative;margin-bottom:.55em} | |
| ul.plain li::before{content:"→";position:absolute;left:0;color:var(--teal);font-family:"IBM Plex Mono",monospace;font-size:.8em;top:.28em} | |
| .pull{ | |
| border-left:3px solid var(--gold); | |
| padding:2px 0 2px 20px;margin:28px 0; | |
| font-size:1.06rem;color:var(--ink); | |
| } | |
| /* ---------- triad cards ---------- */ | |
| .triad{display:grid;grid-template-columns:repeat(3,1fr);gap:1px;background:var(--rule);border:1px solid var(--rule);margin:28px 0} | |
| .triad div{background:var(--card);padding:18px} | |
| .triad h3{font-size:.95rem;margin-bottom:8px} | |
| .triad p{font-size:.92rem;margin:0;color:var(--ink-2);line-height:1.5} | |
| .triad .tag{font-family:"IBM Plex Mono",monospace;font-size:.68rem;letter-spacing:.1em;color:var(--teal);display:block;margin-bottom:10px} | |
| /* ---------- table ---------- */ | |
| .tablewrap{overflow-x:auto;margin:26px 0 10px;-webkit-overflow-scrolling:touch} | |
| table{border-collapse:collapse;width:100%;min-width:640px;font-family:"IBM Plex Mono",monospace;font-size:.76rem;background:var(--card)} | |
| th,td{padding:10px 12px;border-bottom:1px solid var(--rule);text-align:center} | |
| thead th{font-weight:500;color:var(--ink-2);letter-spacing:.04em;text-transform:uppercase;font-size:.66rem;vertical-align:bottom;border-bottom:1px solid var(--ink)} | |
| tbody th{text-align:left;font-weight:500;white-space:nowrap} | |
| tr.ours{background:rgba(20,98,94,.07)} | |
| tr.ours th{color:var(--teal)} | |
| .yes{color:var(--teal)} | |
| .no{color:var(--plum)} | |
| .meh{color:var(--ink-2);font-size:.9em} | |
| /* ---------- pseudocode ---------- */ | |
| .algo{border:1px solid var(--rule);background:var(--card);padding:18px 20px;margin:26px 0;font-family:"IBM Plex Mono",monospace;font-size:.78rem;line-height:1.75;overflow-x:auto} | |
| .algo .name{font-size:.68rem;letter-spacing:.12em;text-transform:uppercase;color:var(--plum);display:block;margin-bottom:12px} | |
| .algo .cm{color:var(--ink-2)} | |
| .algo .kw{color:var(--teal)} | |
| .algo .ind{padding-left:2ch;display:block} | |
| /* ---------- math blocks ---------- */ | |
| .mathblock{margin:24px 0;overflow-x:auto;overflow-y:hidden;padding:4px 0} | |
| .katex{font-size:1.03em} | |
| /* ---------- citation ---------- */ | |
| .cite{position:relative;border:1px solid var(--rule);background:var(--card);padding:18px 20px;margin:22px 0 0} | |
| .cite pre{margin:0;font-family:"IBM Plex Mono",monospace;font-size:.74rem;line-height:1.6;white-space:pre-wrap;word-break:break-word;color:var(--ink-2)} | |
| .copy{position:absolute;top:12px;right:12px} | |
| footer{margin-top:72px;border-top:1px solid var(--rule);padding:28px 0 60px;font-size:.85rem;color:var(--ink-2)} | |
| footer p{margin:0 0 .5em} | |
| @media (max-width:700px){ | |
| body{font-size:17.5px} | |
| .triad{grid-template-columns:1fr} | |
| header.hero{padding-top:44px} | |
| canvas{height:205px} | |
| } | |
| @media (prefers-reduced-motion:no-preference){ | |
| .fade{opacity:0;transform:translateY(10px);animation:rise .7s cubic-bezier(.2,.7,.3,1) forwards} | |
| .fade:nth-of-type(2){animation-delay:.08s} | |
| @keyframes rise{to{opacity:1;transform:none}} | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="topbar"> | |
| <div class="wide"> | |
| <span class="eyebrow">CBMM Memo 057 · arXiv:1610.06160</span> | |
| <nav> | |
| <a href="#framework">Framework</a> | |
| <a href="#streaming">Streaming</a> | |
| <a href="#lp">L<sub>p</sub></a> | |
| <a href="#results">Results</a> | |
| <a href="#code">Code</a> | |
| </nav> | |
| </div> | |
| </div> | |
| <div class="disclaimer"> | |
| <div class="wide"> | |
| <span class="flag">AI-generated</span> | |
| <p>This page was written by an AI language model summarizing <a href="https://arxiv.org/abs/1610.06160">arXiv:1610.06160</a>. It is not written, reviewed, or endorsed by the authors, and it may contain errors or misread the paper. Treat it as a reading aid and check anything that matters against <a href="https://arxiv.org/pdf/1610.06160">the original PDF</a>. The animated figure below is a toy illustration of the estimator, not a trained network — for a working PyTorch implementation and a real CIFAR-10 training demo, go to <a href="https://github.com/liaoq/StreamingNorm">github.com/liaoq/StreamingNorm</a>.</p> | |
| </div> | |
| </div> | |
| <header class="hero"> | |
| <div class="wrap"> | |
| <div class="kicker fade"> | |
| <span class="eyebrow">19 October 2016</span> | |
| <span class="eyebrow">Center for Brains, Minds and Machines · MIT</span> | |
| </div> | |
| <h1 class="fade">Streaming Normalization | |
| <span class="lede">Normalization statistics gathered online from every sample and every timestep seen so far — so one layer covers online, batch, convolutional and recurrent learning alike.</span> | |
| </h1> | |
| <p class="byline fade">Qianli Liao, Kenji Kawaguchi, Tomaso Poggio | |
| <span class="aff">Center for Brains, Minds and Machines, McGovern Institute · CSAIL, MIT</span> | |
| </p> | |
| <ul class="links fade"> | |
| <li><a href="https://arxiv.org/abs/1610.06160">arXiv abstract</a></li> | |
| <li><a href="https://arxiv.org/pdf/1610.06160">PDF</a></li> | |
| <li><a href="https://cbmm.mit.edu/sites/default/files/publications/CBMM-Memo-057.pdf">CBMM Memo 057</a></li> | |
| <li><a href="https://github.com/liaoq/StreamingNorm" class="code">Code · PyTorch</a></li> | |
| <li><a href="#cite">Cite</a></li> | |
| </ul> | |
| </div> | |
| </header> | |
| <!-- ===================== SIGNATURE ===================== --> | |
| <div class="stream"> | |
| <div class="stream-frame"> | |
| <div class="stream-head"> | |
| <span class="eyebrow">The estimator, running</span> | |
| <span class="eyebrow" id="tick">batch 0 · update 0</span> | |
| </div> | |
| <canvas id="plot" tabindex="0" role="img" aria-label="Animated plot of normalization statistics estimated from a stream of mini-batches. Drag horizontally or use the arrow keys to scroll back through the trace."></canvas> | |
| <div class="legend"> | |
| <span><i class="sw dots" style="border-color:#7E8C82"></i><b>s</b> raw batch statistic</span> | |
| <span><i class="sw dash" style="border-color:var(--plum)"></i><b>short</b> avg since last update</span> | |
| <span><i class="sw" style="border-color:var(--teal)"></i><b>long</b> exponential average</span> | |
| <span><i class="sw thick" style="border-color:var(--gold)"></i><b>ŝ</b> what the layer divides by</span> | |
| </div> | |
| <div class="controls"> | |
| <button id="toggle" aria-pressed="false">Pause</button> | |
| <button id="live" hidden>Back to live</button> | |
| <label>α<sub>1</sub> (long-term weight) <input id="alpha" type="range" min="0" max="1" step="0.05" value="0.7"><output id="alphaOut">0.70</output></label> | |
| <label>batches per update <input id="bpu" type="range" min="1" max="6" step="1" value="3"><output id="bpuOut">3</output></label> | |
| <label class="chk"><input id="onlyView" type="checkbox" checked> run only when viewed</label> | |
| </div> | |
| </div> | |
| <p class="caption">Mini-batches arrive left to right. <b>ŝ<sub>short</sub></b> is the exact average of the batch statistics since the last weight update; at each update (dotted line) it folds into the exponential average <b>ŝ<sub>long</sub></b> and resets. What the layer actually divides by is the blend <b>ŝ = α<sub>1</sub>ŝ<sub>long</sub> + α<sub>2</sub>ŝ<sub>short</sub></b>. Drag α<sub>1</sub> to 0 and the whole thing collapses back to batch normalization. Drag the plot sideways (or use ← →) to scroll back through the retained trace; generation stops while you're looking at history. This is a scalar illustration on a synthetic stream — the <a href="https://github.com/liaoq/StreamingNorm">PyTorch implementation</a> is where to go for the real thing.</p> | |
| </div> | |
| <!-- ===================== IN BRIEF ===================== --> | |
| <section id="brief"> | |
| <div class="wrap"> | |
| <div class="sec-head"> | |
| <span class="eyebrow">§1 · In brief</span> | |
| <h2>Two things batch norm cannot do</h2> | |
| </div> | |
| <p>Batch normalization estimates its mean and standard deviation from the samples sitting in the current mini-batch. That works beautifully for feedforward training with a healthy batch size, and it breaks in two places. With one or two samples per batch there is nothing to estimate from. In a recurrent network, activation distributions differ across timesteps, so a single set of statistics is wrong and per-timestep statistics don't generalize to sequence lengths you never trained on.</p> | |
| <p>The fixes available in 2016 each solved half the problem. Time-specific batch normalization keeps separate statistics for every timestep — accurate, but memory grows with sequence length, it still needs large batches, and no plausible biological mechanism resets a neuron's gain on a per-timestep schedule. Layer normalization estimates from a single sample, which handles online and recurrent learning cleanly, but it measurably underperforms on convolutional layers.</p> | |
| <p>This paper's move is to stop asking where the statistics come from within a step, and ask instead that they accumulate <em>across</em> steps. Keep a running estimate that streams over every sample and every timestep already seen. One layer, one set of statistics, every learning scenario.</p> | |
| <div class="tablewrap"> | |
| <table> | |
| <caption class="visually-hidden"></caption> | |
| <thead> | |
| <tr> | |
| <th scope="col">Approach</th> | |
| <th scope="col">FF + FC</th> | |
| <th scope="col">FF + conv</th> | |
| <th scope="col">Rec + FC</th> | |
| <th scope="col">Rec + conv</th> | |
| <th scope="col">Online</th> | |
| <th scope="col">Small batch</th> | |
| <th scope="col">All combined</th> | |
| </tr> | |
| </thead> | |
| <tbody> | |
| <tr><th scope="row">Batch norm</th><td class="yes">✓</td><td class="yes">✓</td><td class="no">✗</td><td class="no">✗</td><td class="no">✗</td><td class="meh">weak</td><td class="no">✗</td></tr> | |
| <tr><th scope="row">Time-specific BN</th><td class="yes">✓</td><td class="yes">✓</td><td class="meh">limited</td><td class="meh">limited</td><td class="no">✗</td><td class="meh">weak</td><td class="no">✗</td></tr> | |
| <tr><th scope="row">Layer norm</th><td class="yes">✓</td><td class="meh">weak</td><td class="yes">✓</td><td class="meh">weak</td><td class="yes">✓</td><td class="yes">✓</td><td class="meh">weak</td></tr> | |
| <tr class="ours"><th scope="row">Streaming norm</th><td class="yes">✓</td><td class="yes">✓</td><td class="yes">✓</td><td class="yes">✓</td><td class="yes">✓</td><td class="yes">✓</td><td class="yes">✓</td></tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| <p class="caption">Adapted from Table 1. <em>Limited</em>: per-timestep statistics may not transfer to unseen sequence lengths. <em>Weak</em>: trains, but well below the best method for that setting. FF = feedforward, Rec = recurrent, FC = fully connected.</p> | |
| </div> | |
| </section> | |
| <!-- ===================== FRAMEWORK ===================== --> | |
| <section id="framework"> | |
| <div class="wrap"> | |
| <div class="sec-head"> | |
| <span class="eyebrow">§3 · A general framework</span> | |
| <h2>Three parts, and every method is a choice of each</h2> | |
| </div> | |
| <p>Any normalization scheme can be described as: pick some set of activations to look at, boil them down to a few numbers, then use those numbers to rescale a neuron. Naming the three parts separately is what makes the design space visible.</p> | |
| <div class="triad"> | |
| <div> | |
| <span class="tag">NormRef</span> | |
| <h3>What you look at</h3> | |
| <p>The reference set \(R_i\) of activations feeding neuron \(i\)'s statistics. Batch norm: the same channel across the mini-batch. Layer norm: the whole layer, one sample.</p> | |
| </div> | |
| <div> | |
| <span class="tag">NormStats</span> | |
| <h3>What you keep</h3> | |
| <p>\(s_i = S(R_i)\) — whatever numbers the operation needs. Usually a centre \(\mu_i\) and a scale \(\sigma_i\).</p> | |
| </div> | |
| <div> | |
| <span class="tag">NormOp</span> | |
| <h3>What you do</h3> | |
| <p>\(N(x_i, s_i)\) applied to the neuron. Throughout this paper it stays the familiar \((x_i-\mu_i)/\sigma_i\).</p> | |
| </div> | |
| </div> | |
| <p>Fixing NormOp and varying the other two sorts existing methods into three families, distinguished only by how much history the reference set covers:</p> | |
| <ul class="plain"> | |
| <li><b>Sample normalization</b> — statistics from the current sample alone. Layer normalization lives here. Training and test behave identically, and online learning is free.</li> | |
| <li><b>General batch normalization (GBN)</b> — statistics from all samples in the mini-batch. Test time needs a stand-in, so you keep a running average. The original batch norm is one member.</li> | |
| <li><b>Streaming normalization</b> — statistics accumulated online from every sample seen so far.</li> | |
| </ul> | |
| <p>The learnable shift and gain that batch norm bolts on are pulled out into a separate layer here, applied identically after every normalization variant, so comparisons aren't confounded by them.</p> | |
| </div> | |
| </section> | |
| <!-- ===================== STREAMING ===================== --> | |
| <section id="streaming"> | |
| <div class="wrap"> | |
| <div class="sec-head"> | |
| <span class="eyebrow">§3.3 · The proposal</span> | |
| <h2>Streaming statistics, streaming gradients</h2> | |
| </div> | |
| <p>Accumulating statistics over all past samples creates an awkward dependency: today's activation depends on every sample ever seen. Exact backpropagation through that history is not just expensive, it's incoherent — you cannot backpropagate past a weight update you've already applied, and earlier mini-batches are usually gone. So the paper drops exactness and uses two matched heuristics instead.</p> | |
| <h3 style="font-size:1.05rem;margin:32px 0 12px">Streaming NormStats</h3> | |
| <p>Each layer keeps a short-term and a long-term estimate. The short-term one, \(\hat{s}_{short}\), is the exact average of the batch statistics observed since the last weight update. The long-term one is an exponential average that only moves when the weights move:</p> | |
| <div class="mathblock">$$\hat{s}_{long} \leftarrow \kappa_1\,\hat{s}_{long} + \kappa_2\,\hat{s}_{short}, \qquad \kappa_1+\kappa_2 = 1$$</div> | |
| <p>and the value actually used to normalize is a blend of the two:</p> | |
| <div class="mathblock">$$\hat{s} = \alpha_1\,\hat{s}_{long} + \alpha_2\,\hat{s}_{short}, \qquad \alpha_1+\alpha_2 = 1$$</div> | |
| <p>At the update, the counter resets and \(\hat{s}_{short}\) is cleared. Before testing, it isn't: the final weight update is skipped and the layer keeps normalizing with the statistics inherited from the last training batch, which works fine in practice.</p> | |
| <h3 style="font-size:1.05rem;margin:32px 0 12px">Streaming gradients</h3> | |
| <p>The same treatment applies on the way back. A second table holds short- and long-term estimates of \(\partial E/\partial\hat{s}\), updated by the same exponential rule with \(\kappa_3,\kappa_4\), and the gradient passed further down is a three-way blend that includes the current one:</p> | |
| <div class="mathblock">$$\widehat{\frac{\partial E}{\partial \hat{s}}} = \beta_1\,\hat{g}_{long} + \beta_2\,\hat{g}_{short} + \beta_3\,\frac{\partial E}{\partial \hat{s}}, \qquad \beta_1+\beta_2+\beta_3 = 1$$</div> | |
| <div class="algo"> | |
| <span class="name">Streaming normalization layer — forward</span> | |
| <span class="cm">in: mini-batch x, NormOp N, statistic fn S, table H₁, update fn F</span> | |
| <span class="cm">out: mini-batch y; H₁ stays inside the layer, latest ŝ kept for testing</span> | |
| <span class="kw">if</span> training <span class="kw">then</span> | |
| <span class="ind">s = S(x)</span> | |
| <span class="ind">{H₁, ŝ} = F(H₁, s)</span> | |
| <span class="ind">y = N(x, ŝ)</span> | |
| <span class="kw">else</span> | |
| <span class="ind">y = N(x, ŝ) <span class="cm">// inherited, not recomputed</span></span> | |
| <span class="kw">end</span> | |
| </div> | |
| <div class="algo"> | |
| <span class="name">Streaming normalization layer — backward</span> | |
| <span class="cm">in: ∂E/∂y, x, ŝ, table H₂, update fn G</span> | |
| <span class="cm">out: ∂E/∂x; H₂ stays inside the layer</span> | |
| ∂E/∂ŝ = chain rule | |
| {H₂, est. ∂E/∂ŝ} = G(H₂, ∂E/∂ŝ) | |
| ∂E/∂x = chain rule, using <span class="kw">est.</span> ∂E/∂ŝ | |
| </div> | |
| <div class="pull"> | |
| Set \(n=1,\ \alpha_1=0,\ \beta_1=\beta_2=0\) and every trace of history drops out: the layer reduces exactly to general batch normalization, and with the usual NormRef, to batch normalization itself. Streaming normalization is a strict generalization, not a competitor. | |
| </div> | |
| <h3 style="font-size:1.05rem;margin:32px 0 12px">Decoupled accumulation and update</h3> | |
| <p>One small training change makes the rest work. Conventionally, gradients accumulate over a mini-batch and the weights update immediately. Decouple the two: accumulate over \(n\) mini-batches of \(m\) samples each, then update once and clear. Two knobs, samples per batch \(m\) and batches per update \(n\), with \(n=1\) recovering ordinary training and \(m=1\) giving pure online learning.</p> | |
| <p>This is not the same as one large batch of \(m \times n\) samples, because each mini-batch is normalized on arrival and cannot be revisited. On its own it already rescues much of batch norm's collapse at small batch sizes. For recurrent streaming normalization, \(n=2\) is often enough and often better than \(n=1\): the first mini-batch sweeps up statistics from all timesteps so the second is normalized against something stable.</p> | |
| </div> | |
| </section> | |
| <!-- ===================== Lp ===================== --> | |
| <section id="lp"> | |
| <div class="wrap"> | |
| <div class="sec-head"> | |
| <span class="eyebrow">§3.4 · A second, separable idea</span> | |
| <h2>L<sub>p</sub> normalization</h2> | |
| </div> | |
| <p>The centre \(\mu\) is always the mean of the reference set. The divisor \(\sigma\) is more open than convention suggests. Take the \(p\)-th root of the \(p\)-th absolute moment about a point \(c\):</p> | |
| <div class="mathblock">$$\int |x-c|^p\,P(x)\,dx \qquad\longrightarrow\qquad \frac{1}{N}\sum_{i=1}^{N}|x_i - c|^p$$</div> | |
| <p>Three choices of \(c\) give three settings: the mean of the reference set (A), a running estimate of the mean (B), or zero (C). Settings B and C survive online learning, where A degenerates to \(\sigma = 0\) with a single sample; with a reasonable batch, A and B behave alike.</p> | |
| <p>Setting A with \(p=2\) is the standard deviation, which is what batch norm and layer norm already use. The interesting case is \(p=1\), where \(\sigma\) is just the mean absolute value. Across the experiments it tracks \(p=2\) almost exactly while being cheaper to compute, easier to implement, and — the authors' motivating reason — trivially differentiable, which matters if you care about hardware or about neurons. Moments up to \(p=7\) still train; only the highest one slips.</p> | |
| <p>Because L<sub>p</sub> only changes the statistic function \(S(\cdot)\), it drops into layer normalization, batch normalization, or any of the reference-set variants independently of the streaming machinery.</p> | |
| </div> | |
| </section> | |
| <!-- ===================== RECURRENT ===================== --> | |
| <section id="recurrent"> | |
| <div class="wrap"> | |
| <div class="sec-head"> | |
| <span class="eyebrow">§4–5 · Recurrent networks</span> | |
| <h2>Streaming through time as well as samples</h2> | |
| </div> | |
| <p>The recurrent extension is the part that needs no new mechanism. Sample normalization already generalizes, since it never looks beyond the current timestep. Recurrent GBN is the time-specific approach: the unrolled network gets one GBN layer per step, each with its own memory. Recurrent streaming normalization instead uses the layer in the <em>rolled</em> network — all unrolled copies share one set of running estimates, so the stream runs over past samples and past timesteps together.</p> | |
| <p>One caveat, stated and then measured: the estimates drift slightly as time advances, so by the time backpropagation returns to a layer, its statistics are no longer quite the ones used in the forward pass. Empirically this doesn't hurt, and \(n>1\) damps it further.</p> | |
| <p>Applied to a GRU, every matrix product gets normalized before it is combined — input and hidden paths separately:</p> | |
| <div class="mathblock">$$\begin{aligned} | |
| g_r &= \mathrm{Sigmoid}\!\left(\mathrm{Norm}(W_{xr}x_t) + \mathrm{Norm}(W_{hr}h_{t-1})\right)\\[2pt] | |
| g_z &= \mathrm{Sigmoid}\!\left(\mathrm{Norm}(W_{xz}x_t) + \mathrm{Norm}(W_{hz}h_{t-1})\right)\\[2pt] | |
| h_{new} &= \mathrm{NonLinear}\!\left(\mathrm{Norm}(W_{xh}x_t) + \mathrm{Norm}(W_{hh}(h_{t-1}\odot g_r))\right)\\[2pt] | |
| h_t &= g_z \odot h_{new} + (1-g_z)\odot h_{t-1} | |
| \end{aligned}$$</div> | |
| <p>The plain normalized RNN follows the same pattern: \(h_t = \mathrm{NonLinear}(\mathrm{Norm}(W_x x_t) + \mathrm{Norm}(W_h h_{t-1}))\). Experiments used tanh; ReLU also worked.</p> | |
| </div> | |
| </section> | |
| <!-- ===================== RESULTS ===================== --> | |
| <section id="results"> | |
| <div class="wrap"> | |
| <div class="sec-head"> | |
| <span class="eyebrow">§7 · Experiments</span> | |
| <h2>What the curves showed</h2> | |
| </div> | |
| <p>Everything runs on CIFAR-10 in MatConvNet, across four architectures: a feedforward fully-connected net, a feedforward convnet, a ResNet-like convolutional RNN, and a densely recurrent convnet. Learning rate 0.1 for 25 epochs then 0.01 for 5, momentum 0.9.</p> | |
| <ul class="plain"> | |
| <li><b>Moments barely matter.</b> On the convnet, p = 1 through 4 are indistinguishable; L<sub>1</sub> matches L<sub>2</sub> at lower cost. Only L<sub>7</sub> falls behind.</li> | |
| <li><b>Decoupling helps on its own.</b> Batch norm with a tiny batch fails badly at \(n=1\) and recovers substantially at \(n>1\), before streaming statistics enter the picture at all.</li> | |
| <li><b>Online learning works.</b> With one sample per batch, and again with two, streaming normalization trains and beats layer normalization. (Batch size is irrelevant to layer norm, which is the point of comparing there.)</li> | |
| <li><b>Per-channel reference sets earn their keep.</b> Among the batch-style variants, normalizing each feature map separately — what batch norm does — consistently helps. The sample-style variants, layer norm included, cluster together.</li> | |
| <li><b>It matches time-specific BN without the bookkeeping.</b> On both recurrent convolutional architectures, streaming normalization lands at similar accuracy while keeping a single set of statistics for all timesteps instead of one per step.</li> | |
| </ul> | |
| <p>Reported settings for the streaming runs: \(\alpha_1 = \beta_1 = 0.7,\ \alpha_2 = 0.3\) for the recurrent and online experiments, \(0.5/0.5\) for the feedforward ones, with \(\beta_3\) either 0 or 0.3.</p> | |
| <p>One practical warning. Because backpropagation here is deliberately inexact, gradient checking the full model will fail. Check the model without the layer, then add a correct implementation — or reduce the layer to standard batch norm using the hyperparameters above and gradient check that.</p> | |
| <p><a href="https://arxiv.org/pdf/1610.06160">The figures are worth opening</a>; this page summarizes them rather than reproducing them.</p> | |
| </div> | |
| </section> | |
| <!-- ===================== CODE ===================== --> | |
| <section id="code"> | |
| <div class="wrap"> | |
| <div class="sec-head"> | |
| <span class="eyebrow">Implementation</span> | |
| <h2>Running it for real</h2> | |
| </div> | |
| <p>The paper's experiments were built in MatConvNet. A later PyTorch port lives at <a href="https://github.com/liaoq/StreamingNorm">github.com/liaoq/StreamingNorm</a>, exposing <code>StreamingNorm1d</code> and <code>StreamingNorm2d</code> as drop-in replacements for the corresponding <code>BatchNorm</code> layers, plus a CIFAR-10 training script that compares against batch, layer, and group normalization.</p> | |
| <p>Two differences from the paper are worth knowing before you read its defaults. The port folds the current batch statistic into the forward blend as a third term, \(\hat{s} = \alpha_1\hat{s}_{long} + \alpha_2\hat{s}_{short} + \alpha_3 s\), mirroring the \(\beta_3\) term the paper already had on the backward pass; setting \(\alpha_3 = 0\) recovers the published formulation. It also detects weight updates from the optimizer automatically, and implements L<sub>2</sub> only, so the L<sub>1</sub> result above isn't exercised by that code.</p> | |
| <p>Its README also flags a practical failure mode the paper doesn't dwell on: NaNs early in training usually mean the streaming buffers haven't seen enough data to form a usable estimate yet, and a few warm-up mini-batches fix it rather than indicating anything wrong with the method.</p> | |
| </div> | |
| </section> | |
| <!-- ===================== CITE ===================== --> | |
| <section id="cite"> | |
| <div class="wrap"> | |
| <div class="sec-head"> | |
| <span class="eyebrow">Reference</span> | |
| <h2>Cite this paper</h2> | |
| </div> | |
| <div class="cite"> | |
| <button class="copy" id="copyBtn">Copy</button> | |
| <pre id="bib">@article{liao2016streaming, | |
| title={Streaming normalization: Towards simpler and more biologically-plausible normalizations for online and recurrent learning}, | |
| author={Liao, Qianli and Kawaguchi, Kenji and Poggio, Tomaso}, | |
| journal={arXiv preprint arXiv:1610.06160}, | |
| year={2016} | |
| }</pre> | |
| </div> | |
| </div> | |
| </section> | |
| <footer> | |
| <div class="wrap"> | |
| <p>An unofficial, AI-generated reader's guide to <a href="https://arxiv.org/abs/1610.06160">arXiv:1610.06160</a>, written in summary form and not reviewed by the authors. All results, figures and claims belong to them — read the paper for the actual argument.</p> | |
| <p>Supported in part by the Center for Brains, Minds and Machines, funded by NSF STC award CCF-1231216.</p> | |
| </div> | |
| </footer> | |
| <script defer src="https://cdn.jsdelivr.net/npm/katex@0.16.9/dist/katex.min.js" integrity="sha384-XjKyOOlGwcjNTAIQHIpgOno0Hl1YQqzUOEleOLALmuqehneUG+vnGctmUb0ZY0l8" crossorigin="anonymous"></script> | |
| <script defer src="https://cdn.jsdelivr.net/npm/katex@0.16.9/dist/contrib/auto-render.min.js" integrity="sha384-+VBxd3r6XgURycqtZ117nYw44OOcIax56Z4dCRWbxyPt0Koah1uHoK0o4+/RRE05" crossorigin="anonymous" | |
| onload="renderMathInElement(document.body,{delimiters:[{left:'$$',right:'$$',display:true},{left:'\\(',right:'\\)',display:false}],throwOnError:false});"></script> | |
| <script> | |
| /* ---------- copy citation ---------- */ | |
| document.getElementById('copyBtn').addEventListener('click', function(){ | |
| var btn = this; | |
| navigator.clipboard.writeText(document.getElementById('bib').textContent).then(function(){ | |
| btn.textContent = 'Copied'; | |
| setTimeout(function(){ btn.textContent = 'Copy'; }, 1600); | |
| }).catch(function(){ | |
| btn.textContent = 'Select it'; | |
| setTimeout(function(){ btn.textContent = 'Copy'; }, 1600); | |
| }); | |
| }); | |
| /* ---------- the stream ---------- */ | |
| (function(){ | |
| var canvas = document.getElementById('plot'); | |
| var ctx = canvas.getContext('2d'); | |
| var css = getComputedStyle(document.documentElement); | |
| var C = { | |
| teal: css.getPropertyValue('--teal').trim(), | |
| plum: css.getPropertyValue('--plum').trim(), | |
| gold: css.getPropertyValue('--gold').trim(), | |
| rule: css.getPropertyValue('--rule').trim(), | |
| paper: css.getPropertyValue('--card').trim() | |
| }; | |
| var W = 0, H = 0; | |
| function resize(){ | |
| var r = canvas.getBoundingClientRect(); | |
| var dpr = window.devicePixelRatio || 1; | |
| W = r.width; H = r.height; | |
| canvas.width = Math.round(W * dpr); | |
| canvas.height = Math.round(H * dpr); | |
| ctx.setTransform(dpr, 0, 0, dpr, 0, 0); | |
| } | |
| resize(); | |
| window.addEventListener('resize', resize); | |
| var VIEW = 90; // batches visible at once | |
| var BUF = 900; // batches retained for scrolling back | |
| var hist = []; // {s, short, long, blend, update} | |
| var sLong = 1, sShortSum = 0, sShortN = 0; | |
| var batchCount = 0, updateCount = 0, phase = 0; | |
| var alpha1 = 0.7, bpu = 3; | |
| var K1 = 0.7; // kappa1 | |
| var running = true; // play/pause | |
| var visible = true; // figure intersecting the viewport | |
| var gateOnView = true; // "run only when viewed" | |
| var offset = 0; // batches scrolled back from the live edge | |
| var rafId = null; | |
| var docVisible = true; // tab in the foreground | |
| var PAD = { t:16, r:60, b:30, l:30 }; | |
| function step(){ | |
| phase += 0.055; | |
| // a drifting "true" scale the network's activations would have | |
| var truth = 1 + 0.45 * Math.sin(phase) + 0.18 * Math.sin(phase * 2.7); | |
| var s = Math.max(0.15, truth + (Math.random() - 0.5) * 0.34); | |
| sShortSum += s; sShortN += 1; | |
| var sShort = sShortSum / sShortN; | |
| var blend = alpha1 * sLong + (1 - alpha1) * sShort; | |
| batchCount += 1; | |
| var didUpdate = (batchCount % bpu === 0); | |
| hist.push({s:s, short:sShort, long:sLong, blend:blend, update:didUpdate}); | |
| if (hist.length > BUF) hist.shift(); | |
| if (didUpdate){ | |
| sLong = K1 * sLong + (1 - K1) * sShort; | |
| sShortSum = 0; sShortN = 0; | |
| updateCount += 1; | |
| } | |
| status(); | |
| } | |
| function status(){ | |
| var s; | |
| if (offset > 0) s = 'history · ' + offset + ' batches back'; | |
| else if (gateOnView && !visible) s = 'idle · offscreen'; | |
| else if (!docVisible) s = 'idle · tab hidden'; | |
| else if (!running) s = 'paused'; | |
| else s = 'batch ' + batchCount + ' · update ' + updateCount; | |
| document.getElementById('tick').textContent = s; | |
| liveBtn.hidden = (offset === 0); | |
| } | |
| // the slice of history currently on screen | |
| function windowStart(){ | |
| return Math.max(0, hist.length - offset - VIEW); | |
| } | |
| function maxOffset(){ | |
| return Math.max(0, hist.length - VIEW); | |
| } | |
| function draw(){ | |
| ctx.clearRect(0, 0, W, H); | |
| if (!hist.length) return; | |
| var padT = PAD.t, padB = PAD.b, padL = PAD.l, padR = PAD.r; | |
| var lo = 0.2, hi = 2.1; | |
| var start = windowStart(); | |
| var view = hist.slice(start, hist.length - offset); | |
| if (!view.length) return; | |
| var x = function(i){ return padL + (W - padL - padR) * (i / (VIEW - 1)); }; | |
| var y = function(v){ | |
| var t = (v - lo) / (hi - lo); | |
| return H - padB - Math.max(0, Math.min(1, t)) * (H - padT - padB); | |
| }; | |
| ctx.font = '10px "IBM Plex Mono", monospace'; | |
| ctx.textBaseline = 'middle'; | |
| // gridlines, with values | |
| ctx.lineWidth = 1; | |
| [0.5, 1.0, 1.5, 2.0].forEach(function(v){ | |
| ctx.strokeStyle = C.rule; ctx.globalAlpha = 0.6; | |
| ctx.beginPath(); ctx.moveTo(padL, y(v)); ctx.lineTo(W - padR, y(v)); ctx.stroke(); | |
| ctx.globalAlpha = 1; | |
| ctx.fillStyle = '#8B968D'; ctx.textAlign = 'right'; | |
| ctx.fillText(v.toFixed(1), padL - 7, y(v)); | |
| }); | |
| // weight-update markers | |
| ctx.strokeStyle = '#B0B8AC'; ctx.lineWidth = 1; ctx.setLineDash([2, 4]); | |
| view.forEach(function(p, i){ | |
| if (!p.update) return; | |
| ctx.beginPath(); ctx.moveTo(x(i), padT - 8); ctx.lineTo(x(i), H - padB); ctx.stroke(); | |
| }); | |
| ctx.setLineDash([]); | |
| // raw batch statistics | |
| ctx.fillStyle = '#7E8C82'; | |
| view.forEach(function(p, i){ | |
| ctx.beginPath(); ctx.arc(x(i), y(p.s), 1.7, 0, Math.PI * 2); ctx.fill(); | |
| }); | |
| function line(key, color, width, dash, halo){ | |
| ctx.lineJoin = 'round'; ctx.lineCap = 'round'; | |
| var passes = halo ? 2 : 1; | |
| for (var k = 0; k < passes; k++){ | |
| var isHalo = halo && k === 0; | |
| ctx.strokeStyle = isHalo ? C.paper : color; | |
| ctx.lineWidth = isHalo ? width + 3.5 : width; | |
| ctx.setLineDash(isHalo ? [] : dash); | |
| ctx.beginPath(); | |
| var started = false; | |
| for (var i = 0; i < view.length; i++){ | |
| // the short-term estimate restarts after every weight update | |
| var prev = (i > 0) ? view[i-1] : hist[start-1]; | |
| if (key === 'short' && prev && prev.update){ | |
| ctx.stroke(); ctx.beginPath(); started = false; | |
| } | |
| if (!started){ ctx.moveTo(x(i), y(view[i][key])); started = true; } | |
| else ctx.lineTo(x(i), y(view[i][key])); | |
| } | |
| ctx.stroke(); | |
| } | |
| ctx.setLineDash([]); | |
| } | |
| line('short', C.plum, 1.6, [5, 3], false); | |
| line('long', C.teal, 1.8, [], false); | |
| line('blend', C.gold, 3.2, [], true); | |
| // labels riding the end of each line, nudged apart when they collide | |
| var last = view[view.length - 1]; | |
| var lx = x(view.length - 1); | |
| var tags = [ | |
| { t:'s', v:last.s, c:'#7E8C82', w:400 }, | |
| { t:'short', v:last.short, c:C.plum, w:500 }, | |
| { t:'long', v:last.long, c:C.teal, w:500 }, | |
| { t:'\u015D', v:last.blend, c:C.gold, w:700 } | |
| ].map(function(o){ o.y = y(o.v); return o; }); | |
| tags.sort(function(a, b){ return a.y - b.y; }); | |
| for (var i = 1; i < tags.length; i++){ | |
| if (tags[i].y - tags[i-1].y < 12) tags[i].y = tags[i-1].y + 12; | |
| } | |
| var over = tags[tags.length-1].y - (H - padB); | |
| if (over > 0) tags.forEach(function(o){ o.y -= over; }); | |
| ctx.textAlign = 'left'; | |
| tags.forEach(function(o){ | |
| ctx.strokeStyle = o.c; ctx.globalAlpha = 0.45; ctx.lineWidth = 1; | |
| ctx.beginPath(); ctx.moveTo(lx + 2, y(o.v)); ctx.lineTo(lx + 8, o.y); ctx.stroke(); | |
| ctx.globalAlpha = 1; | |
| ctx.font = o.w + ' 10px "IBM Plex Mono", monospace'; | |
| ctx.fillStyle = o.c; | |
| ctx.fillText(o.t, lx + 11, o.y); | |
| }); | |
| // horizontal position track: where this window sits in the retained buffer | |
| var trackY = H - 9, tx0 = padL, tx1 = W - padR, tw = tx1 - tx0; | |
| var span = Math.max(VIEW, hist.length); | |
| var thumbW = Math.max(24, tw * VIEW / span); | |
| var pos = (span - VIEW) ? (start / (span - VIEW)) : 1; | |
| ctx.strokeStyle = C.rule; ctx.lineWidth = 3; ctx.lineCap = 'round'; | |
| ctx.beginPath(); ctx.moveTo(tx0, trackY); ctx.lineTo(tx1, trackY); ctx.stroke(); | |
| ctx.strokeStyle = offset > 0 ? C.plum : '#98A29A'; | |
| ctx.beginPath(); | |
| ctx.moveTo(tx0 + pos * (tw - thumbW) + 1.5, trackY); | |
| ctx.lineTo(tx0 + pos * (tw - thumbW) + thumbW - 1.5, trackY); | |
| ctx.stroke(); | |
| } | |
| // ---- animation loop: only alive when it has something to do ------------- | |
| var lastT = 0; | |
| function frame(t){ | |
| if (t - lastT > 110){ step(); lastT = t; } | |
| draw(); | |
| rafId = requestAnimationFrame(frame); | |
| } | |
| function shouldRun(){ | |
| return running && offset === 0 && docVisible && (visible || !gateOnView); | |
| } | |
| function sync(){ | |
| if (shouldRun()){ | |
| if (rafId === null){ lastT = 0; rafId = requestAnimationFrame(frame); } | |
| } else { | |
| if (rafId !== null){ cancelAnimationFrame(rafId); rafId = null; } | |
| draw(); | |
| } | |
| status(); | |
| } | |
| var liveBtn = document.getElementById('live'); | |
| // seed a little history so the panel is never empty | |
| for (var i = 0; i < 40; i++) step(); | |
| // ---- pause when scrolled out of the viewport ---------------------------- | |
| if ('IntersectionObserver' in window){ | |
| new IntersectionObserver(function(entries){ | |
| visible = entries[0].isIntersecting; | |
| sync(); | |
| }, { threshold: 0 }).observe(canvas); | |
| } else { | |
| gateOnView = false; | |
| } | |
| document.addEventListener('visibilitychange', function(){ | |
| docVisible = !document.hidden; | |
| sync(); | |
| }); | |
| // ---- scrolling back through the trace ----------------------------------- | |
| function pan(batches){ | |
| var next = Math.max(0, Math.min(maxOffset(), offset + batches)); | |
| if (next === offset) return; | |
| offset = next; | |
| sync(); | |
| } | |
| function pxPerBatch(){ | |
| return (W - PAD.l - PAD.r) / (VIEW - 1); | |
| } | |
| var dragging = false, dragX = 0, dragAcc = 0; | |
| canvas.addEventListener('pointerdown', function(e){ | |
| dragging = true; dragX = e.clientX; dragAcc = 0; | |
| canvas.setPointerCapture(e.pointerId); | |
| canvas.classList.add('grabbing'); | |
| }); | |
| canvas.addEventListener('pointermove', function(e){ | |
| if (!dragging) return; | |
| dragAcc += (e.clientX - dragX) / pxPerBatch(); | |
| dragX = e.clientX; | |
| var whole = Math.trunc(dragAcc); | |
| if (whole){ dragAcc -= whole; pan(whole); } | |
| }); | |
| ['pointerup','pointercancel'].forEach(function(ev){ | |
| canvas.addEventListener(ev, function(e){ | |
| dragging = false; | |
| canvas.classList.remove('grabbing'); | |
| if (canvas.hasPointerCapture(e.pointerId)) canvas.releasePointerCapture(e.pointerId); | |
| }); | |
| }); | |
| canvas.addEventListener('wheel', function(e){ | |
| var d = Math.abs(e.deltaX) > Math.abs(e.deltaY) ? e.deltaX : (e.shiftKey ? e.deltaY : 0); | |
| if (!d) return; // let plain vertical wheel scroll the page | |
| e.preventDefault(); | |
| pan(-Math.sign(d) * Math.max(1, Math.round(Math.abs(d) / 12))); | |
| }, { passive: false }); | |
| canvas.addEventListener('keydown', function(e){ | |
| if (e.key === 'ArrowLeft'){ e.preventDefault(); pan(e.shiftKey ? 10 : 1); } | |
| if (e.key === 'ArrowRight'){ e.preventDefault(); pan(e.shiftKey ? -10 : -1); } | |
| }); | |
| liveBtn.addEventListener('click', function(){ offset = 0; sync(); }); | |
| // ---- controls ----------------------------------------------------------- | |
| var reduced = window.matchMedia('(prefers-reduced-motion: reduce)'); | |
| var btn = document.getElementById('toggle'); | |
| if (reduced.matches){ | |
| running = false; | |
| btn.textContent = 'Play'; | |
| btn.setAttribute('aria-pressed', 'true'); | |
| } | |
| btn.addEventListener('click', function(){ | |
| running = !running; | |
| if (running) offset = 0; // resuming returns to the live edge | |
| btn.textContent = running ? 'Pause' : 'Play'; | |
| btn.setAttribute('aria-pressed', String(!running)); | |
| sync(); | |
| }); | |
| var gate = document.getElementById('onlyView'); | |
| gate.addEventListener('change', function(){ | |
| gateOnView = gate.checked; | |
| sync(); | |
| }); | |
| var aIn = document.getElementById('alpha'), aOut = document.getElementById('alphaOut'); | |
| aIn.addEventListener('input', function(){ | |
| alpha1 = parseFloat(aIn.value); | |
| K1 = Math.max(0.2, alpha1); | |
| aOut.textContent = alpha1.toFixed(2); | |
| }); | |
| var bIn = document.getElementById('bpu'), bOut = document.getElementById('bpuOut'); | |
| bIn.addEventListener('input', function(){ | |
| bpu = parseInt(bIn.value, 10); | |
| bOut.textContent = bpu; | |
| }); | |
| sync(); | |
| })(); | |
| </script> | |
| </body> | |
| </html> |