import './style.css'; import { EXPERIMENTS } from './experiments.js'; import {validateExploration} from '../public/exploration.js'; const $=s=>document.querySelector(s); $('#app').innerHTML=`
first words 100% in your browser

THE TINY MODEL LAB

From noise.
To first words.

Watch a tiny model learn to write.
One character at a time, right before your eyes.

01 A little text02 Random weights03 Learn, then try againNanoGPT, modded or jax-js · randomly initialized weights

Three ways to watch it learn.

Choose your starting point.
${EXPERIMENTS.map(e=>``).join('')}

WATCH IT LEARN

Same prompt. New words.

Same prompt · same seed

Before

STEP 0

The first characters will appear here, before any training.

Now

WAITING

Start training to watch the text change with each step.

Two samples from the same model, before and after training. These are its actual outputs.

LIVE TRAINING

Less surprised. More precise.

Backend pending

LOSS TRAIN 20-step mean—
STEPS0
PARAMETERS—
STEP ms—

Actual timings will appear during training.

The loss curve will appear here.
Every point is a real training step.
Raw training loss Smoothed meanCross-entropy · nats / character
Open sample history No samples yet

The latest samples and the very first sample are kept here.

`; const urlSettings=new URLSearchParams(location.search); if(['nanogpt','modded','auto','wasm','webgpu'].includes(urlSettings.get('engine')))$('#backend').value=urlSettings.get('engine'); if(['balanced','fast','quality','stories-fast','paragraphs','original'].includes(urlSettings.get('recipe')))$('#recipe').value=urlSettings.get('recipe'); let worker=null,state='idle',history=[],samples=[],ema=null,experiment='stories',startedAt=0,totalMs=0,trainMs=0,exploring=false,exploreId=0,exploreDraft=null; function corpusCount(){$('#chars').textContent=`${[...$('#corpus').value].length.toLocaleString('en-US')} chars`;} function connectionNote(){if(['nanogpt','modded'].includes($('#backend').value)&&!window.isSecureContext)return 'NanoGPT needs HTTPS or localhost to verify its runtime. Use HTTPS or select jax-js · WASM / CPU.';if($('#backend').value==='modded')return 'Modded · official research variant adapted to CPU FP32 · Muon + AdamW. This is not the 9.65-second record version. Runtime download: about 61 MB.';if($('#backend').value==='nanogpt')return 'Original NanoGPT · PyTorch in WebAssembly on CPU. First download: about 61 MB of Python/PyTorch; no pretrained model. Experimental.';return window.isSecureContext?'Auto tries WebGPU, then WASM if no compatible GPU is available.':'HTTP connection: WASM training available. WebGPU requires HTTPS.';} $('#backend-note').textContent=connectionNote(); function engineSettings(){ const python=['nanogpt','modded'].includes($('#backend').value);for(const name of ['paragraphs','stories-fast'])$('#recipe option[value="'+name+'"]').disabled=python; if(python&&['paragraphs','stories-fast'].includes($('#recipe').value))$('#recipe').value='balanced'; const storyFast=$('#recipe').value==='stories-fast',paragraphs=storyFast||$('#recipe').value==='paragraphs'; const modded=$('#backend').value==='modded',original=$('#recipe').value==='original',fast=$('#recipe').value==='fast',quality=$('#recipe').value==='quality'; $('#rate-label').textContent=modded?'Muon learning rate':'Learning rate';$('#rate').max=modded?'0.05':'0.02';$('#rate').value=modded?'0.025':original?'0.003':'0.006';$('#backend-note').textContent=connectionNote();if(paragraphs)$('#backend-note').textContent='Specialized for the provided stories: favors complete narratives. Less suited to other text positions; its loss is not directly comparable to random-window recipes.'; $('#context').value=paragraphs?'256':quality?'128':'32';$('#batch').value='4';$('#schedule').value=original&&!modded?'constant':'cooldown'; // The official modded adapter has its own mandatory cooldown schedule. $('#architecture').disabled=['nanogpt','modded'].includes($('#backend').value);if(original||paragraphs)$('#architecture').value='rms';$('#schedule').disabled=modded;$('#schedule').title=modded?'Modded always uses its official cooldown schedule.':''; $('#temperature').value=experiment==='pattern'?'0.5':experiment==='names'?'0.8':storyFast?'0.4':original?'0.8':'0.6'; $('#sampleEvery').value=original?'25':fast||paragraphs?'200':'100';$('#sampleLength').value=original||experiment!=='stories'?'80':quality||paragraphs?'240':'160'; const e=EXPERIMENTS.find(e=>e.id===experiment);$('#maxSteps').value=experiment==='stories'?(fast?600:storyFast?350:paragraphs?500:1000):(e?.maxSteps??1000); } $('#backend').onchange=engineSettings;$('#recipe').onchange=engineSettings; function selectExperiment(id){ const e=EXPERIMENTS.find(e=>e.id===id);experiment=id; if(id!=='stories'&&['paragraphs','stories-fast'].includes($('#recipe').value))$('#recipe').value='balanced'; $('#corpus').value=e.text;$('#prefix').value=e.prefix;$('#size').value='tiny'; engineSettings(); $('#experiment-guide').textContent=e.guide;$('#error').hidden=true; document.querySelectorAll('[data-experiment]').forEach(b=>b.setAttribute('aria-pressed',String(b.dataset.experiment===id))); corpusCount(); } document.querySelectorAll('[data-experiment]').forEach(b=>b.onclick=()=>{if(state==='idle')selectExperiment(b.dataset.experiment);}); $('#corpus').addEventListener('input',()=>{corpusCount();experiment='custom';document.querySelectorAll('[data-experiment]').forEach(b=>b.setAttribute('aria-pressed','false'));$('#experiment-guide').textContent='Your world, your text. Compare the initial characters with those generated after training.';}); $('#restore').onclick=()=>selectExperiment('stories'); selectExperiment(EXPERIMENTS.some(e=>e.id===urlSettings.get('experiment'))?urlSettings.get('experiment'):'stories'); if(['rms','nanogpt','rotary'].includes(urlSettings.get('architecture'))&&!['nanogpt','modded'].includes($('#backend').value))$('#architecture').value=urlSettings.get('architecture'); function setState(s,message){state=s;$('#start').disabled=['loading','pausing','error','complete'].includes(s);$('#start').textContent=s==='running'?'Pause Ⅱ':s==='paused'?'Resume training ↗':s==='complete'?'Training complete ✓':s==='loading'?'Preparing…':s==='pausing'?'Pausing…':'Start training ↗';document.querySelectorAll('aside input,aside select,aside textarea,#restore,[data-experiment]').forEach(el=>el.disabled=s!=='idle');$('#schedule').disabled=s!=='idle'||$('#backend').value==='modded';$('#architecture').disabled=s!=='idle'||['nanogpt','modded'].includes($('#backend').value);$('#experiment-lock').textContent=s==='idle'?'Choose your starting point.':'Reset to change the experiment.';if(message)$('#status').textContent=message;syncExplore();} function error(message){$('#error').hidden=false;$('#error').textContent=message;setState('error','Reset to change settings or select another backend.');worker?.terminate();worker=null;} function start(){ if(state==='running'){worker.postMessage({type:'pause'});setState('pausing','Finishing the current step…');return;} if(state==='paused'){worker.postMessage({type:'resume'});setState('running','Training resumes.');return;} const req={type:'start',text:$('#corpus').value,prefix:$('#prefix').value,size:$('#size').value,backend:$('#backend').value,rate:+$('#rate').value,temperature:+$('#temperature').value,sampleEvery:+$('#sampleEvery').value,maxSteps:+$('#maxSteps').value,context:+$('#context').value,batch:+$('#batch').value,schedule:$('#schedule').value,optimizer:$('#recipe').value==='original'?'original':'tuned',sampleLength:+$('#sampleLength').value,architecture:$('#architecture').value,sampling:['paragraphs','stories-fast'].includes($('#recipe').value)?'paragraphs':'random',fastSampling:['paragraphs','stories-fast'].includes($('#recipe').value)}; for(const id of ['rate','temperature','maxSteps'])if(!$('#'+id).reportValidity())return; if(!req.prefix||[...req.prefix].some(c=>!req.text.includes(c))){$('#error').hidden=false;$('#error').textContent='The prompt must be nonempty and use only characters from the dataset.';return;} startedAt=performance.now();totalMs=0;trainMs=0;$('#error').hidden=true;setState('loading','Initializing in a Worker…'); worker=['nanogpt','modded'].includes(req.backend)?new Worker(new URL(`${import.meta.env.BASE_URL}nanogpt/worker.js${req.backend==='modded'?'?engine=modded':''}`,location.href),{type:'module'}):new Worker(new URL('./worker.js',import.meta.url),{type:'module'});worker.onerror=e=>{if(worker===activeWorker)error(e.message||'The Worker failed.');}; const activeWorker=worker; worker.onmessage=({data:d})=>{ if(worker!==activeWorker)return; if(d.type==='error')return error(d.message); if(d.type.startsWith('explore-')){receiveExploration(d);return;} if(d.type==='status')$('#status').textContent=d.message; if(d.type==='ready'){$('#backend-badge').textContent=d.backend==='modded'?'● Modded · Muon / CPU':d.backend==='nanogpt'?'● NanoGPT · PyTorch / CPU':d.backend==='webgpu'?'● jax-js · WebGPU':'● jax-js · WASM / CPU';$('#backend-badge').title=d.fallback||'Computing in the Worker';$('#backend-note').textContent=d.fallback||(d.backend==='webgpu'?'WebGPU in your browser. Step time includes computation and synchronization.':'WASM on CPU, in your browser.');if(req.sampling==='paragraphs')$('#backend-note').textContent+=' Specialized recipe: paragraph starts. This loss is not directly comparable to random-window loss.';if(d.config.architecture==='rotary')$('#backend-note').textContent+=' RoPE relative positions · experimental: lower loss in some trials, without guaranteed coherence.';if(d.config.architecture==='nanogpt')$('#backend-note').textContent+=' Bias-free NanoGPT architecture, implemented in jax-js and checked against PyTorch.';if(d.loadMs)$('#backend-note').textContent+=` Loading: ${(d.loadMs/1000).toFixed(1)} s.`;$('#params').textContent=d.params.toLocaleString('en-US');$('#status').textContent=d.fallback||'Backend ready.';} if(d.type==='running')setState('running','Training · weights update at every step.'); if(d.type==='step'){ema=ema===null?d.loss:0.9*ema+0.1*d.loss;history.push({...d,ema});const recent=history.slice(-20);$('#loss').textContent=(recent.reduce((s,p)=>s+p.loss,0)/recent.length).toFixed(3);trainMs=d.elapsed;updateTiming();$('#steps').textContent=d.step;$('#speed').textContent=Math.round(d.ms).toLocaleString('en-US');scheduleDraw();} if(d.type==='sampling'){$('#sample-label').textContent=`Generating · step ${d.step}…`;if(state!=='pausing')setState('running','Sampling · training pauses during generation.');} if(d.type==='sample-progress'){updateSample(d);} if(d.type==='sample'){updateSample(d);$('#sample-label').textContent='Same prompt · same seed';if(state==='running')$('#status').textContent='Training in progress.';} if(d.type==='complete'){updateTiming();$('#explore-prefix').value=req.prefix;$('#explore-temperature').value=req.temperature;$('#explore-length').value=req.sampleLength;setState('complete','Training complete. Try another prompt below, or reset.');} if(d.type==='paused'&&state!=='complete')setState('paused','Paused · weights and optimizer state are preserved.'); }; worker.postMessage(req); } function updateTiming(){totalMs=performance.now()-startedAt;$('#timing').textContent=`Training compute: ${(trainMs/1000).toFixed(1)} s · elapsed: ${(totalMs/1000).toFixed(1)} s (including loading, generation and pauses).`;} $('#start').onclick=start; $('#reset').onclick=()=>{worker?.terminate();worker=null;exploring=false;exploreId++;exploreDraft=null;$('#explore-error').hidden=true;$('#explore-seed').value='123';$('#explore-status').textContent='Weights and training history stay unchanged.';$('#explore-text').textContent='Your next text will appear here.';$('#explore-text').classList.add('placeholder');history=[];samples=[];ema=null;totalMs=0;trainMs=0;$('#timing').textContent='Actual timings will appear during training.';$('#error').hidden=true;$('#loss').textContent='—';$('#steps').textContent='0';$('#params').textContent='—';$('#speed').textContent='—';$('#backend-badge').textContent='Backend pending';$('#backend-badge').removeAttribute('title');$('#backend-note').textContent=connectionNote();$('#sample-label').textContent='No samples yet';setState('idle','Reset. The next model starts with random weights.');renderSamples();draw();}; function updateSample(d){samples=samples.filter(s=>s.step!==d.step);samples.unshift(d);if(samples.length>8){const initial=samples.find(s=>s.step===0);samples=samples.slice(0,initial?7:8);if(initial)samples.push(initial);}renderSamples();} function renderComparison(){ const first=samples.find(s=>s.step===0), latest=samples.find(s=>s.step>0); function fill(selector,s,empty){const node=$(selector);node.replaceChildren();node.classList.toggle('placeholder',!s);if(!s){node.textContent=empty;return;}const prefix=document.createElement('span');prefix.className='prefix';prefix.textContent=s.prefix;node.append(prefix,document.createTextNode(s.text));} fill('#before-text',first,'The first characters will appear here, before any training.'); fill('#after-text',latest,'The next sample will show the first effects of training.'); $('#after-step').textContent=latest?`STEP ${latest.step}`:'WAITING'; } function renderSamples(){renderComparison();const root=$('#samples');root.replaceChildren();if(!samples.length){const div=document.createElement('div');div.className='empty-sample';div.textContent='The next sample will appear here.';root.append(div);}for(const s of samples){const article=document.createElement('article');article.className='sample';const header=document.createElement('div');header.className='sample-head';header.textContent=s.step===0?'STEP 0 · RANDOM WEIGHTS':`STEP ${s.step}`;const p=document.createElement('p'),prefix=document.createElement('span');prefix.className='prefix';prefix.textContent=s.prefix;p.append(prefix,document.createTextNode(s.text));article.append(header,p);root.append(article);}} let drawPending=false;function scheduleDraw(){if(!drawPending){drawPending=true;requestAnimationFrame(()=>{drawPending=false;draw();});}} function draw(){const canvas=$('#chart'),rect=canvas.getBoundingClientRect(),dpr=window.devicePixelRatio||1;canvas.width=rect.width*dpr;canvas.height=rect.height*dpr;const ctx=canvas.getContext('2d');ctx.scale(dpr,dpr);const w=rect.width,h=rect.height,left=38,right=12,top=14,bottom=28;ctx.clearRect(0,0,w,h);$('#chart-empty').hidden=history.length>0;const ymax=history.length?Math.max(...history.map(p=>p.loss))*1.15:5;ctx.font='11px ui-monospace, monospace';ctx.fillStyle='#8c918d';for(let i=0;i<=4;i++){const y=top+(h-top-bottom)*i/4;ctx.strokeStyle='#e9ece6';ctx.beginPath();ctx.moveTo(left,y);ctx.lineTo(w-right,y);ctx.stroke();ctx.fillText((ymax*(1-i/4)).toFixed(1),3,y+4);}if(!history.length)return;const xmax=Math.max(25,history.at(-1).step);for(let i=0;i<=4;i++){ctx.textAlign=i===4?'right':i===0?'left':'center';ctx.fillText(String(Math.round(xmax*i/4)),left+(w-left-right)*i/4,h-6);}ctx.textAlign='left';const line=(key,color,width)=>{ctx.strokeStyle=color;ctx.lineWidth=width;ctx.beginPath();history.forEach((p,i)=>{const x=left+p.step/xmax*(w-left-right),y=top+(1-p[key]/ymax)*(h-top-bottom);i?ctx.lineTo(x,y):ctx.moveTo(x,y);});ctx.stroke();};line('loss','#b6d1c2',1);line('ema','#287956',2.4);} new ResizeObserver(draw).observe($('#chart'));window.addEventListener('pagehide',()=>worker?.terminate()); // Read-only diagnostics for browser validation; no weights or optimizer exposed. window.__lab={get state(){return state;},get experiment(){return experiment;},get engine(){return $('#backend').value;},get steps(){return history.length;},get losses(){return history.map(p=>p.loss);},get timings(){return {trainMs,totalMs};},get exploration(){return {busy:exploring,result:exploreDraft?{...exploreDraft}:null};},get samples(){return samples.map(({step,text})=>({step,text}));}}; function syncExplore(){ $('#exploration').hidden=state!=='complete';$('#explore-jump').hidden=state!=='complete'; document.querySelectorAll('#exploration input,#exploration select,#explore-start,#explore-again').forEach(el=>el.disabled=state!=='complete'||exploring); $('#explore-cancel').hidden=!exploring;$('#explore-start').textContent=exploring?'Generating…':'Generate with this model ↗'; } function renderExploration(){ if(!exploreDraft)return;const node=$('#explore-text');node.classList.remove('placeholder');node.replaceChildren(); const prefix=document.createElement('span');prefix.className='prefix';prefix.textContent=exploreDraft.prefix;node.append(prefix,document.createTextNode(exploreDraft.text)); } function explore(another=false){ if(state!=='complete'||exploring||!worker)return; $('#explore-error').hidden=true; try{ const seed=+$('#explore-seed').value; const options=validateExploration({prefix:$('#explore-prefix').value,temperature:+$('#explore-temperature').value,seed,length:+$('#explore-length').value},$('#corpus').value); if(another){options.seed=(options.seed+1)>>>0;$('#explore-seed').value=options.seed;} exploring=true;exploreDraft={...options,text:''};exploreId++;renderExploration();syncExplore(); $('#explore-status').textContent='Generating · weights are frozen.'; worker.postMessage({type:'explore',id:exploreId,...options}); }catch(e){$('#explore-error').textContent=e.message;$('#explore-error').hidden=false;} } function receiveExploration(d){ if(d.id!==exploreId||!exploring)return; if(d.type==='explore-error'){$('#explore-error').textContent=d.message;$('#explore-error').hidden=false;$('#explore-status').textContent='Generation failed. Try again or reset.';exploring=false;syncExplore();return;} exploreDraft={...exploreDraft,...d};renderExploration(); if(d.type==='explore-result'){ exploring=false;syncExplore();$('#explore-status').textContent=`${d.cancelled?'Generation stopped':'Generation complete'} · ${[...d.text].length} characters · ${(d.ms/1000).toFixed(1)} s · model preserved at step ${d.step}.`; } } $('#explore-start').onclick=()=>explore();$('#explore-again').onclick=()=>explore(true); $('#explore-cancel').onclick=()=>{if(exploring){worker?.postMessage({type:'cancel-explore'});$('#explore-status').textContent='Stopping…';}}; syncExplore();