brain-lab / brain_lab /static /arcade-rl.js
Chungulus's picture
Deploy Brain Lab CPU classroom app with verified saved research playback
4155f6b verified
Raw History Blame Contribute Delete
24.1 kB
import {api,card,escape} from './common.js';
import {flightView,crossingView} from './arcade-views.js?v=brain-inputs-41';
const specs={
flappy:{name:'Flappy Bird',unit:'pipes',title:'A flock that learns when to flap.',description:'Gravity pulls every bird down. A flap gives it an upward kick. Choose endless pipes or a fixed pipe count. Each bird receives the next two pipes’ positions and gap boundaries, and the whole flock shares what it learns.',goal:20,actions:['Coast','Flap'],manual:'Press Space or tap Flap to begin. Each press gives one flap; gravity keeps running between presses.',source:'FlapPyBird assets and reference are mirrored locally with the MIT notice. The headless physics, streaming courses, collision boxes, and Q-learning are Brain Lab’s classroom adaptation; the original pygame game is not running.',links:[['FlapPyBird reference','https://github.com/sourabhv/FlapPyBird'],['Pinned source receipt','/static/flappy-assets/SOURCE_LOCK.json'],['MIT notice','/static/flappy-assets/LICENSE']]},
crossy:{name:'Crossy Road',unit:'rows',title:'Wait, hop, and learn to read the traffic.',description:'Choose an endless road or set your own number of rows of grass and moving traffic. Cars keep moving while the chickens wait or hop. Each chicken has its own attempt; they share learned action values, using nearby car coordinates and velocities.',goal:null,actions:['Wait','Forward','Left','Right','Back'],manual:'Use the arrow keys or the touch arrows. Your first move starts the traffic clock. Stay alert: waiting too long ends an attempt.',source:'This is an independent Crossy Road-style classroom game with original voxel models, roads, traffic, and trees. It is not the commercial mobile game. The road is generated as you move and contains traffic and safe grass; it does not include rivers or trains.',links:[]}
};
export const flappy=root=>arcadeRL(root,'flappy');
export const crossy=root=>arcadeRL(root,'crossy');
async function arcadeRL(root,game){
const spec=specs[game],path='arcade-rl/'+game;
root.innerHTML=card(spec.title,`<p>${escape(spec.description)}</p><p class="note" id="rl-course-summary">Local classroom edition · endless or custom-length ${spec.unit} · shared learning from an empty table</p><div class="mario-settings"><label>Difficulty<select id="rl-difficulty"><option value="easy">Learning course</option><option value="standard">Tighter ${game==='flappy'?'gaps':'traffic'}</option></select></label><label>Agents per round<input id="rl-population" type="number" min="1" max="128" value="32"></label><label>Target game ticks per second<select id="rl-speed"><option value="30">30 · watch closely</option><option value="120" selected>120 · faster</option><option value="600">600 · fast learning</option><option value="2400">2,400 · turbo</option></select></label></div>
<div class="mario-settings crossy-course-settings"><label>Course length<select id="rl-course-mode"><option value="endless" selected>Endless · keep going</option><option value="fixed">Fixed number of ${spec.unit}</option></select></label><label id="rl-course-rows-label" hidden>Number of ${spec.unit}<input id="rl-course-rows" type="number" min="1" max="1000000" step="1" value="100"></label></div><p class="note" id="rl-course-help">Apply course changes with New learner from settings.</p>
<div class="arcade-rl-stage ${game==='crossy'?'is-crossy':'is-flappy'}"><canvas id="rl-canvas" width="900" height="525" tabindex="0" aria-label="${spec.name} live reinforcement learning game"></canvas><span class="arcade-rl-hud" id="rl-hud">READY AT THE START</span><div class="arcade-rl-caption" id="rl-mode">FRESH POLICY · EMPTY ACTION VALUES</div></div>
<div class="world-controls"><button class="primary" id="rl-start">Start learning</button><button class="secondary" id="rl-pause">Pause</button><button class="secondary" id="rl-step">One tick</button><button class="secondary" id="rl-burst">Train 10,000 ticks</button><button class="secondary" id="rl-test">Test on 4 new layouts</button><button class="secondary" id="rl-human">Play yourself</button><button class="secondary" id="rl-reset">New learner from settings</button></div>
<div class="arcade-human-controls" id="rl-human-controls" hidden>${game==='flappy'?'<button class="primary flap-touch" data-human-action="1">Flap · Space</button>':'<div class="crossy-touch"><button class="secondary" data-human-action="1" aria-label="Hop forward">↑</button><button class="secondary" data-human-action="2" aria-label="Hop left">←</button><button class="secondary" data-human-action="0">Wait</button><button class="secondary" data-human-action="3" aria-label="Hop right">→</button><button class="secondary" data-human-action="4" aria-label="Hop backward">↓</button></div>'}</div>
<p id="rl-message" class="message" role="status">Preparing the local game. All learning starts from zero.</p>
<div class="mario-stats"><div><strong id="rl-round">1</strong><span>round</span></div><div><strong id="rl-active">32</strong><span>active agents</span></div><div><strong id="rl-best">0</strong><span>best ${spec.unit} this round</span></div><div><strong id="rl-wins">0</strong><span id="rl-finish-label">courses finished this run</span></div><div><strong id="rl-experiences">0</strong><span>experiences</span></div><div><strong id="rl-updates">0</strong><span>Q-value updates</span></div></div>
<details class="simple-details rl-input-panel" open><summary>What the learner sees · actual game inputs</summary><p class="note" id="rl-input-source">These values are fed into the learning policy, then encoded into overlapping feature buckets.</p><dl id="rl-inputs" class="rl-observation-grid"></dl></details><p class="note">The counters and chart describe training. Tests and manual attempts run separately.</p><label class="range">Starting random exploration<input id="rl-exploration" type="range" min="0" max=".5" step=".01" value=".2"><output id="rl-exploration-value">20%</output></label><label class="control-check"><input type="checkbox" id="rl-auto-exploration" checked> Reduce random exploration as learning grows</label><p class="note" id="rl-exploration-now"></p><p class="note">A physics tick is 1/60 second of game time. The speed control changes training speed; manual play runs at normal speed. At high training speeds the display samples frames, while every physics tick still runs. Agents are independent and do not collide with each other.</p>
<div id="rl-test-result" class="arcade-test-result" hidden></div><div class="mario-analysis"><section><h3>Progress across completed rounds</h3><canvas id="rl-chart" width="720" height="180" aria-label="Mean and best ${spec.unit} reached in completed training rounds"></canvas><p class="note" id="rl-chart-note">Green: class average. Gold: best attempt. Distance is measured in ${spec.unit}. A lucky training attempt is different from a test of the learned policy.</p></section><section><h3>Learned values · training snapshot</h3><div id="rl-values"></div><p id="rl-learned" class="note"></p><p class="note">These values describe the last training snapshot. They estimate future reward and are not probabilities; test and manual games use separate states.</p></section></div>
<div class="world-controls"><button class="secondary" id="rl-save">Save learning locally</button><button class="secondary" id="rl-load">Load saved learning</button><span id="rl-save-note" class="note">Checkpoints belong to this game and classroom tab.</span></div>
<details class="simple-details"><summary>How learning works, rewards & game sources</summary><p>Real Q-learning starts with empty learned values. Overlapping feature buckets let similar observations share experience. Flappy also replays earlier attempts to learn from them again. Agents choose actions, experience consequences, and update shared estimates. When agents update the same feature and action together, their errors are averaged. Crossy retains its local traffic checks and adds nearby car coordinates as extra learned features. No pretrained policy or scripted controller supplies the actions.</p><p>${game==='flappy'?'Inputs include the bird’s height and velocity plus the next two pipes’ horizontal positions and exact gap boundaries. Passing a pipe and staying alive earn reward; crashing loses reward. A small potential-based shaping term rewards changes in gap alignment. A flap decision lasts three physics ticks.':'Inputs include the chicken’s column, blocked cells now and one hop ahead, and the nearest left/right car positions, speed, and length in the current and next two lanes. This is engineered state information rather than learned pixel vision. Reaching a new row earns reward; collisions, invalid moves, and excessive waiting lose reward. A hop lasts eight physics ticks, with collisions checked throughout it.'}</p><p>Training rounds use changing layouts. Tests use four fixed layouts absent from the training seed sequence, turn exploration off, and never update the table. The replay shows the first test; the table reports all four. Changing difficulty, population, or course length requires a new learner. Endless tests stop after at most 6,000 ticks (100 seconds of game time) and report distance; they have no finish line. Saving and loading preserves learned values and starts a fresh round.</p><p class="note">${escape(spec.source)}</p><div class="source-links">${spec.links.map(([name,url])=>`<a href="${url}" target="_blank" rel="noreferrer">${escape(name)} ↗</a>`).join('')}</div></details>`,'ACTUAL LOCAL PHYSICS · SHARED Q-LEARNING');
const $=s=>root.querySelector(s),canvas=$('#rl-canvas');root.querySelectorAll('button').forEach(b=>b.disabled=true);
let pauseRequested=false,alive=true,data,frame,view,busy=false,running=false,manual=false,manualStarted=false,dirty=false,saved=false,generation=0,timer,raf,finishAnimation,queued=0,heldTouch=0;const keys=new Set();
const mounted=()=>alive&&canvas.isConnected;
view=game==='flappy'?await flightView(canvas):crossingView(canvas);if(!canvas.isConnected){view.dispose();return()=>{alive=false}}
function controls(){if(!mounted())return;for(const id of ['start','step','burst','test','human','reset','save','load','difficulty','population','course-mode','course-rows'])$('#rl-'+id).disabled=busy||running;{const fixed=$('#rl-course-mode').value==='fixed';$('#rl-course-rows-label').hidden=!fixed;$('#rl-course-help').textContent=fixed?`Choose 1–1,000,000 ${spec.unit}. Apply changes with New learner from settings.`:`Endless ${spec.unit} continue until ${game==='flappy'?'a crash':'a collision or too much waiting'}. Apply changes with New learner from settings.`;}for(const id of ['start','step','burst','test','human'])$('#rl-'+id).disabled||=dirty;$('#rl-load').disabled||=!saved;$('#rl-pause').disabled=false;$('#rl-start').textContent=running?'Learning…':'Start / resume learning';$('#rl-human-controls').hidden=!manual;root.querySelectorAll('[data-human-action]').forEach(b=>b.disabled=!manual)}
function draw(){if(!mounted()||!frame)return;view.draw(frame);$('#rl-inputs').innerHTML=(data?.input_schema||[]).map((item,i)=>{const value=frame.input_values?.[i]??0;return `<div><dt>${escape(item.name)}</dt><dd>${item.unit==='flag'?(value?'Yes':'No'):Number(value).toFixed(item.unit.includes('tick')?3:1)} <small>${item.unit==='flag'?'':escape(item.unit)}</small></dd></div>`}).join('');const leader=frame.runners[frame.leader]||frame.runners[0];$('#rl-hud').textContent=`${manual?'YOU':frame.round?'ROUND '+frame.round:'POLICY TEST'} · ${frame.endless?leader.score+' '+spec.unit+' · ENDLESS':leader.score+' / '+(frame.goal??data?.goal??spec.goal)+' '+spec.unit}`}
function chart(){const ctx=$('#rl-chart').getContext('2d'),h=data.history||[],peak=Math.max(0,...h.map(r=>r.best_score),...data.frame.runners.map(r=>r.score)),scale=Math.max(1,Math.min(data.goal||Infinity,Math.max(10,Math.ceil(peak))));ctx.fillStyle='#eff4e8';ctx.fillRect(0,0,720,180);ctx.strokeStyle='#d7e1ce';for(let y=20;y<170;y+=35){ctx.beginPath();ctx.moveTo(35,y);ctx.lineTo(710,y);ctx.stroke()}for(const [key,color] of [['best_score','#c69c55'],['mean_score','#3b7d60']]){ctx.strokeStyle=color;ctx.lineWidth=2;ctx.beginPath();h.forEach((r,i)=>{const x=35+i/Math.max(1,h.length-1)*670,y=160-r[key]/scale*140;i?ctx.lineTo(x,y):ctx.moveTo(x,y)});ctx.stroke()}ctx.fillStyle='#75876a';ctx.font='12px sans-serif';ctx.fillText(String(scale),8,22);ctx.fillText('0',15,165)}
function update(s,settings=false){data={...data,...s};frame=s.frame;for(const key of ['round','active','wins','experiences','updates'])$('#rl-'+key).textContent=s[key==='wins'&&s.endless?'episodes':key].toLocaleString();$('#rl-best').textContent=Math.max(...s.frame.runners.map(r=>r.score));$('#rl-values').innerHTML=s.q_values.map((v,i)=>`<div class="mario-value"><span>${escape(s.action_names[i])}</span><b>${v.toFixed(2)}</b></div>`).join('');$('#rl-learned').textContent=`${s.states.toLocaleString()} feature buckets updated · ${s.transitions.toLocaleString()} action transitions`;
{$('#rl-finish-label').textContent=s.endless?'attempts ended this run':'courses finished this run';$('#rl-course-summary').textContent=`Local classroom edition · ${s.endless?'endless '+spec.unit:s.goal.toLocaleString()+'-'+spec.unit+' course'} · shared learning from an empty table`;$('#rl-chart-note').textContent=`Green: class average. Gold: best attempt. ${s.endless?'Endless mode has no finish line.':`The target is ${s.goal.toLocaleString()} ${spec.unit}.`} The distance axis scales with progress.`}if(settings){dirty=false;saved=s.checkpoint_available;$('#rl-save-note').textContent=s.checkpoint_note||'Checkpoints belong to this game and classroom tab.';{$('#rl-course-mode').value=s.endless?'endless':'fixed';if(!s.endless)$('#rl-course-rows').value=s.goal;$('#rl-course-rows-label').hidden=s.endless;}$('#rl-population').value=s.population;$('#rl-difficulty').value=s.difficulty;$('#rl-exploration').value=s.exploration;$('#rl-exploration-value').textContent=Math.round(s.exploration*100)+'%';$('#rl-auto-exploration').checked=s.auto_exploration}$('#rl-exploration-now').textContent=`Currently ${Math.round(s.effective_exploration*100)}% random actions · ${s.auto_exploration?'automatically reducing':'fixed setting'} · ${s.replay_size.toLocaleString()} stored transitions`;chart();draw()}
function stop(){running=false;manual=false;manualStarted=false;generation++;clearTimeout(timer);cancelAnimationFrame(raf);finishAnimation?.();queued=heldTouch=0;keys.clear();controls()}
function animate(frames,duration,token){if(!frames.length)return Promise.resolve();const start=performance.now();cancelAnimationFrame(raf);return new Promise(resolve=>{let done=false;const finish=()=>{if(done)return;done=true;clearTimeout(fallback);cancelAnimationFrame(raf);if(mounted()&&token===generation){frame=frames.at(-1);draw()}finishAnimation=null;resolve()};finishAnimation=finish;const fallback=setTimeout(finish,duration+60);function tick(now){if(!mounted()||token!==generation){finish();return}frame=frames[Math.min(frames.length-1,Math.floor((now-start)/duration*frames.length))];draw();if(now-start>=duration)finish();else raf=requestAnimationFrame(tick)}raf=requestAnimationFrame(tick)})}
function settlePause(){if(pauseRequested&&mounted()){pauseRequested=false;$('#rl-message').textContent='Paused. Learning is retained; test, resume, or play yourself.'}}
async function once(fn){stop();busy=true;controls();const token=generation;try{await fn(token)}catch(e){if(mounted())$('#rl-message').textContent=e.message}finally{busy=false;if(mounted()){controls();settlePause()}}}
async function learn(){if(!mounted()||!running||busy)return;busy=true;controls();const token=generation;try{const s=await api(path+'/train',{ticks:Math.max(1,Math.round(Number($('#rl-speed').value)/5))});if(!mounted()||token!==generation)return;update(s);$('#rl-mode').textContent=`${s.population} INDEPENDENT AGENTS · SHARED LEARNING`;$('#rl-message').textContent=`Round ${s.round}: ${s.active} still playing. ${s.endless?s.episodes+' attempts ended':s.wins+' courses finished'} during training. Test on new layouts to check the learned policy.`;await animate(s.frames,200,token)}catch(e){if(mounted()){stop();$('#rl-message').textContent=e.message}}finally{busy=false;if(mounted()){controls();settlePause();if(running)timer=setTimeout(learn,0)}}}
$('#rl-start').onclick=()=>{if(running)return;stop();running=true;learn()};$('#rl-pause').onclick=()=>{stop();pauseRequested=busy;$('#rl-mode').textContent='PAUSED';$('#rl-message').textContent=busy?'Pausing after the current batch finishes.':'Paused. Learning is retained; test, resume, or play yourself.'};
$('#rl-step').onclick=()=>once(async token=>{const s=await api(path+'/train',{ticks:1});if(!mounted()||token!==generation)return;update(s);$('#rl-mode').textContent='ONE PHYSICS TICK';$('#rl-message').textContent=`Advanced one tick. Actions are evaluated over ${s.hold} ticks, or when an attempt ends.`});
$('#rl-reset').onclick=()=>once(async token=>{const n=Number($('#rl-population').value);if(!Number.isInteger(n)||n<1||n>128)throw Error('Choose 1–128 agents.');const length=$('#rl-course-mode').value==='fixed'?Number($('#rl-course-rows').value):null;if(length!==null&&(!Number.isInteger(length)||length<1||length>1000000))throw Error(`Choose a whole number from 1 to 1,000,000 ${spec.unit}.`);const s=await api(path+'/reset',{population:n,difficulty:$('#rl-difficulty').value,seed:7,course_length:length});if(!mounted()||token!==generation)return;update(s,true);$('#rl-test-result').hidden=true;$('#rl-mode').textContent='NEW LEARNER · EVERYONE AT THE START';$('#rl-message').textContent='The table is empty. Everyone starts from the beginning with no scripted moves.'});
for(const id of ['difficulty','population','course-mode','course-rows'])$('#rl-'+id).onchange=()=>{dirty=Number($('#rl-population').value)!==data.population||$('#rl-difficulty').value!==data.difficulty||($('#rl-course-mode').value==='endless'?null:Number($('#rl-course-rows').value))!==data.course_length;if(dirty)$('#rl-message').textContent='Settings changed. Choose New learner from settings to apply them and clear the learned table.';controls()};
$('#rl-burst').onclick=()=>once(async token=>{const budget=10000;$('#rl-mode').textContent='FIXED PRACTICE BUDGET';for(let ticks=0;ticks<budget;ticks+=360){const s=await api(path+'/train',{ticks:Math.min(360,budget-ticks)});if(!mounted()||token!==generation)return;update(s);$('#rl-message').textContent=`Practiced ${Math.min(budget,ticks+360).toLocaleString()} / 10,000 ticks. Pause stops after the current batch.`}$('#rl-message').textContent='Practice batch complete. Test on new layouts to compare distance.'});
$('#rl-test').onclick=()=>once(async token=>{const r=await api(path+'/evaluate',{});if(!mounted()||token!==generation)return;$('#rl-mode').textContent='POLICY TEST · EXPLORATION OFF · NO LEARNING';$('#rl-test-result').hidden=false;$('#rl-test-result').innerHTML=`<h3>Four layouts outside the training seed sequence</h3><p>Average: <b>${r.mean_score.toFixed(1)}${r.endless?' '+spec.unit:' / '+r.goal+' '+spec.unit}</b>${r.endless?' · Endless distance test':' · Courses finished: <b>'+r.successes+' / '+r.runs.length+'</b>'}</p><table><thead><tr><th>Layout seed</th><th>${spec.unit}</th><th>Result</th></tr></thead><tbody>${r.runs.map(x=>`<tr><td>${x.seed}</td><td>${x.score}</td><td>${escape(x.reason)}</td></tr>`).join('')}</tbody></table><p class="note">Replay: seed ${r.test_seeds[0]}. Training stopped at ${r.training_experiences.toLocaleString()} experiences. Each test runs for at most 6,000 ticks; “test budget” means the run was still alive when the budget ended. These four trials are a small demonstration, not a general win-rate estimate.</p>`;$('#rl-message').textContent=`Test average ${r.mean_score.toFixed(1)} ${spec.unit}; ${r.endless?'endless course':r.successes+' of four courses finished'}. Replaying the first test with learning disabled.`;await animate(r.frames,Math.min(6500,Math.max(1600,r.frames.length*22)),token)});
$('#rl-exploration').oninput=e=>$('#rl-exploration-value').textContent=Math.round(e.target.value*100)+'%';const explorationChanged=async()=>{try{const result=await api(path+'/config',{exploration:Number($('#rl-exploration').value),auto_exploration:$('#rl-auto-exploration').checked});data={...data,...result};$('#rl-exploration-now').textContent=`Currently ${Math.round(result.effective_exploration*100)}% random actions · ${result.auto_exploration?'automatically reducing':'fixed setting'}`}catch(error){if(mounted())$('#rl-message').textContent=error.message}};$('#rl-exploration').onchange=explorationChanged;$('#rl-auto-exploration').onchange=explorationChanged;
function movement(){if(game==='flappy'){const v=queued;queued=0;return v}if(queued){const v=queued;queued=0;return v}if(heldTouch)return heldTouch;return keys.has('ArrowUp')?1:keys.has('ArrowLeft')?2:keys.has('ArrowRight')?3:keys.has('ArrowDown')?4:0}
async function play(){if(!mounted()||!manual)return;if(!manualStarted){timer=setTimeout(play,35);return}try{frame=await api(path+'/human',{action:movement()});if(!mounted()||!manual)return;draw();if(frame.runners[0].done){manual=false;$('#rl-message').textContent=frame.runners[0].reason==='goal'?'You finished the course!':`Attempt ended: ${frame.runners[0].reason}. Play yourself to try again.`;controls();return}timer=setTimeout(play,game==='flappy'?45:60)}catch(e){if(mounted()){stop();$('#rl-message').textContent=e.message}}}
$('#rl-human').onclick=()=>once(async token=>{const r=await api(path+'/human/reset',{});if(!mounted()||token!==generation)return;frame=r;manual=true;manualStarted=false;$('#rl-mode').textContent='YOUR TURN · NO LEARNING';$('#rl-message').textContent=spec.manual;draw();controls();canvas.focus();play()});
const onDown=e=>{if(!manual)return;if(game==='flappy'&&(e.key===' '||e.key==='ArrowUp')){e.preventDefault();if(!e.repeat){manualStarted=true;queued=1}}else if(game==='crossy'&&['ArrowUp','ArrowDown','ArrowLeft','ArrowRight'].includes(e.key)){e.preventDefault();manualStarted=true;keys.add(e.key)}},onUp=e=>keys.delete(e.key),onBlur=()=>keys.clear();canvas.addEventListener('keydown',onDown);canvas.addEventListener('keyup',onUp);canvas.addEventListener('blur',onBlur);
for(const b of root.querySelectorAll('[data-human-action]')){b.onpointerdown=e=>{if(!manual)return;e.preventDefault();b.setPointerCapture(e.pointerId);manualStarted=true;queued=Number(b.dataset.humanAction);if(game==='crossy')heldTouch=queued};b.onpointerup=b.onpointercancel=()=>heldTouch=0;b.onclick=e=>{if(!manual||e.detail!==0)return;manualStarted=true;queued=Number(b.dataset.humanAction)}}
canvas.addEventListener('pointerdown',()=>{if(manual&&game==='flappy'){manualStarted=true;queued=1;canvas.focus()}});
$('#rl-save').onclick=()=>once(async token=>{const r=await api(path+'/save',{});if(!mounted()||token!==generation)return;saved=true;$('#rl-save-note').textContent=`Saved after ${r.experiences.toLocaleString()} experiences.`});$('#rl-load').onclick=()=>once(async token=>{const s=await api(path+'/load',{});if(!mounted()||token!==generation)return;update(s,true);$('#rl-mode').textContent='SAVED LEARNER · FRESH ROUND';$('#rl-message').textContent='Learned values loaded. Everyone begins a new round at the start.'});
const onResize=()=>draw();window.addEventListener('resize',onResize);data=await api(path);if(!canvas.isConnected){view.dispose();window.removeEventListener('resize',onResize);return()=>{alive=false}}update(data,true);if(data.experiences)$('#rl-mode').textContent='SESSION RETAINED · PAUSED';$('#rl-message').textContent=data.experiences?'Your session is retained. Resume or test on new layouts.':'Start with an empty table: test the blank policy, train a few rounds, then test again.';controls();
return()=>{alive=false;stop();window.removeEventListener('resize',onResize);canvas.removeEventListener('keydown',onDown);canvas.removeEventListener('keyup',onUp);canvas.removeEventListener('blur',onBlur);view.dispose()};
}