Spaces:
Running
Running
| ; | |
| const test=require('node:test'),assert=require('node:assert/strict'),fs=require('node:fs'),path=require('node:path'),zlib=require('node:zlib'),crypto=require('node:crypto'); | |
| const root=path.resolve(__dirname,'..'); | |
| const data=JSON.parse(fs.readFileSync(path.join(root,'experiments/data/results.json'))); | |
| const summary=JSON.parse(fs.readFileSync(path.join(root,'experiments/data/summary.json'))); | |
| const manifest=JSON.parse(fs.readFileSync(path.join(root,'experiments/data/traces.json'))); | |
| const registry=JSON.parse(fs.readFileSync(path.join(root,'models.json'))); | |
| const {validateObservatoryTrace}=require('../trace-validation.js'); | |
| const sha=b=>crypto.createHash('sha256').update(b).digest('hex'); | |
| test('all 24 new topology captures retain valid measured frames and both checksums',()=>{ | |
| assert.equal(manifest.length,24);let frames=0; | |
| for(const row of manifest){ | |
| const compressed=fs.readFileSync(path.join(root,row.file)),plain=zlib.gunzipSync(compressed),trace=JSON.parse(plain); | |
| assert.equal(sha(compressed),row.gzip_sha256);assert.equal(sha(plain),row.sha256);assert.equal(trace.id,row.id);assert.equal(trace.result.reason,'eos');assert.equal(trace.frames.length,row.tokens); | |
| validateObservatoryTrace(trace,registry);frames+=trace.frames.length; | |
| } | |
| assert.equal(frames,80);assert.equal(summary.full_capture_frames,frames); | |
| }); | |
| test('prompt pairs change exactly one word and compare the first prediction',()=>{ | |
| const cases=data.experiments.reroute.cases;assert.equal(cases.length,6); | |
| for(const item of cases){ | |
| const a=item.prompts[0].text.split(/\s+/),b=item.prompts[1].text.split(/\s+/);assert.equal(a.length,b.length);assert.equal(a.filter((v,i)=>v!==b[i]).length,1); | |
| assert.equal(item.comparison.prompt_words_changed,1);assert.deepEqual(item.comparison.prompt_token_counts,[item.conditions[0].metrics.prompt_tokens,item.conditions[1].metrics.prompt_tokens]); | |
| if(item.id.endsWith('flint')){ | |
| assert.equal(item.comparison.routed_blocks_compared,47);assert.ok(item.comparison.blocks_with_different_expert_sets>0); | |
| for(const condition of item.conditions){assert.equal(condition.routing.layer,item.comparison.displayed_zero_based_block);assert.equal(condition.routing.tokens[0].experts.length,6);} | |
| }else for(const condition of item.conditions)assert.equal(condition.routing,undefined); | |
| } | |
| }); | |
| test('every blind pair uses one matched Q4_1 and Q8_0 checkpoint and keeps identical answers',()=>{ | |
| const pairs=data.experiments.quant.cases;assert.equal(pairs.length,12); | |
| assert.equal(pairs.filter(x=>x.conditions[0].output===x.conditions[1].output).length,11); | |
| for(const item of pairs){assert.deepEqual(item.conditions.map(x=>x.quant).sort(),['Q4_1','Q8_0']);for(const c of item.conditions){assert.equal(c.metrics.complete,true);assert.equal(c.metrics.termination,'eos');assert.ok(c.metrics.process_peak_rss_kib>0);}} | |
| const arithmetic=pairs.find(x=>x.id==='flint-arithmetic');assert.equal(17*23-19,372); | |
| for(const c of arithmetic.conditions){assert.notEqual(Number(c.output),17*23-19);assert.equal(c.metrics.objective_check_passed,false);} | |
| }); | |
| test('every saved intervention frame follows its exact requested scale',()=>{ | |
| let altered=0,unchanged=0,total=0; | |
| for(const item of data.experiments.surgery.cases){ | |
| assert.equal(item.conditions.length,3);const base=item.conditions[0]; | |
| assert.deepEqual(item.conditions.map(c=>c.settings.intervention_scale),[1,.5,0]); | |
| for(const c of item.conditions){ | |
| assert.equal(c.metrics.termination,'eos'); | |
| for(const f of c.measurements){ | |
| const row=f.layers.find(x=>x.layer===12&&x.tensor==='ffn_out');assert.ok(row);const expected=row.before_rms*c.settings.intervention_scale; | |
| assert.ok(Math.abs(row.after_rms-expected)<=Math.max(1e-10,Math.abs(expected)*1e-7));total++; | |
| } | |
| const layers=c.measurements[0].layers,original=base.measurements[0].layers; | |
| const rms=(rows,layer,tensor)=>rows.find(x=>x.layer===layer&&x.tensor===tensor).after_rms; | |
| assert.equal(rms(layers,11,'ffn_out'),rms(original,11,'ffn_out'));unchanged++; | |
| if(c.settings.intervention_scale!==1){assert.notEqual(rms(layers,13,'l_out'),rms(original,13,'l_out'));altered++;} | |
| } | |
| } | |
| assert.equal(total,132);assert.equal(unchanged,18);assert.equal(altered,12); | |
| for(const id of ['baseline','half','zero'])assert.equal(data.experiments.surgery.cases.filter(x=>x.conditions.find(c=>c.id===id).check.passed).length,4); | |
| }); | |
| test('memory results separately retain original-fact and correction tasks',()=>{ | |
| const cases=data.experiments.memory.cases;assert.equal(cases.length,6);assert.equal(cases.filter(x=>x.kind==='current').length,3);assert.equal(cases.filter(x=>x.kind==='original').length,3); | |
| let count=0;for(const item of cases){assert.ok(item.choices.includes(item.ground_truth));for(const c of item.conditions){assert.equal(c.output.trim(),item.ground_truth);assert.equal(c.metrics.exact_answer_passed,true);assert.ok(c.metrics.prompt_tokens<=144);count++;}} | |
| assert.equal(count,12);assert.equal(summary.memory_answers_passed,count); | |
| }); | |
| test('checkpoint and assembled source provenance agree with the reproduction manifests',()=>{ | |
| const checkpoints=JSON.parse(fs.readFileSync(path.join(root,'experiments/reproduce/checkpoints.json'))); | |
| for(const c of data.provenance.checkpoints){const model=checkpoints[c.model][c.quantization];assert.equal(c.file_bytes,model.bytes);assert.equal(c.sha256,model.sha256);} | |
| assert.equal(data.provenance.source_tree,'83e9d4c276f84753cce063502a2a9e2325f15ce3'); | |
| assert.equal(sha(fs.readFileSync(path.join(root,'experiments/reproduce/llama-flint-runtime.patch'))),data.provenance.source_patch_sha256); | |
| assert.equal(registry.find(x=>x.id==='qwen').title,'alphaBell Qwen Open 27B'); | |
| }); | |
| test('downloadable clips have verified media headers and WAVs exactly matching the public mapping',()=>{ | |
| const media=JSON.parse(fs.readFileSync(path.join(root,'experiments/assets/media.json'))); | |
| const sound=require('../experiments/sound.js'),original=JSON.parse(fs.readFileSync(path.join(root,'traces/index.json'))); | |
| for(const entry of media.files){const bytes=fs.readFileSync(path.join(root,'experiments/assets',entry.file));assert.equal(bytes.length,entry.bytes);assert.equal(sha(bytes),entry.sha256);assert.equal(bytes.subarray(0,4).toString('hex'),entry.file.endsWith('.wav')?'52494646':'1a45dfa3');} | |
| for(const model of ['flint','qwen']){ | |
| const row=original.find(x=>x.model===model),trace=JSON.parse(zlib.gunzipSync(fs.readFileSync(path.join(root,row.file)))); | |
| const expected=Buffer.from(sound.encodeWAV(sound.renderPCM(sound.buildScore(trace,model==='flint'?12:0,.35)),.35)); | |
| assert.deepEqual(fs.readFileSync(path.join(root,'experiments/assets',model==='flint'?'flint-music.wav':'qwen-open-music.wav')),expected); | |
| } | |
| }); | |