'use strict'; const test=require('node:test'),assert=require('node:assert/strict'),fs=require('node:fs'),path=require('node:path'),zlib=require('node:zlib'),crypto=require('node:crypto'); const root=path.resolve(__dirname,'..'); const data=JSON.parse(fs.readFileSync(path.join(root,'experiments/data/results.json'))); const summary=JSON.parse(fs.readFileSync(path.join(root,'experiments/data/summary.json'))); const manifest=JSON.parse(fs.readFileSync(path.join(root,'experiments/data/traces.json'))); const registry=JSON.parse(fs.readFileSync(path.join(root,'models.json'))); const {validateObservatoryTrace}=require('../trace-validation.js'); const sha=b=>crypto.createHash('sha256').update(b).digest('hex'); test('all 24 new topology captures retain valid measured frames and both checksums',()=>{ assert.equal(manifest.length,24);let frames=0; for(const row of manifest){ const compressed=fs.readFileSync(path.join(root,row.file)),plain=zlib.gunzipSync(compressed),trace=JSON.parse(plain); assert.equal(sha(compressed),row.gzip_sha256);assert.equal(sha(plain),row.sha256);assert.equal(trace.id,row.id);assert.equal(trace.result.reason,'eos');assert.equal(trace.frames.length,row.tokens); validateObservatoryTrace(trace,registry);frames+=trace.frames.length; } assert.equal(frames,80);assert.equal(summary.full_capture_frames,frames); }); test('prompt pairs change exactly one word and compare the first prediction',()=>{ const cases=data.experiments.reroute.cases;assert.equal(cases.length,6); for(const item of cases){ const a=item.prompts[0].text.split(/\s+/),b=item.prompts[1].text.split(/\s+/);assert.equal(a.length,b.length);assert.equal(a.filter((v,i)=>v!==b[i]).length,1); assert.equal(item.comparison.prompt_words_changed,1);assert.deepEqual(item.comparison.prompt_token_counts,[item.conditions[0].metrics.prompt_tokens,item.conditions[1].metrics.prompt_tokens]); if(item.id.endsWith('flint')){ assert.equal(item.comparison.routed_blocks_compared,47);assert.ok(item.comparison.blocks_with_different_expert_sets>0); for(const condition of item.conditions){assert.equal(condition.routing.layer,item.comparison.displayed_zero_based_block);assert.equal(condition.routing.tokens[0].experts.length,6);} }else for(const condition of item.conditions)assert.equal(condition.routing,undefined); } }); test('every blind pair uses one matched Q4_1 and Q8_0 checkpoint and keeps identical answers',()=>{ const pairs=data.experiments.quant.cases;assert.equal(pairs.length,12); assert.equal(pairs.filter(x=>x.conditions[0].output===x.conditions[1].output).length,11); for(const item of pairs){assert.deepEqual(item.conditions.map(x=>x.quant).sort(),['Q4_1','Q8_0']);for(const c of item.conditions){assert.equal(c.metrics.complete,true);assert.equal(c.metrics.termination,'eos');assert.ok(c.metrics.process_peak_rss_kib>0);}} const arithmetic=pairs.find(x=>x.id==='flint-arithmetic');assert.equal(17*23-19,372); for(const c of arithmetic.conditions){assert.notEqual(Number(c.output),17*23-19);assert.equal(c.metrics.objective_check_passed,false);} }); test('every saved intervention frame follows its exact requested scale',()=>{ let altered=0,unchanged=0,total=0; for(const item of data.experiments.surgery.cases){ assert.equal(item.conditions.length,3);const base=item.conditions[0]; assert.deepEqual(item.conditions.map(c=>c.settings.intervention_scale),[1,.5,0]); for(const c of item.conditions){ assert.equal(c.metrics.termination,'eos'); for(const f of c.measurements){ const row=f.layers.find(x=>x.layer===12&&x.tensor==='ffn_out');assert.ok(row);const expected=row.before_rms*c.settings.intervention_scale; assert.ok(Math.abs(row.after_rms-expected)<=Math.max(1e-10,Math.abs(expected)*1e-7));total++; } const layers=c.measurements[0].layers,original=base.measurements[0].layers; const rms=(rows,layer,tensor)=>rows.find(x=>x.layer===layer&&x.tensor===tensor).after_rms; assert.equal(rms(layers,11,'ffn_out'),rms(original,11,'ffn_out'));unchanged++; if(c.settings.intervention_scale!==1){assert.notEqual(rms(layers,13,'l_out'),rms(original,13,'l_out'));altered++;} } } assert.equal(total,132);assert.equal(unchanged,18);assert.equal(altered,12); for(const id of ['baseline','half','zero'])assert.equal(data.experiments.surgery.cases.filter(x=>x.conditions.find(c=>c.id===id).check.passed).length,4); }); test('memory results separately retain original-fact and correction tasks',()=>{ const cases=data.experiments.memory.cases;assert.equal(cases.length,6);assert.equal(cases.filter(x=>x.kind==='current').length,3);assert.equal(cases.filter(x=>x.kind==='original').length,3); let count=0;for(const item of cases){assert.ok(item.choices.includes(item.ground_truth));for(const c of item.conditions){assert.equal(c.output.trim(),item.ground_truth);assert.equal(c.metrics.exact_answer_passed,true);assert.ok(c.metrics.prompt_tokens<=144);count++;}} assert.equal(count,12);assert.equal(summary.memory_answers_passed,count); }); test('checkpoint and assembled source provenance agree with the reproduction manifests',()=>{ const checkpoints=JSON.parse(fs.readFileSync(path.join(root,'experiments/reproduce/checkpoints.json'))); for(const c of data.provenance.checkpoints){const model=checkpoints[c.model][c.quantization];assert.equal(c.file_bytes,model.bytes);assert.equal(c.sha256,model.sha256);} assert.equal(data.provenance.source_tree,'83e9d4c276f84753cce063502a2a9e2325f15ce3'); assert.equal(sha(fs.readFileSync(path.join(root,'experiments/reproduce/llama-flint-runtime.patch'))),data.provenance.source_patch_sha256); assert.equal(registry.find(x=>x.id==='qwen').title,'alphaBell Qwen Open 27B'); }); test('downloadable clips have verified media headers and WAVs exactly matching the public mapping',()=>{ const media=JSON.parse(fs.readFileSync(path.join(root,'experiments/assets/media.json'))); const sound=require('../experiments/sound.js'),original=JSON.parse(fs.readFileSync(path.join(root,'traces/index.json'))); for(const entry of media.files){const bytes=fs.readFileSync(path.join(root,'experiments/assets',entry.file));assert.equal(bytes.length,entry.bytes);assert.equal(sha(bytes),entry.sha256);assert.equal(bytes.subarray(0,4).toString('hex'),entry.file.endsWith('.wav')?'52494646':'1a45dfa3');} for(const model of ['flint','qwen']){ const row=original.find(x=>x.model===model),trace=JSON.parse(zlib.gunzipSync(fs.readFileSync(path.join(root,row.file)))); const expected=Buffer.from(sound.encodeWAV(sound.renderPCM(sound.buildScore(trace,model==='flint'?12:0,.35)),.35)); assert.deepEqual(fs.readFileSync(path.join(root,'experiments/assets',model==='flint'?'flint-music.wav':'qwen-open-music.wav')),expected); } });