tests/experiment-data.test.cjs

'use strict';
const test=require('node:test'),assert=require('node:assert/strict'),fs=require('node:fs'),path=require('node:path'),zlib=require('node:zlib'),crypto=require('node:crypto');
const root=path.resolve(__dirname,'..');
const data=JSON.parse(fs.readFileSync(path.join(root,'experiments/data/results.json')));
const summary=JSON.parse(fs.readFileSync(path.join(root,'experiments/data/summary.json')));
const manifest=JSON.parse(fs.readFileSync(path.join(root,'experiments/data/traces.json')));
const registry=JSON.parse(fs.readFileSync(path.join(root,'models.json')));
const {validateObservatoryTrace}=require('../trace-validation.js');
const sha=b=>crypto.createHash('sha256').update(b).digest('hex');
test('all 24 new topology captures retain valid measured frames and both checksums',()=>{
 assert.equal(manifest.length,24);let frames=0;
 for(const row of manifest){
  const compressed=fs.readFileSync(path.join(root,row.file)),plain=zlib.gunzipSync(compressed),trace=JSON.parse(plain);
  assert.equal(sha(compressed),row.gzip_sha256);assert.equal(sha(plain),row.sha256);assert.equal(trace.id,row.id);assert.equal(trace.result.reason,'eos');assert.equal(trace.frames.length,row.tokens);
  validateObservatoryTrace(trace,registry);frames+=trace.frames.length;
 }
 assert.equal(frames,80);assert.equal(summary.full_capture_frames,frames);
});
test('prompt pairs change exactly one word and compare the first prediction',()=>{
 const cases=data.experiments.reroute.cases;assert.equal(cases.length,6);
 for(const item of cases){
  const a=item.prompts[0].text.split(/\s+/),b=item.prompts[1].text.split(/\s+/);assert.equal(a.length,b.length);assert.equal(a.filter((v,i)=>v!==b[i]).length,1);
  assert.equal(item.comparison.prompt_words_changed,1);assert.deepEqual(item.comparison.prompt_token_counts,[item.conditions[0].metrics.prompt_tokens,item.conditions[1].metrics.prompt_tokens]);
  if(item.id.endsWith('flint')){
   assert.equal(item.comparison.routed_blocks_compared,47);assert.ok(item.comparison.blocks_with_different_expert_sets>0);
   for(const condition of item.conditions){assert.equal(condition.routing.layer,item.comparison.displayed_zero_based_block);assert.equal(condition.routing.tokens[0].experts.length,6);}
  }else for(const condition of item.conditions)assert.equal(condition.routing,undefined);
 }
});
test('every blind pair uses one matched Q4_1 and Q8_0 checkpoint and keeps identical answers',()=>{
 const pairs=data.experiments.quant.cases;assert.equal(pairs.length,12);
 assert.equal(pairs.filter(x=>x.conditions[0].output===x.conditions[1].output).length,11);
 for(const item of pairs){assert.deepEqual(item.conditions.map(x=>x.quant).sort(),['Q4_1','Q8_0']);for(const c of item.conditions){assert.equal(c.metrics.complete,true);assert.equal(c.metrics.termination,'eos');assert.ok(c.metrics.process_peak_rss_kib>0);}}
 const arithmetic=pairs.find(x=>x.id==='flint-arithmetic');assert.equal(17*23-19,372);
 for(const c of arithmetic.conditions){assert.notEqual(Number(c.output),17*23-19);assert.equal(c.metrics.objective_check_passed,false);}
});
test('every saved intervention frame follows its exact requested scale',()=>{
 let altered=0,unchanged=0,total=0;
 for(const item of data.experiments.surgery.cases){
  assert.equal(item.conditions.length,3);const base=item.conditions[0];
  assert.deepEqual(item.conditions.map(c=>c.settings.intervention_scale),[1,.5,0]);
  for(const c of item.conditions){
   assert.equal(c.metrics.termination,'eos');
   for(const f of c.measurements){
    const row=f.layers.find(x=>x.layer===12&&x.tensor==='ffn_out');assert.ok(row);const expected=row.before_rms*c.settings.intervention_scale;
    assert.ok(Math.abs(row.after_rms-expected)<=Math.max(1e-10,Math.abs(expected)*1e-7));total++;
   }
   const layers=c.measurements[0].layers,original=base.measurements[0].layers;
   const rms=(rows,layer,tensor)=>rows.find(x=>x.layer===layer&&x.tensor===tensor).after_rms;
   assert.equal(rms(layers,11,'ffn_out'),rms(original,11,'ffn_out'));unchanged++;
   if(c.settings.intervention_scale!==1){assert.notEqual(rms(layers,13,'l_out'),rms(original,13,'l_out'));altered++;}
  }
 }
 assert.equal(total,132);assert.equal(unchanged,18);assert.equal(altered,12);
 for(const id of ['baseline','half','zero'])assert.equal(data.experiments.surgery.cases.filter(x=>x.conditions.find(c=>c.id===id).check.passed).length,4);
});
test('memory results separately retain original-fact and correction tasks',()=>{
 const cases=data.experiments.memory.cases;assert.equal(cases.length,6);assert.equal(cases.filter(x=>x.kind==='current').length,3);assert.equal(cases.filter(x=>x.kind==='original').length,3);
 let count=0;for(const item of cases){assert.ok(item.choices.includes(item.ground_truth));for(const c of item.conditions){assert.equal(c.output.trim(),item.ground_truth);assert.equal(c.metrics.exact_answer_passed,true);assert.ok(c.metrics.prompt_tokens<=144);count++;}}
 assert.equal(count,12);assert.equal(summary.memory_answers_passed,count);
});
test('checkpoint and assembled source provenance agree with the reproduction manifests',()=>{
 const checkpoints=JSON.parse(fs.readFileSync(path.join(root,'experiments/reproduce/checkpoints.json')));
 for(const c of data.provenance.checkpoints){const model=checkpoints[c.model][c.quantization];assert.equal(c.file_bytes,model.bytes);assert.equal(c.sha256,model.sha256);}
 assert.equal(data.provenance.source_tree,'83e9d4c276f84753cce063502a2a9e2325f15ce3');
 assert.equal(sha(fs.readFileSync(path.join(root,'experiments/reproduce/llama-flint-runtime.patch'))),data.provenance.source_patch_sha256);
 assert.equal(registry.find(x=>x.id==='qwen').title,'alphaBell Qwen Open 27B');
});
test('downloadable clips have verified media headers and WAVs exactly matching the public mapping',()=>{
 const media=JSON.parse(fs.readFileSync(path.join(root,'experiments/assets/media.json')));
 const sound=require('../experiments/sound.js'),original=JSON.parse(fs.readFileSync(path.join(root,'traces/index.json')));
 for(const entry of media.files){const bytes=fs.readFileSync(path.join(root,'experiments/assets',entry.file));assert.equal(bytes.length,entry.bytes);assert.equal(sha(bytes),entry.sha256);assert.equal(bytes.subarray(0,4).toString('hex'),entry.file.endsWith('.wav')?'52494646':'1a45dfa3');}
 for(const model of ['flint','qwen']){
  const row=original.find(x=>x.model===model),trace=JSON.parse(zlib.gunzipSync(fs.readFileSync(path.join(root,row.file))));
  const expected=Buffer.from(sound.encodeWAV(sound.renderPCM(sound.buildScore(trace,model==='flint'?12:0,.35)),.35));
  assert.deepEqual(fs.readFileSync(path.join(root,'experiments/assets',model==='flint'?'flint-music.wav':'qwen-open-music.wav')),expected);
 }
});