Spaces:

HalcyonSolutions
/

KGQA-Statistics-Lab

Running

App Files Files Community
KGQA-Statistics-Lab / stats-worker.js
muttsurini
Initial Push
8a34897 verified 18 days ago
Raw History Blame Contribute Delete
40.5 kB
'use strict';
importScripts('./stats-core.js?v=11','./vendor/jszip.min.js?v=11');
let busy=false;
const SEP='\u0001';
const post=(type,detail={})=>self.postMessage({type,...detail});
const quantile=(arr,p)=>self.KGCore.quantile(arr,p);
function mean(a){return a.length?a.reduce((x,y)=>x+y,0)/a.length:0;}
function std(a){if(!a.length)return 0; const m=mean(a); return Math.sqrt(mean(a.map(x=>(x-m)*(x-m))));}
function summarize(a){const sorted=[...a].filter(Number.isFinite).sort((x,y)=>x-y); if(!sorted.length)return {n:0,mean:0,median:0,p90:0,p95:0,p99:0,max:0,std:0}; return {n:sorted.length,mean:mean(sorted),median:quantile(sorted,.5),p90:quantile(sorted,.9),p95:quantile(sorted,.95),p99:quantile(sorted,.99),max:sorted[sorted.length-1],std:std(sorted)};}
function csvFields(line,separator){
  if(separator==='\t')return line.split('\t');
  const cells=[];let cell='',quote=false;
  for(let i=0;i<line.length;i++){
    const c=line[i]; if(c==='"'){ if(quote&&line[i+1]==='"'){cell+='"';i++;} else quote=!quote; }
    else if(c===separator&&!quote){cells.push(cell);cell='';}
    else cell+=c;
  }
  cells.push(cell); return cells;
}
function detectSeparator(line, fallback='auto'){
  if(fallback && fallback!=='auto') return fallback;
  const commas=(line.match(/,/g)||[]).length, tabs=(line.match(/\t/g)||[]).length, semis=(line.match(/;/g)||[]).length;
  if(tabs>=commas&&tabs>=semis&&tabs>0)return '\t';
  if(commas>=semis&&commas>0)return ',';
  return ';';
}
// The page keeps a Blob backup of completed downloads, allowing this Worker
// to be terminated on Cancel without losing cached datasets. No disk writes.
const archiveCache=new Map();
const restoredArchiveBlobs=new Map();
const remoteFileCache=new Map();
const cachedBytes=new Map();
function publishCacheStatus(){
  post('cache-status',{entries:cachedBytes.size,bytes:[...cachedBytes.values()].reduce((sum,size)=>sum+size,0)});
}
function rememberDownload(key,size){
  cachedBytes.set(key,size);
  publishCacheStatus();
}
async function getArchive(archiveUrl){
  const cacheKey=`gcs:${archiveUrl}`;
  if(!archiveCache.has(archiveUrl)){
    const loading=(async()=>{
      if(restoredArchiveBlobs.has(archiveUrl)){
        const saved=restoredArchiveBlobs.get(archiveUrl);
        post('progress',{phase:'Using cached dataset',detail:'Restoring GCS archive after cancellation',fraction:null});
        try{return await self.JSZip.loadAsync(await saved.arrayBuffer(),{checkCRC32:false});}
        catch(error){
          // If cached bytes cannot be reopened, evict and retry the network.
          restoredArchiveBlobs.delete(archiveUrl);
          cachedBytes.delete(cacheKey);
          post('cache-evict',{provider:'gcs',url:archiveUrl});
          publishCacheStatus();
        }
      }
      post('progress',{phase:'Downloading Google Cloud Storage',detail:decodeURIComponent(archiveUrl.split('?')[0].split('%2F').pop()),fraction:null});
      let response;
      try{response=await fetch(archiveUrl,{redirect:'follow'});}catch(error){
        throw new Error(`Cannot read the Google Cloud Storage archive. Check public object access, the GCS JSON API URL, and browser network access. ${error?.message||error}`);
      }
      if(!response.ok){
        // Keep the response detail for billing and access errors, without downloading another file.
        const detail=(await response.text().catch(()=>'' )).slice(0,350).replace(/\s+/g,' ');
        throw new Error(`Google Cloud Storage archive download failed (HTTP ${response.status})${detail?`: ${detail}`:''}: ${archiveUrl}`);
      }
      const contentType=response.headers.get('content-type')||'';
      const buffer=await response.arrayBuffer();
      if(buffer.byteLength<4 || new Uint8Array(buffer,0,2)[0]!==0x50 || new Uint8Array(buffer,0,2)[1]!==0x4b)throw new Error(`The GCS JSON API response is not a ZIP archive (content type: ${contentType||'unknown'}). Verify that this object is publicly readable.`);
      post('progress',{phase:'Opening archive',detail:`${(buffer.byteLength/1048576).toFixed(1)} MiB · ${decodeURIComponent(archiveUrl.split('?')[0].split('%2F').pop())}`,fraction:null});
      let zip;
      try{zip=await self.JSZip.loadAsync(buffer,{checkCRC32:false});}
      catch(error){throw new Error(`Unable to decode downloaded ZIP (${contentType}). ${error?.message||error}`);}
      // Post only validated archives. Blob cloning lets the page retain the
      // bytes even if the current Worker must be terminated for cancellation.
      post('cache-entry',{provider:'gcs',url:archiveUrl,blob:new Blob([buffer],{type:'application/zip'})});
      rememberDownload(cacheKey,buffer.byteLength);
      return zip;
    })();
    archiveCache.set(archiveUrl,loading);
    loading.catch(()=>{
      if(archiveCache.get(archiveUrl)===loading){archiveCache.delete(archiveUrl);cachedBytes.delete(cacheKey);publishCacheStatus();}
    });
  }else{
    post('progress',{phase:'Using cached dataset',detail:'Google Cloud Storage archive',fraction:null});
  }
  return archiveCache.get(archiveUrl);
}
async function getRemoteFile(url){
  const cacheKey=`hf:${url}`;
  if(!remoteFileCache.has(url)){
    const loading=(async()=>{
      post('progress',{phase:'Downloading Hugging Face',detail:decodeURIComponent(url.split('/').pop()),fraction:null});
      const response=await fetch(url,{redirect:'follow'});
      if(!response.ok)throw Error(`Hugging Face download failed (HTTP ${response.status}): ${url}`);
      const file=await response.blob();
      post('cache-entry',{provider:'hf',url,blob:file});
      rememberDownload(cacheKey,file.size);
      return file;
    })();
    remoteFileCache.set(url,loading);
    loading.catch(()=>{
      if(remoteFileCache.get(url)===loading){remoteFileCache.delete(url);cachedBytes.delete(cacheKey);publishCacheStatus();}
    });
  }else{
    post('progress',{phase:'Using cached dataset',detail:'Hugging Face file',fraction:null});
  }
  return remoteFileCache.get(url);
}
function findArchiveFile(zip,root,path){
  // Direct archive paths are not published separately. The documented ZIPs
  // contain <dataset_root>/<relative_path>, optionally under another folder.
  const cleaned=path.replace(/^\/+/,''), prefix=(root||'').replace(/^\/+|\/+$/g,'');
  const preferred=[`${prefix}/${cleaned}`,`raw_data/${prefix}/${cleaned}`,cleaned];
  for(const candidate of preferred){const entry=zip.files[candidate];if(entry&&!entry.dir)return entry;}
  const tail=`/${prefix}/${cleaned}`;
  const candidates=Object.entries(zip.files).filter(([name,entry])=>!entry.dir && (name.endsWith(tail) || (!prefix&&name.endsWith('/'+cleaned))));
  if(candidates.length===1)return candidates[0][1];
  if(candidates.length>1)throw new Error(`Ambiguous archive entry for ${cleaned}: ${candidates.length} matches.`);
  throw new Error(`Required file '${prefix}/${cleaned}' was not found in the selected archive. Check the published archive contents.`);
}
async function archiveEntryBlob(source){
  const zip=await getArchive(source.archiveUrl);
  const entry=findArchiveFile(zip,source.root,source.entryPath);
  post('progress',{phase:'Extracting archive',detail:source.entryPath,fraction:null});
  const bytes=await entry.async('uint8array',m=>post('progress',{phase:'Extracting archive',detail:`${source.entryPath} · ${Math.round(m.percent)}%`,fraction:null}));
  return new Blob([bytes]);
}
function csvMappings(content){
  const lines=content.replace(/^\ufeff/,'').split(/\r?\n/).filter(x=>x.trim());
  if(!lines.length)return [];
  const sep=detectSeparator(lines[0],'auto');
  const header=csvFields(lines[0],sep).map(x=>x.trim().toLowerCase());
  const id=header.findIndex(x=>['qid','eid','property','rid','id','entity','relation'].includes(x));
  const title=header.findIndex(x=>['title','name','label'].includes(x));
  if(id<0||title<0)return [];
  const values=[];
  for(const line of lines.slice(1)){
    const cells=csvFields(line,sep),k=String(cells[id]??'').trim(),v=String(cells[title]??'').trim();
    if(k&&v)values.push([k,v]);
  }
  return values;
}
async function publishArchiveLabels(source){
  if(!source||source.mode!=='archive')return;
  if(!source.relationPath&&!source.entityPath)return;
  const zip=await getArchive(source.archiveUrl);
  async function getEntries(path){
    if(!path)return [];
    try{return csvMappings(await findArchiveFile(zip,source.root,path).async('string'));}
    catch(error){post('progress',{phase:'Metadata warning',detail:`${path}: ${error?.message||error}`,fraction:null});return [];}
  }
  const [relations,entities]=await Promise.all([getEntries(source.relationPath),getEntries(source.entityPath)]);
  post('labels',{relations,entities});
}
async function consume(source,consumeLine,onProgress){
  let stream,total=0;
  if(source.mode==='url'){
    const file=await getRemoteFile(source.url);
    total=file.size; stream=file.stream();
  }else if(source.mode==='archive'){
    const file=await archiveEntryBlob(source);
    total=file.size; stream=file.stream();
  }else{
    if(!source.file) throw Error('No file selected.');
    total=source.file.size; stream=source.file.stream();
  }
  const reader=stream.getReader(), decoder=new TextDecoder('utf-8'); let pending='', read=0, last=0;
  while(true){ const {done,value}=await reader.read(); if(done) break; read+=value.byteLength; pending+=decoder.decode(value,{stream:true});
    let lineEnd, at=0; while((lineEnd=pending.indexOf('\n',at))>=0){ consumeLine(pending.slice(at,lineEnd)); at=lineEnd+1; }
    pending=pending.slice(at);
    if(performance.now()-last>200){ onProgress(read,total); last=performance.now(); }
  }
  pending+=decoder.decode(); if(pending) consumeLine(pending); onProgress(read,total);
}
function parsePyLiteral(text){
  let i=0; text=String(text??'').trim();
  function skip(){ while(i<text.length && /\s/.test(text[i])) i++; }
  function parseString(quote){ i++; let out=''; while(i<text.length){ const ch=text[i++]; if(ch==='\\'){ if(i<text.length){ const next=text[i++]; const map={n:'\n',r:'\r',t:'\t',"'":"'",'"':'"','\\':'\\'}; out+=map[next]!==undefined?map[next]:next; } }
      else if(ch===quote) return out; else out+=ch; } return out; }
  function parseToken(){ const start=i; while(i<text.length && /[^\s,\]\)\}]/.test(text[i])) i++; const token=text.slice(start,i); if(token==='None') return null; if(token==='True') return true; if(token==='False') return false; if(/^[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?$/.test(token)) return Number(token); return token; }
  function parseValue(){ skip(); const ch=text[i]; if(ch==='['||ch==='('){ const close=ch==='['?']':')'; i++; const arr=[]; skip(); if(text[i]===close){ i++; return arr; }
      while(i<text.length){ arr.push(parseValue()); skip(); if(text[i]===','){ i++; skip(); if(text[i]===close){ i++; break; } continue; } if(text[i]===close){ i++; break; } break; } return arr; }
    if(ch==='"'||ch==="'") return parseString(ch); return parseToken(); }
  try{return parseValue();}catch{return text;}
}
function parseMaybeLiteral(value){
  if(value===null||value===undefined) return value;
  if(Array.isArray(value)) return value;
  const s=String(value).trim(); if(!s) return '';
  if((s.startsWith('[')&&s.endsWith(']'))||(s.startsWith('(')&&s.endsWith(')'))||(s.startsWith('{')&&s.endsWith('}'))) return parsePyLiteral(s);
  return s;
}
function normalizeAnswers(value){
  const parsed=parseMaybeLiteral(value);
  if(Array.isArray(parsed)){
    const out=[]; const stack=[parsed];
    while(stack.length){ const item=stack.pop(); if(Array.isArray(item)) for(let j=item.length-1;j>=0;j--) stack.push(item[j]); else if(item!==null&&item!==undefined&&String(item).trim()) out.push(String(item).trim()); }
    return [...new Set(out)];
  }
  const s=String(parsed??'').trim(); return s?[s]:[];
}
function isTriplet(x){ return Array.isArray(x) && x.length>=3 && !Array.isArray(x[0]) && !Array.isArray(x[1]) && !Array.isArray(x[2]); }
function flattenMaterializedPaths(value){
  const parsed=parseMaybeLiteral(value); const out=[];
  function walk(node){
    if(!Array.isArray(node)) return;
    if(isTriplet(node)){ out.push([[String(node[0]),String(node[1]),String(node[2])]]); return; }
    if(node.length && node.every(isTriplet)){ out.push(node.map(t=>[String(t[0]),String(t[1]),String(t[2])])); return; }
    for(const child of node) walk(child);
  }
  walk(parsed); return out;
}
function countDistinctParaphrases(row){
  const values=[String(row['Question']||'').trim()];
  const alt=parseMaybeLiteral(row['Question-Paraphrased']);
  if(Array.isArray(alt)) for(const x of alt){ const s=String(x??'').trim(); if(s) values.push(s); }
  else if(String(alt||'').trim()) values.push(String(alt).trim());
  return [...new Set(values.filter(Boolean))].length;
}
function normalizeQuestionVariants(row){
  const canonical=String(row['Question']??'');
  const parsed=parseMaybeLiteral(row['Question-Paraphrased']);
  const variants=[];
  if(Array.isArray(parsed)) for(const value of parsed){if(value!==null&&value!==undefined&&String(value).trim())variants.push(String(value));}
  else if(parsed!==null&&parsed!==undefined&&String(parsed).trim()) variants.push(String(parsed));
  if(canonical&&!variants.includes(canonical))variants.unshift(canonical);
  if(!variants.length&&canonical)variants.push(canonical);
  return [...new Set(variants)];
}
function distinctPaths(paths){
  const seen=new Set(),out=[];
  for(const path of paths){const key=JSON.stringify(path);if(!seen.has(key)){seen.add(key);out.push(path);}}
  return out;
}
function referencePathProbability(path,graph,numSteps,useSelfLoops){
  if(path.length>numSteps || !path.length)return NaN;
  let p=1;
  for(const [h,r,t] of path){
    const choices=graph.actionTargets.get(h)||[];
    if(!choices.length||!graph.actionTriples.has(h+SEP+r+SEP+t))return NaN;
    p/=choices.length;
  }
  if(path.length<numSteps){
    if(!useSelfLoops)return NaN;
    const endpoint=path[path.length-1][2];
    const choices=graph.actionTargets.get(endpoint)||[];
    if(!choices.length)return NaN;
    for(let i=path.length;i<numSteps;i++)p/=choices.length;
  }
  return p;
}
function getPathKeys(row){
  // Match the Python evaluator's field preference and two distinct formats.
  // PQ/PQL: ['r1->r2', 'r3->r4'] means separate complete chains.
  // Legacy: ['r1', 'r2'] denotes the relations in one complete chain.
  for(const key of ['Multi-Paths-Key','Paths-Key','Path-Key','Path-Keys']){
    if(!(key in row))continue;
    const parsed=parseMaybeLiteral(row[key]);
    if(Array.isArray(parsed)){
      if(!parsed.length)return [];
      if(parsed.every(x=>typeof x==='string' && x.includes('->')))return [...new Set(parsed.map(x=>x.trim()))];
      return [parsed.map(x=>String(x).trim()).join('->')];
    }
    if(parsed!==null && parsed!==undefined && String(parsed).trim())return [String(parsed).trim()];
    return [];
  }
  return [];
}
function releasedReferencePaths(row){
  // Released reference annotations must not be mixed with Graph-Multi-Paths.
  // Keep Python's preference order and deduplicate identical entity paths.
  const multi=distinctPaths(flattenMaterializedPaths(row['Multi-Paths']));
  if(multi.length)return multi;
  return distinctPaths(flattenMaterializedPaths(row['Paths']));
}
function fieldNumber(row,key){
  const v=String(row[key]??'').trim();
  if(!v)return null;
  const num=Number(v);return Number.isFinite(num)?num:null;
}
function getFamilyId(row, idx){
  const direct=String(row['Question-Family-ID']||'').trim(); if(direct) return direct;
  const qn=String(row['Question-Number']||'').trim(); if(qn) return qn;
  const parts=[String(row['Source-Entity']||'').trim(), String(row['Path-Key']||'').trim(), String(row['Hops']||'').trim()];
  return parts.some(Boolean)?parts.join('|'):`row-${idx}`;
}
function nStepDistributionSparse(source, steps, targetsMap){
  let dist=new Map([[source,1]]);
  for(let step=0; step<steps; step++){
    const next=new Map();
    for(const [node,p] of dist){
      const targets=targetsMap.get(node);
      if(!targets||!targets.length){ next.set(node,(next.get(node)||0)+p); continue; }
      const mass=p/targets.length;
      for(const t of targets) next.set(t,(next.get(t)||0)+mass);
    }
    dist=next;
  }
  return dist;
}
function sumAnswerProbability(dist, answers){ let total=0; for(const a of answers) total+=dist.get(a)||0; return total; }
function resolvePathsByKey(source, pathKey, answersSet, seqIndex, limit=10000){
  const rels=String(pathKey||'').split('->').map(x=>x.trim()).filter(Boolean); if(!rels.length) return [];
  const out=[], path=[];
  function dfs(node, idx){
    if(out.length>=limit) return;
    if(idx===rels.length){ if(answersSet.has(node)) out.push(path.slice()); return; }
    const tails=seqIndex.get(node+SEP+rels[idx])||[];
    for(const tail of tails){ path.push([node, rels[idx], tail]); dfs(tail, idx+1); path.pop(); if(out.length>=limit) break; }
  }
  dfs(source,0); return out;
}
function hasCycle(path){ const entities=path.length?[path[0][0], ...path.map(t=>t[2])]:[]; return entities.length!==new Set(entities).size; }
function hasImmediateBacktrack(path){ const entities=path.length?[path[0][0], ...path.map(t=>t[2])]:[]; for(let i=0;i+2<entities.length;i++) if(entities[i]===entities[i+2]) return true; return false; }
function questionReferenceStateStats(paths, actionTargets, relCounts){
  const seen=new Set(), actionCounts=[], relationCounts=[];
  for(const path of paths){ if(!path.length) continue; const entities=[path[0][0], ...path.map(t=>t[2])]; for(let hop=0; hop<entities.length-1; hop++){
      const key=hop+SEP+entities[hop]; if(seen.has(key)) continue; seen.add(key); actionCounts.push((actionTargets.get(entities[hop])||[]).length); relationCounts.push(relCounts.get(entities[hop])||0);
  }}
  return [actionCounts, relationCounts];
}
async function loadGraph(source, entityFile, options, entityIdColumn=null){
  const t0=performance.now(), acc=self.KGCore.createAccumulator(); let ignored=0, header=false, chosen=null, first=true;
  const unique=new Set(), actionTriples=new Set(), actionTargets=new Map(), neighborTargets=new Map(), relationCounts=new Map(), seqIndex=new Map(), nodes=new Set();
  const addTraversal=(from,relation,to)=>{
    // The Python evaluator adds a reverse action for EACH unique released
    // directed triple. Keep the resulting action multiplicity even if a
    // reverse triple already exists in the source graph.
    actionTriples.add(from+SEP+relation+SEP+to);
    if(!actionTargets.has(from))actionTargets.set(from,[]);
    actionTargets.get(from).push(to);
    if(!neighborTargets.has(from))neighborTargets.set(from,new Set());
    neighborTargets.get(from).add(to);
    if(!relationCounts.has(from))relationCounts.set(from,new Set());
    relationCounts.get(from).add(relation);
  };
  const addUnique=(h,r,t)=>{
    const key=h+SEP+r+SEP+t; if(unique.has(key)) return; unique.add(key); nodes.add(h); nodes.add(t);
    // Path reconstruction uses only forward triples from the active KG.
    const seqKey=h+SEP+r;
    if(!seqIndex.has(seqKey))seqIndex.set(seqKey,[]);
    seqIndex.get(seqKey).push(t);
    addTraversal(h,r,t);
    if(options?.useUndirectedActions)addTraversal(t,r,h);
  };
  const handleLine=line=>{
    line=line.replace(/\r$/,'').replace(/^\ufeff/,''); if(!line.trim()) return;
    if(!chosen) chosen=detectSeparator(line, source.delimiter);
    const fields=csvFields(line, chosen);
    if(first){ first=false; if(fields.length>=3 && /^head$|^source$|^subject$|^head[-_ ]entity$/i.test(fields[0].trim()) && /^relation$|^predicate$|^relationship$/i.test(fields[1].trim()) && /^tail$|^target$|^object$|^tail[-_ ]entity$/i.test(fields[2].trim())){ header=true; return; } }
    if(fields.length!==3){ ignored++; return; }
    const h=String(fields[0]||'').trim(), r=String(fields[1]||'').trim(), t=String(fields[2]||'').trim();
    acc.add(h,r,t); if(h&&r&&t) addUnique(h,r,t);
  };
  post('progress',{phase:'Reading graph',detail:'Loading graph triples',fraction:0});
  await consume(source, handleLine, (done,total)=>post('progress',{phase:'Reading graph',detail:`${acc.rows.toLocaleString()} triples read`,fraction:total?done/total:null}));
  if(acc.rows===0) throw Error('No valid triples were found in the selected graph source.');
  if(entityFile && options?.includeMetadataEntities){
    post('progress',{phase:'Reading graph',detail:'Including metadata-only entities by request',fraction:null});
    let firstEntity=true, entityDelimiter=null;
    await consume({mode:'file',file:entityFile}, line=>{
      const value=line.replace(/^\ufeff/,'').replace(/\r$/,''); if(!value.trim()) return;
      if(!entityDelimiter) entityDelimiter=detectSeparator(value,'auto');
      const fields=csvFields(value,entityDelimiter);
      // Multi-column entity metadata has a header. Single-column lists may
      // optionally have a standard ID header, or start directly with IDs.
      if(firstEntity){
        firstEntity=false;
        if(fields.length>1 || /^(qid|eid|entity|entity[-_ ]?id|node|node[-_ ]?id|id)$/i.test(fields[0].trim())) return;
      }
      const column=entityIdColumn===null?0:entityIdColumn;
      const id=String(fields[column]??'').trim();
      if(id){acc.addNode(id);nodes.add(id);}
    }, ()=>{});
  }
  post('progress',{phase:'Computing graph metrics',detail:`${nodes.size.toLocaleString()} entities`,fraction:null});
  const result=acc.output(options||{}); result.parse.ignored_malformed_rows=ignored; result.parse.detected_delimiter=chosen==='\t'?'Tab':chosen===','?'Comma':'Semicolon'; result.parse.header_skipped=header; result.runtime_seconds=(performance.now()-t0)/1000; result.source={name:source.name,mode:source.mode,repo:source.repo||null,archive_url:source.archiveUrl||null,entry_path:source.entryPath||null,filename:source.file?.name||null,entity_filename:entityFile?.name||null,include_metadata_only_entities:Boolean(entityFile && options?.includeMetadataEntities)};
  // Optional explicit labeled SELF_LOOP action at every entity.
  // These artificial actions affect only the joint random walk, not graph structure.
  if(options?.useSelfLoops){
    for(const node of nodes){
      if(!actionTargets.has(node)) actionTargets.set(node,[]);
      actionTargets.get(node).push(node);
      actionTriples.add(node+SEP+'SELF_LOOP'+SEP+node);
      if(!neighborTargets.has(node)) neighborTargets.set(node,new Set());
      neighborTargets.get(node).add(node);
      if(!relationCounts.has(node)) relationCounts.set(node,new Set());
      relationCounts.get(node).add('SELF_LOOP');
    }
  }
  return {result, actionTargets, neighborTargets:new Map([...neighborTargets].map(([k,v])=>[k,[...v]])), relationCounts:new Map([...relationCounts].map(([k,v])=>[k,v.size])), seqIndex, actionTriples, nodes};
}
async function loadQA(source, splitFilter){
  const t0=performance.now(); let ignored=0, chosen=null, header=null, first=true, rows=[];
  post('progress',{phase:'Reading QA',detail:'Loading QA collection',fraction:0});
  await consume(source, line=>{
    line=line.replace(/\r$/,'').replace(/^\ufeff/,''); if(!line.trim()) return;
    if(!chosen) chosen=detectSeparator(line, source.delimiter);
    const fields=csvFields(line, chosen);
    if(first){ first=false; header=fields.map(x=>String(x).trim()); return; }
    if(!header || !header.length){ ignored++; return; }
    if(fields.length<2){ ignored++; return; }
    const row={}; for(let i=0;i<header.length;i++) row[header[i]]=fields[i]!==undefined?fields[i]:''; rows.push(row);
  }, (done,total)=>post('progress',{phase:'Reading QA',detail:`${rows.length.toLocaleString()} rows read`,fraction:total?done/total:null}));
  if(!rows.length) throw Error('No QA rows were found in the selected QA source.');
  if(splitFilter && splitFilter!=='all' && rows.some(r=>'SplitLabel' in r)) rows=rows.filter(r=>String(r['SplitLabel']||'').trim().toLowerCase()===splitFilter);
  const qa={rows, parse:{ignored_malformed_rows:ignored, detected_delimiter:chosen==='\t'?'Tab':chosen===','?'Comma':'Semicolon'}, runtime_seconds:(performance.now()-t0)/1000, source:{name:source.name,mode:source.mode,repo:source.repo||null,archive_url:source.archiveUrl||null,entry_path:source.entryPath||null,filename:source.file?.name||null}};
  return qa;
}
function analyzeQA(qa){
  const rows=qa.rows||[]; const hopCounts={}, splitCounts={}, wordingModes={canonical_only:0, has_paraphrases:0};
  const answerCounts=[], graphAnswerCounts=[], paraphraseCounts=[], refCounts=[], graphRefCounts=[], pathKeyCounts=[];
  let families=0, rowsWithMat=0, rowsWithKey=0, rowsWithPathCountField=0, rowsWithSplitLabel=0, topicIsAnswer=0, graphTopicIsAnswer=0, rowsCycle=0, rowsBack=0, rowsGraphAnswer=0, rowsGraphPaths=0, rowsGraphPathCountField=0;
  const famSet=new Set();
  const familyStats=new Map();
  const explicitFamilyIDs=rows.some(r=>String(r['Question-Family-ID']??'').trim() || String(r['Question-Number']??'').trim());
  rows.forEach((row,idx)=>{
    const family=getFamilyId(row, idx); if(family) famSet.add(family);
    const hops=Number(row['Hops']); if(Number.isFinite(hops)) hopCounts[hops]=(hopCounts[hops]||0)+1;
    const split=String(row['SplitLabel']||'').trim(); if(split){ splitCounts[split]=(splitCounts[split]||0)+1; rowsWithSplitLabel++; }
    const answers=normalizeAnswers(row['Answer-Entity'] ?? row['Answer']); answerCounts.push(answers.length); if(answers.includes(String(row['Source-Entity']||'').trim())) topicIsAnswer++;
    if(!familyStats.has(family))familyStats.set(family,{released:[],graph:[],paraphrases:[]});
    familyStats.get(family).released.push(answers.length);
    const pc=countDistinctParaphrases(row); paraphraseCounts.push(pc); familyStats.get(family).paraphrases.push(pc); if(pc>1) wordingModes.has_paraphrases++; else wordingModes.canonical_only++;
    const graphAnswers=normalizeAnswers(row['Graph-Answer-Entity']);
    if('Graph-Answer-Entity' in row){graphAnswerCounts.push(graphAnswers.length);familyStats.get(family).graph.push(graphAnswers.length);rowsGraphAnswer++;if(graphAnswers.includes(String(row['Source-Entity']||'').trim()))graphTopicIsAnswer++;}
    const mat=releasedReferencePaths(row);
    const keys=getPathKeys(row);
    pathKeyCounts.push(keys.length);
    if(keys.length)rowsWithKey++;
    if(mat.length){ rowsWithMat++; if(mat.some(hasCycle)) rowsCycle++; if(mat.some(hasImmediateBacktrack)) rowsBack++; }
    const explicitPathCount=fieldNumber(row,'Path-Count');
    if(explicitPathCount!==null)rowsWithPathCountField++;
    // Released paths and graph-relative paths represent different evidence sets.
    refCounts.push(mat.length || (explicitPathCount ?? keys.length));
    const graphMat=distinctPaths(flattenMaterializedPaths(row['Graph-Multi-Paths']));
    if(graphMat.length)rowsGraphPaths++;
    const graphCount=fieldNumber(row,'Graph-Path-Count');
    if(graphCount!==null)rowsGraphPathCountField++;
    if('Graph-Multi-Paths' in row||graphCount!==null)graphRefCounts.push(graphMat.length || (graphCount ?? 0));
  });
  const ans=summarize(answerCounts), ga=summarize(graphAnswerCounts), para=summarize(paraphraseCounts), refs=summarize(refCounts), gr=summarize(graphRefCounts), k=summarize(pathKeyCounts); families=famSet.size;
  const familyMacro=field=>{const values=[...familyStats.values()].filter(f=>f[field].length).map(f=>mean(f[field]));return values.length?mean(values):null;};
  qa.summary={qa_rows:rows.length,families,max_hops:Math.max(0,...Object.keys(hopCounts).map(Number)),answer_count_mean:ans.mean,answer_count_median:ans.median,answer_count_p95:ans.p95,answer_count_max:ans.max,paraphrase_count_mean:para.mean,paraphrase_count_median:para.median,paraphrase_count_p95:para.p95,paraphrase_count_max:para.max,reference_path_count_mean:refs.mean,reference_path_count_median:refs.median,reference_path_count_p95:refs.p95,reference_path_count_max:refs.max,
    graph_answer_count_mean:rowsGraphAnswer?ga.mean:null,graph_answer_count_median:rowsGraphAnswer?ga.median:null,graph_answer_count_p95:rowsGraphAnswer?ga.p95:null,graph_answer_count_max:rowsGraphAnswer?ga.max:null,
    graph_path_count_mean:graphRefCounts.length?gr.mean:null,graph_path_count_median:graphRefCounts.length?gr.median:null,graph_path_count_p95:graphRefCounts.length?gr.p95:null,graph_path_count_max:graphRefCounts.length?gr.max:null,
    relation_chain_count_mean:k.mean,relation_chain_count_max:k.max,rows_with_graph_answers:rowsGraphAnswer,rows_with_graph_paths:rowsGraphPaths,rows_with_graph_path_count_field:rowsGraphPathCountField,topic_is_graph_answer_rows:graphTopicIsAnswer,
    rows_with_materialized_paths:rowsWithMat,rows_with_path_key:rowsWithKey,rows_with_path_count_field:rowsWithPathCountField,rows_with_split_label:rowsWithSplitLabel,topic_is_answer_rows:topicIsAnswer,rows_with_cyclic_materialized_paths:rowsCycle,rows_with_backtrack_materialized_paths:rowsBack};
  qa.dashboard={has_family_ids:explicitFamilyIDs, family_macro:{
    answer_count_mean:familyMacro('released'), graph_answer_count_mean:familyMacro('graph'), paraphrase_count_mean:familyMacro('paraphrases')
  }};
  qa.hops=hopCounts; qa.splits=splitCounts; qa.wording_modes=wordingModes; qa.answer_distribution=Object.entries(answerCounts.reduce((m,v)=>(m[v]=(m[v]||0)+1,m),{})).map(([value,count])=>({value:Number(value),count})).sort((a,b)=>b.count-a.count||a.value-b.value);
  return qa;
}
function buildMetricRows(records, metric){
  const numeric=records.filter(r=>Number.isFinite(r[metric])); const allValues=numeric.map(r=>r[metric]); const out=[];
  function pushRows(subsetName, subsetRecords){
    const vals=subsetRecords.filter(r=>Number.isFinite(r[metric])).map(r=>r[metric]); if(!vals.length) return;
    const s=summarize(vals); out.push({metric, subset:subsetName, aggregation:'instance-micro', n:vals.length, mean:s.mean, median:s.median, std:s.std});
    const fam=new Map(); for(const r of subsetRecords){ if(!Number.isFinite(r[metric])) continue; const fid=r['Question-Family-ID']; if(!fid) continue; if(!fam.has(fid)) fam.set(fid,[]); fam.get(fid).push(r[metric]); }
    if(fam.size){ const perFamily=[...fam.values()].map(v=>mean(v)); const fs=summarize(perFamily); out.push({metric, subset:subsetName, aggregation:'family-macro', n:perFamily.length, mean:fs.mean, median:fs.median, std:fs.std}); }
  }
  pushRows('All', numeric);
  const hopGroups={}; for(const r of numeric){ const h=r.Hops; if(!Number.isFinite(h)) continue; (hopGroups[h]||(hopGroups[h]=[])).push(r); }
  Object.keys(hopGroups).sort((a,b)=>Number(a)-Number(b)).forEach(h=>pushRows(`${h}H`, hopGroups[h]));
  return out;
}
function analyzeJoint(graph, qa, options){
  const rows=qa.rows||[];
  const maxHop=Math.max(1,...rows.map(r=>Number(r['Hops'])||0));
  const runRW=options.runRandomWalk!==false;
  const rwStepsOpt=runRW&&options.rwSteps!=='auto'?Number(options.rwSteps):null;
  const numSteps=runRW?(rwStepsOpt||maxHop):null;
  if(runRW&&numSteps<maxHop)throw Error(`Random-walk horizon ${numSteps} is smaller than the selected QA's maximum hop count (${maxHop}).`);
  const useSelfLoops=Boolean(options.useSelfLoops);
  // Mirrors Python: expand only released rows grouped by Question-Number, not
  // rows with explicit Question-Family-ID. Apply after filtering by SplitLabel.
  const canExpand=Boolean(options.explodeParaphrases) && rows.some(r=>'Question-Paraphrased' in r) &&
    rows.some(r=>'Question-Number' in r) && !rows.some(r=>String(r['Question-Family-ID']??'').trim());
  const actionDistCache=new Map(), neighborDistCache=new Map(), refCache=new Map();
  const records=[];let missingReferencePaths=0, skippedMissingSource=0;
  const allRefActionCounts=[],allRefRelationCounts=[];
  const nodes=graph.nodes;
  for(let idx=0;idx<rows.length;idx++){
    const row=rows[idx];
    const source=String(row['Source-Entity']||'').trim();
    const variants=canExpand?normalizeQuestionVariants(row):[String(row['Question']??'')];
    if(!variants.length)throw Error(`QA row ${idx} has no usable canonical or paraphrased question.`);
    if(!source||!nodes.has(source)){skippedMissingSource+=variants.length;continue;}
    if(runRW&&!actionDistCache.has(source)){
      actionDistCache.set(source,nStepDistributionSparse(source,numSteps,graph.actionTargets));
      neighborDistCache.set(source,nStepDistributionSparse(source,numSteps,graph.neighborTargets));
    }
    const actionDist=runRW?actionDistCache.get(source):null;
    const neighborDist=runRW?neighborDistCache.get(source):null;
    const answers=normalizeAnswers(row['Answer-Entity']??row['Answer']);
    const graphAnswers=('Graph-Answer-Entity' in row)?normalizeAnswers(row['Graph-Answer-Entity']):null;
    const releasedPathCount=fieldNumber(row,'Path-Count');
    const graphPathCount=fieldNumber(row,'Graph-Path-Count');
    let refPaths=releasedReferencePaths(row);
    if(!refPaths.length){
      const keys=getPathKeys(row);
      if(keys.length&&answers.length){
        const ansKey=[...answers].sort().join('|');
        for(const key of keys){
          const cacheKey=source+SEP+key+SEP+ansKey;
          if(!refCache.has(cacheKey))refCache.set(cacheKey,resolvePathsByKey(source,key,new Set(answers),graph.seqIndex));
          refPaths.push(...refCache.get(cacheKey));
        }
        refPaths=distinctPaths(refPaths);
      }
    }
    let actionCounts=[],relationCounts=[];
    if(refPaths.length)[actionCounts,relationCounts]=questionReferenceStateStats(refPaths,graph.actionTargets,graph.relationCounts);
    let refProbability=NaN;
    if(runRW){
      const pathProbabilities=refPaths.map(p=>referencePathProbability(p,graph,numSteps,useSelfLoops)).filter(Number.isFinite);
      refProbability=pathProbabilities.length?Math.min(1,pathProbabilities.reduce((a,b)=>a+b,0)):NaN;
    }
    for(const variant of variants){
      const rec={
        'Question-Number':row['Question-Number']||'',
        'Question':variant,
        'Question-Family-ID':getFamilyId(row,idx),
        Hops:Number(row['Hops'])||NaN,
        'Answer-Count::Answer-Entity':answers.length,
        'Reference-Path-Count':refPaths.length,
        'Released-Answer-Count':answers.length,
        'Topic-Is-Released-Answer':answers.includes(source)?1:0
      };
      if(runRW){
        rec['Reachable-N-Count']=actionDist.size;
        rec['RW-Ans@p-actions::Answer-Entity']=sumAnswerProbability(actionDist,answers);
        rec['RW-Ans@p-neighbors::Answer-Entity']=sumAnswerProbability(neighborDist,answers);
        rec['RW-RefPath@p-actions']=refProbability;
      }
      if(graphAnswers!==null){
        rec['Answer-Count::Graph-Answer-Entity']=graphAnswers.length;
        if(runRW){
          rec['RW-Ans@p-actions::Graph-Answer-Entity']=sumAnswerProbability(actionDist,graphAnswers);
          rec['RW-Ans@p-neighbors::Graph-Answer-Entity']=sumAnswerProbability(neighborDist,graphAnswers);
        }
        rec['Graph-Answer-Count']=graphAnswers.length;
        rec['Topic-Is-Graph-Answer']=graphAnswers.includes(source)?1:0;
        rec['Graph-Answer-Expansion']=graphAnswers.length-answers.length;
      }
      if(releasedPathCount!==null)rec['Path-Count']=releasedPathCount;
      if(graphPathCount!==null){
        rec['Graph-Path-Count']=graphPathCount;
        if(releasedPathCount!==null)rec['Graph-Path-Expansion']=graphPathCount-releasedPathCount;
      }
      if(refPaths.length){
        rec['Released-Reference-Has-Cycle']=refPaths.some(hasCycle)?1:0;
        rec['Released-Reference-Has-Immediate-Backtrack']=refPaths.some(hasImmediateBacktrack)?1:0;
        if(actionCounts.length){
          const a=summarize(actionCounts);
          rec['Ref-Actions-Mean']=a.mean;rec['Ref-Actions-Median']=a.median;
          rec['Ref-Actions-P95']=a.p95;rec['Ref-Actions-Max']=a.max;
          rec['Ref-Actions-Frac-GT10']=mean(actionCounts.map(v=>v>10?1:0));
          rec['Ref-Actions-Frac-GT50']=mean(actionCounts.map(v=>v>50?1:0));
          rec['Ref-Actions-Frac-GT100']=mean(actionCounts.map(v=>v>100?1:0));
        }
        if(relationCounts.length){
          const r=summarize(relationCounts);
          rec['Ref-Relations-Mean']=r.mean;rec['Ref-Relations-P95']=r.p95;rec['Ref-Relations-Max']=r.max;
        }
        allRefActionCounts.push(...actionCounts);
        allRefRelationCounts.push(...relationCounts);
      }else missingReferencePaths++;
      records.push(rec);
    }
  }
  const metricNames=[
    'RW-Ans@p-actions::Answer-Entity','RW-Ans@p-neighbors::Answer-Entity',
    'RW-Ans@p-actions::Graph-Answer-Entity','RW-Ans@p-neighbors::Graph-Answer-Entity',
    'RW-RefPath@p-actions','Reachable-N-Count',
    'Released-Answer-Count','Graph-Answer-Count','Graph-Answer-Expansion',
    'Topic-Is-Released-Answer','Topic-Is-Graph-Answer',
    'Path-Count','Graph-Path-Count','Graph-Path-Expansion',
    'Reference-Path-Count','Released-Reference-Has-Cycle',
    'Released-Reference-Has-Immediate-Backtrack','Ref-Actions-Mean','Ref-Relations-Mean'
  ];
  const metric_tables=[];
  for(const metric of metricNames)if(records.some(r=>Number.isFinite(r[metric])))metric_tables.push({metric,rows:buildMetricRows(records,metric)});
  const actionSummary=summarize(allRefActionCounts),relationSummary=summarize(allRefRelationCounts);
  return {
    summary:{
      canonical_qa_rows:rows.length,qa_rows_evaluated:records.length,
      families_evaluated:new Set(records.map(r=>r['Question-Family-ID']).filter(Boolean)).size,
      skipped_missing_source:skippedMissingSource,num_rollout_steps:numSteps,random_walk_enabled:runRW,
      use_self_loops:useSelfLoops,use_undirected_actions:Boolean(options.useUndirectedActions),action_space:options.useUndirectedActions?'undirected':'directed_outgoing',paraphrases_expanded:canExpand,
      missing_reference_paths:missingReferencePaths,
      reference_paths_rows:records.filter(r=>r['Reference-Path-Count']>0).length,
      reachable_n_mean:runRW?mean(records.map(r=>r['Reachable-N-Count'])):null,
      rw_actions_all_mean:runRW?mean(records.map(r=>r['RW-Ans@p-actions::Answer-Entity'])):null,
      rw_neighbors_all_mean:runRW?mean(records.map(r=>r['RW-Ans@p-neighbors::Answer-Entity'])):null,
      graph_answer_rows:records.filter(r=>Number.isFinite(r['Graph-Answer-Count'])).length,
      rw_graph_actions_all_mean:runRW?mean(records.filter(r=>Number.isFinite(r['RW-Ans@p-actions::Graph-Answer-Entity'])).map(r=>r['RW-Ans@p-actions::Graph-Answer-Entity'])):null,
      rw_graph_neighbors_all_mean:runRW?mean(records.filter(r=>Number.isFinite(r['RW-Ans@p-neighbors::Graph-Answer-Entity'])).map(r=>r['RW-Ans@p-neighbors::Graph-Answer-Entity'])):null
    },
    metric_tables,
    reference_state_actions:allRefActionCounts.length?{
      ...actionSummary,
      fraction_gt10:mean(allRefActionCounts.map(v=>v>10?1:0)),
      fraction_gt50:mean(allRefActionCounts.map(v=>v>50?1:0)),
      fraction_gt100:mean(allRefActionCounts.map(v=>v>100?1:0))
    }:null,
    reference_state_relations:allRefRelationCounts.length?relationSummary:null
  };
}
async function start(job){
  const t0=performance.now(); let kg=null, qa=null, joint=null, graphData=null;
  if(job.graphSource){ graphData=await loadGraph(job.graphSource, job.entityFile, job.options||{}, job.entityIdColumn??null); kg=graphData.result; }
  if(job.qaSource){ qa=analyzeQA(await loadQA(job.qaSource, job.options?.qaSplit||'all')); }
  if(graphData && qa){ post('progress',{phase:'Joint analysis',detail:'Computing QA-conditioned navigation calibration',fraction:null}); joint=analyzeJoint(graphData, qa, job.options||{}); }
  if(job.graphSource?.mode==='archive')await publishArchiveLabels(job.graphSource);
  if(qa) delete qa.rows;
  return {kg, qa, joint, runtime_seconds:(performance.now()-t0)/1000};
}
function seedDownloadedCache(entries){
  if(busy)return;
  for(const entry of entries||[]){
    if(!entry || typeof entry.url!=='string' || !(entry.blob instanceof Blob))continue;
    if(entry.provider==='gcs'){
      restoredArchiveBlobs.set(entry.url,entry.blob);
      cachedBytes.set(`gcs:${entry.url}`,entry.blob.size);
    }else if(entry.provider==='hf'){
      remoteFileCache.set(entry.url,Promise.resolve(entry.blob));
      cachedBytes.set(`hf:${entry.url}`,entry.blob.size);
    }
  }
  publishCacheStatus();
}
self.onmessage=async event=>{
  if(event.data?.action==='seed-cache'){
    seedDownloadedCache(event.data.entries);
    return;
  }
  if(event.data?.action!=='analyze' || busy)return;
  busy=true;
  try{const result=await start(event.data.job);post('complete',{result});}
  catch(e){post('error',{message:String(e?.message||e)});}
  finally{busy=false;}
};