/** Measure search quality of the shipped binaries. Usage: node scripts/eval.ts <data-dir> <eval.json> */
import { readFileSync } from 'node:fs';
import { loadDataset } from './load-node.ts';

const [dataDir, evalPath] = process.argv.slice(2);
const { engine } = loadDataset(dataDir);
const spec = JSON.parse(readFileSync(evalPath, 'utf8')) as {
	queries: Record<string, number[]>;
	attr: Record<string, [string[], string[]]>;
};

const K = [10, 30, 100];
const sum = K.map(() => 0);
for (const [query, relevantIds] of Object.entries(spec.queries)) {
	const relevant = new Set(relevantIds.map((id) => engine.subjects[id]));
	const ranked = engine.rankSubjects(query, 100, false);
	K.forEach((k, i) => (sum[i] += ranked.slice(0, k).filter((s) => relevant.has(s)).length / k));
}
const queryCount = Object.keys(spec.queries).length;
console.log('subject P@10/30/100', K.map((_, i) => (sum[i] / queryCount).toFixed(3)).join(' '), `over ${queryCount} queries`);


function evaluateSprites(weight: number): void {
	engine.attributeWeight = weight;
	let attributeP20 = 0;
	let attributeP100 = 0;
	let conceptP100 = 0;
	for (const [query, [attributes, objects]] of Object.entries(spec.attr)) {
		const result = engine.search(query, 400, false);
		const matches = Array.from(result.sprites).map((sprite) => {
			const tokens = engine.captionTokensOf(sprite);
			return tokens.some((t) => attributes.includes(t)) && objects.some((o) => tokens.some((t) => t.includes(o)));
		});
		const at = (k: number) => matches.slice(0, k).filter(Boolean).length / Math.max(1, Math.min(k, matches.length));
		attributeP20 += at(20);
		attributeP100 += at(100);
		if (weight === 0.6) console.log(`${query.padEnd(16)} sprites ${String(result.sprites.length).padStart(5)} P@20 ${at(20).toFixed(2)} P@100 ${at(100).toFixed(2)} | ${engine.caption(result.sprites[0] ?? 0)}`);
	}
	for (const [query, relevantIds] of Object.entries(spec.queries)) {
		const relevant = new Set(relevantIds.map((id) => engine.subjects[id]));
		const result = engine.search(query, 400, false);
		const top = Array.from(result.sprites.slice(0, 100)).map((sprite) => relevant.has(engine.subjectOf(sprite)));
		conceptP100 += top.filter(Boolean).length / Math.max(1, top.length);
	}
	const attrCount = Object.keys(spec.attr).length;
	console.log(`attributeWeight ${weight}: attr P@20 ${(attributeP20 / attrCount).toFixed(3)} attr P@100 ${(attributeP100 / attrCount).toFixed(3)} concept sprite P@100 ${(conceptP100 / queryCount).toFixed(3)}`);
}
for (const weight of [0.25, 0.4, 0.6, 0.9]) evaluateSprites(weight);

engine.attributeWeight = 0.6;
const show = ['red', 'gold', 'glowing', 'sea creature', 'spooky', 'dragon', 'dra', 'spook', 'sea c', 'cat', 'octop', 'dragn', 'skel', 'feline', 'something to eat', 'rocket ship', 'cute', 'tasty', 'hot', 'angry', 'qwzx'];
for (const query of show) {
	const result = engine.search(query, 2000, true);
	const names = Array.from(new Set(Array.from(result.sprites).map((sprite) => engine.subjectOf(sprite)))).slice(0, 8);
	console.log(`${query.padEnd(18)} n=${String(result.sprites.length).padStart(4)} ${result.computeMs.toFixed(2).padStart(6)}ms  ${names.join(', ')}`);
}

const perQuery = Object.entries(spec.queries)
	.map(([query, ids]) => {
		const relevant = new Set(ids.map((id) => engine.subjects[id]));
		const top = engine.rankSubjects(query, 10, false);
		return { query, score: top.filter((s) => relevant.has(s)).length / 10, top: top.slice(0, 4).join(', ') };
	})
	.sort((a, b) => b.score - a.score);
console.log('best:', perQuery.slice(0, 12).map((r) => `${r.query} ${r.score}`).join(' | '));
for (const row of perQuery.slice(-8)) console.log('worst', row.query, row.score, row.top);
for (const q of ['something to eat', 'a very fast car', 'things found in a kitchen', 'not a dragon', 'scary clown', 'big red truck', 'wizard hat', 'cold drink', 'pet food', 'angry cat', 'blue dragon that breathes ice', 'qwertyuiop', 'dinosore', 'monster truck', 'pirate treasure']) {
	const result = engine.search(q, 500, false);
	const names = Array.from(new Set(Array.from(result.sprites).map((sprite) => engine.subjectOf(sprite)))).slice(0, 6);
	console.log(q.padEnd(30), result.sprites.length, names.join(', '));
}