Spaces:
Running
Running
| /** Measure search quality of the shipped binaries. Usage: node scripts/eval.ts <data-dir> <eval.json> */ | |
| import { readFileSync } from 'node:fs'; | |
| import { loadDataset } from './load-node.ts'; | |
| const [dataDir, evalPath] = process.argv.slice(2); | |
| const { engine } = loadDataset(dataDir); | |
| const spec = JSON.parse(readFileSync(evalPath, 'utf8')) as { | |
| queries: Record<string, number[]>; | |
| attr: Record<string, [string[], string[]]>; | |
| }; | |
| const K = [10, 30, 100]; | |
| const sum = K.map(() => 0); | |
| for (const [query, relevantIds] of Object.entries(spec.queries)) { | |
| const relevant = new Set(relevantIds.map((id) => engine.subjects[id])); | |
| const ranked = engine.rankSubjects(query, 100, false); | |
| K.forEach((k, i) => (sum[i] += ranked.slice(0, k).filter((s) => relevant.has(s)).length / k)); | |
| } | |
| const queryCount = Object.keys(spec.queries).length; | |
| console.log('subject P@10/30/100', K.map((_, i) => (sum[i] / queryCount).toFixed(3)).join(' '), `over ${queryCount} queries`); | |
| function evaluateSprites(weight: number): void { | |
| engine.attributeWeight = weight; | |
| let attributeP20 = 0; | |
| let attributeP100 = 0; | |
| let conceptP100 = 0; | |
| for (const [query, [attributes, objects]] of Object.entries(spec.attr)) { | |
| const result = engine.search(query, 400, false); | |
| const matches = Array.from(result.sprites).map((sprite) => { | |
| const tokens = engine.captionTokensOf(sprite); | |
| return tokens.some((t) => attributes.includes(t)) && objects.some((o) => tokens.some((t) => t.includes(o))); | |
| }); | |
| const at = (k: number) => matches.slice(0, k).filter(Boolean).length / Math.max(1, Math.min(k, matches.length)); | |
| attributeP20 += at(20); | |
| attributeP100 += at(100); | |
| if (weight === 0.6) console.log(`${query.padEnd(16)} sprites ${String(result.sprites.length).padStart(5)} P@20 ${at(20).toFixed(2)} P@100 ${at(100).toFixed(2)} | ${engine.caption(result.sprites[0] ?? 0)}`); | |
| } | |
| for (const [query, relevantIds] of Object.entries(spec.queries)) { | |
| const relevant = new Set(relevantIds.map((id) => engine.subjects[id])); | |
| const result = engine.search(query, 400, false); | |
| const top = Array.from(result.sprites.slice(0, 100)).map((sprite) => relevant.has(engine.subjectOf(sprite))); | |
| conceptP100 += top.filter(Boolean).length / Math.max(1, top.length); | |
| } | |
| const attrCount = Object.keys(spec.attr).length; | |
| console.log(`attributeWeight ${weight}: attr P@20 ${(attributeP20 / attrCount).toFixed(3)} attr P@100 ${(attributeP100 / attrCount).toFixed(3)} concept sprite P@100 ${(conceptP100 / queryCount).toFixed(3)}`); | |
| } | |
| for (const weight of [0.25, 0.4, 0.6, 0.9]) evaluateSprites(weight); | |
| engine.attributeWeight = 0.6; | |
| const show = ['red', 'gold', 'glowing', 'sea creature', 'spooky', 'dragon', 'dra', 'spook', 'sea c', 'cat', 'octop', 'dragn', 'skel', 'feline', 'something to eat', 'rocket ship', 'cute', 'tasty', 'hot', 'angry', 'qwzx']; | |
| for (const query of show) { | |
| const result = engine.search(query, 2000, true); | |
| const names = Array.from(new Set(Array.from(result.sprites).map((sprite) => engine.subjectOf(sprite)))).slice(0, 8); | |
| console.log(`${query.padEnd(18)} n=${String(result.sprites.length).padStart(4)} ${result.computeMs.toFixed(2).padStart(6)}ms ${names.join(', ')}`); | |
| } | |
| const perQuery = Object.entries(spec.queries) | |
| .map(([query, ids]) => { | |
| const relevant = new Set(ids.map((id) => engine.subjects[id])); | |
| const top = engine.rankSubjects(query, 10, false); | |
| return { query, score: top.filter((s) => relevant.has(s)).length / 10, top: top.slice(0, 4).join(', ') }; | |
| }) | |
| .sort((a, b) => b.score - a.score); | |
| console.log('best:', perQuery.slice(0, 12).map((r) => `${r.query} ${r.score}`).join(' | ')); | |
| for (const row of perQuery.slice(-8)) console.log('worst', row.query, row.score, row.top); | |
| for (const q of ['something to eat', 'a very fast car', 'things found in a kitchen', 'not a dragon', 'scary clown', 'big red truck', 'wizard hat', 'cold drink', 'pet food', 'angry cat', 'blue dragon that breathes ice', 'qwertyuiop', 'dinosore', 'monster truck', 'pirate treasure']) { | |
| const result = engine.search(q, 500, false); | |
| const names = Array.from(new Set(Array.from(result.sprites).map((sprite) => engine.subjectOf(sprite)))).slice(0, 6); | |
| console.log(q.padEnd(30), result.sprites.length, names.join(', ')); | |
| } | |