tianxuan / src /ml /debug-quality.ts
masterofthestars's picture
Tianxuan: full project + data (checkpoints, world_model, data without raw) with correct directory structure
db7bc06
Raw History Blame Contribute Delete
2.74 kB
import * as fs from "fs";
import * as path from "path";
import { Engine } from "./engine";
async function main(): Promise<void> {
const engine = Engine.load(path.resolve("data/checkpoints"));
console.log("=== MODEL QUALITY CHECK ===");
console.log("Vocab size:", engine.tokenizer.vocabSize);
console.log("Context window:", engine.model.cfg.ctx);
console.log("Embedding dim:", engine.model.cfg.emb);
console.log("Hidden dim:", engine.model.cfg.hidden);
// Test with a simple prompt
const prompt = "小明";
const promptIds = engine.tokenizer.encode(prompt);
console.log("\nPrompt:", prompt);
console.log("Prompt IDs:", promptIds);
// Reset state and advance with prompt
(engine as any).state = (engine as any).model.newState();
for (const id of promptIds) (engine as any).advance(id);
// Peek logits
const logits = (engine as any).peekLogits(4);
const probs = engine.model.logitsToProbs(logits);
// Top 10 predictions
const sorted = Array.from(probs)
.map((p, i) => ({ id: i, prob: p, char: engine.tokenizer.charOf(i) }))
.sort((a, b) => b.prob - a.prob)
.slice(0, 10);
console.log("\nTop 10 predictions after '小明':");
sorted.forEach((s, i) => {
console.log(` ${i + 1}. "${s.char}" (id=${s.id}): ${(s.prob * 100).toFixed(1)}%`);
});
// Generate with very low temperature for deterministic output
console.log("\n=== GENERATION (temp=0.1, topK=3) ===");
const res1 = await engine.generate({ prompt, maxLen: 20, budget: 3, temperature: 0.1, topK: 3 });
console.log("Output:", res1.text);
console.log("Routes:", res1.routes.map(r => `${r.char}(L${r.depth},conf=${r.confidence.toFixed(2)})`).join(" "));
// Generate with temperature=0
console.log("\n=== GENERATION (temp=0.01, topK=1) ===");
const res2 = await engine.generate({ prompt, maxLen: 20, budget: 3, temperature: 0.01, topK: 1 });
console.log("Output:", res2.text);
console.log("Routes:", res2.routes.map(r => `${r.char}(L${r.depth},conf=${r.confidence.toFixed(2)})`).join(" "));
// Generate with temperature=0.5
console.log("\n=== GENERATION (temp=0.5, topK=10) ===");
const res3 = await engine.generate({ prompt, maxLen: 30, budget: 3, temperature: 0.5, topK: 10 });
console.log("Output:", res3.text);
console.log("Routes:", res3.routes.map(r => `${r.char}(L${r.depth},conf=${r.confidence.toFixed(2)})`).join(" "));
// Test with different prompts
const prompts = ["如果", "美好", "苹果"];
for (const p of prompts) {
console.log(`\n=== Prompt: "${p}" (temp=0.1, topK=3) ===`);
const r = await engine.generate({ prompt: p, maxLen: 15, budget: 3, temperature: 0.1, topK: 3 });
console.log("Output:", r.text);
}
}
main().catch(err => console.error(err));