import React, { useEffect, useMemo, useState } from "react";
import { createRoot } from "react-dom/client";
import {
Activity,
Braces,
Code2,
Database,
FileCode2,
GitBranch,
Layers3,
RefreshCw,
Search,
Server,
Sparkles,
Upload,
Zap
} from "lucide-react";
import "./styles.css";
const DEFAULT_CODE = `from collections import defaultdict
def normalize_user(user):
email = user["email"].strip().lower()
return {
"id": user["id"],
"email": email,
"active": bool(user.get("active", True)),
}
def group_active_users(users):
groups = defaultdict(list)
for user in users:
normalized = normalize_user(user)
if normalized["active"]:
domain = normalized["email"].split("@")[-1]
groups[domain].append(normalized)
return dict(groups)
def find_user_by_email(users, email):
target = email.strip().lower()
for user in users:
normalized = normalize_user(user)
if normalized["email"] == target:
return normalized
return None
`;
const DEFAULT_QUERY =
"Where are active users grouped by email domain after normalizing their records?";
const SOURCE_LABELS = {
raw_code: "Raw code",
cross_file_propagated: "Propagated",
semantic_synthetic: "Semantic"
};
const SOURCE_ICONS = {
raw_code: FileCode2,
cross_file_propagated: GitBranch,
semantic_synthetic: Sparkles
};
let extractorPromise = null;
let embeddingBackend = "hashing";
function getExtractor() {
if (!extractorPromise) {
extractorPromise = import("@xenova/transformers").then(({ env, pipeline }) => {
env.allowLocalModels = false;
env.useBrowserCache = true;
return pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2");
});
}
return extractorPromise;
}
function formatNumber(value) {
if (typeof value !== "number") return value;
return value.toLocaleString(undefined, { maximumFractionDigits: 4 });
}
function sourceName(source) {
return SOURCE_LABELS[source] || source;
}
function tokenize(text) {
const words = (text.toLowerCase().match(/[a-z_][a-z0-9_]*|\d+(?:\.\d+)?/g) || []);
const bigrams = words.slice(1).map((word, index) => `${words[index]}_${word}`);
return [...words, ...bigrams];
}
function hashToken(token, dimensions) {
let hash = 2166136261;
for (let index = 0; index < token.length; index += 1) {
hash ^= token.charCodeAt(index);
hash = Math.imul(hash, 16777619);
}
return Math.abs(hash) % dimensions;
}
function embedTextHash(text, dimensions = 384) {
const vector = new Float32Array(dimensions);
for (const token of tokenize(text)) {
const bucket = hashToken(token, dimensions);
vector[bucket] += token.length % 2 === 0 ? 1 : -1;
}
let norm = 0;
for (const value of vector) norm += value * value;
norm = Math.sqrt(norm);
if (norm > 0) {
for (let index = 0; index < vector.length; index += 1) vector[index] /= norm;
}
return vector;
}
async function embedText(text) {
try {
const extractor = await getExtractor();
const output = await extractor(text || " ", {
pooling: "mean",
normalize: true
});
embeddingBackend = "Xenova/all-MiniLM-L6-v2";
return Float32Array.from(output.data);
} catch (error) {
console.warn("Falling back to hashing embeddings:", error);
embeddingBackend = "hashing-fallback";
return embedTextHash(text);
}
}
function dot(left, right) {
let score = 0;
for (let index = 0; index < left.length; index += 1) score += left[index] * right[index];
return score;
}
function extractFunctions(source, filePath) {
const lines = source.replace(/\r\n/g, "\n").split("\n");
const starts = [];
const defPattern = /^(\s*)(async\s+def|def)\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(/;
lines.forEach((line, index) => {
const match = line.match(defPattern);
if (match) {
starts.push({ line: index, indent: match[1].length, name: match[3] });
}
});
return starts.map((start, index) => {
const next = starts.find((candidate, candidateIndex) => (
candidateIndex > index && candidate.indent <= start.indent
));
const endLine = next ? next.line : lines.length;
const rawCode = lines.slice(start.line, endLine).join("\n").trim();
const calls = Array.from(rawCode.matchAll(/\b([A-Za-z_][A-Za-z0-9_]*)\s*\(/g))
.map((match) => match[1])
.filter((name) => !["if", "for", "while", "return", "def"].includes(name));
const uniqueCalls = [...new Set(calls.filter((name) => name !== start.name))];
const words = start.name.replace(/_/g, " ");
const ownSummary = `Function ${start.name} handles ${words}. Calls: ${uniqueCalls.slice(0, 8).join(", ") || "none"}.`;
return {
key: `${filePath}::${start.name}`,
file_path: filePath,
qualified_name: start.name,
symbol_type: "function",
start_line: start.line + 1,
end_line: endLine,
raw_code: rawCode,
own_sketch: `${lines[start.line]}\n${ownSummary}`,
propagated_text: `${ownSummary}\n\n${rawCode}`,
direct_callees: uniqueCalls
};
});
}
function semanticQuestions(record) {
const nameWords = record.qualified_name.replace(/_/g, " ");
const questions = [
`Where is ${nameWords} implemented?`,
`Which function handles ${nameWords}?`,
`How does ${nameWords} work?`
];
if (/email|domain/i.test(record.raw_code)) {
questions.push("Where are users grouped or matched by email domain?");
}
if (/active/i.test(record.raw_code)) {
questions.push("Which function filters or processes active records?");
}
if (/normalize|strip|lower/i.test(record.raw_code)) {
questions.push("Where is input normalized or cleaned?");
}
return questions;
}
async function buildBrowserIndex(source, filePath) {
const functions = extractFunctions(source, filePath);
const byName = Object.fromEntries(functions.map((record) => [record.qualified_name, record]));
const vectors = [];
for (const record of functions) {
const calleeContext = record.direct_callees
.map((name) => byName[name]?.own_sketch)
.filter(Boolean)
.join("\n");
record.propagated_text = [record.propagated_text, calleeContext].filter(Boolean).join("\n\n");
for (const [sourceNameValue, text] of [
["raw_code", record.raw_code],
["cross_file_propagated", record.propagated_text]
]) {
vectors.push({
record,
source: sourceNameValue,
text,
scope: null,
evidence: [],
vector: await embedText(text)
});
}
for (const question of semanticQuestions(record)) {
vectors.push({
record,
source: "semantic_synthetic",
text: question,
scope: "behavior",
evidence: [record.qualified_name],
vector: await embedText(question)
});
}
}
const dimensions = vectors[0]?.vector.length || 384;
return {
dimensions,
functions,
vectors,
stats: {
schema_version: "coderag-browser-neural-vector-db-v1",
database_path: "browser-memory",
dimensions,
embedding_backend: embeddingBackend,
function_count: functions.length,
vector_count: vectors.length,
sources: vectors.reduce((counts, row) => {
counts[row.source] = (counts[row.source] || 0) + 1;
return counts;
}, {}),
active_index_mode: "browser_static",
active_file_path: filePath
}
};
}
async function searchBrowserIndex(index, query, topK, perSource, selectedSources) {
const queryVector = await embedText(query);
const candidateRows = new Map();
for (const sourceValue of selectedSources) {
const ranked = index.vectors
.filter((row) => row.source === sourceValue)
.map((row) => ({ ...row, score: dot(row.vector, queryVector) }))
.sort((left, right) => right.score - left.score)
.slice(0, perSource);
for (const row of ranked) {
const existing = candidateRows.get(row.record.key);
if (!existing || row.score > existing.best_score) {
candidateRows.set(row.record.key, {
...row.record,
source_scores: { ...(existing?.source_scores || {}), [sourceValue]: Number(row.score.toFixed(6)) },
best_source: sourceValue,
best_score: Number(row.score.toFixed(6)),
matched_text: row.text,
matched_scope: row.scope,
matched_evidence: row.evidence
});
} else {
existing.source_scores[sourceValue] = Number(row.score.toFixed(6));
}
}
}
const hits = [...candidateRows.values()]
.sort((left, right) => right.best_score - left.best_score)
.slice(0, topK);
return {
query,
hits,
candidate_count: candidateRows.size,
per_source: perSource,
sources: selectedSources,
stats: index.stats
};
}
function App() {
const [code, setCode] = useState(DEFAULT_CODE);
const [filename, setFilename] = useState("user_index.py");
const [query, setQuery] = useState(DEFAULT_QUERY);
const [topK, setTopK] = useState(6);
const [perSource, setPerSource] = useState(5);
const [sources, setSources] = useState([
"raw_code",
"cross_file_propagated",
"semantic_synthetic"
]);
const [stats, setStats] = useState(null);
const [result, setResult] = useState(null);
const [activeHit, setActiveHit] = useState(null);
const [status, setStatus] = useState("idle");
const [error, setError] = useState("");
const [clientIndex, setClientIndex] = useState(null);
useEffect(() => {
refreshStats();
}, []);
useEffect(() => {
setActiveHit(result?.hits?.[0] || null);
}, [result]);
const sourceCounts = useMemo(() => stats?.sources || {}, [stats]);
const hasIndexedCode = stats?.active_index_mode === "pasted_code";
async function refreshStats() {
try {
const response = await fetch("/api/coderag/stats");
const payload = await response.json();
if (!response.ok) throw new Error(payload.error || "Stats failed");
setStats(payload);
} catch (err) {
setStatus("indexing");
const fallback = await buildBrowserIndex(code, filename || "user_index.py");
setClientIndex(fallback);
setStats(fallback.stats);
setError("");
setStatus("idle");
}
}
async function indexCode() {
setStatus("indexing");
setError("");
setResult(null);
try {
const response = await fetch("/api/coderag/index-code", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ code, filename })
});
const payload = await response.json();
if (!response.ok) throw new Error(payload.error || "Indexing failed");
setStats(payload);
setStatus("idle");
} catch (err) {
const fallback = await buildBrowserIndex(code, filename || "user_index.py");
setClientIndex(fallback);
setStats(fallback.stats);
setError("");
setStatus("idle");
}
}
async function rebuildSampleDb() {
setStatus("rebuilding");
setError("");
try {
const response = await fetch("/api/coderag/rebuild", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({})
});
const payload = await response.json();
if (!response.ok) throw new Error(payload.error || "Rebuild failed");
setStats(payload);
setResult(null);
setStatus("idle");
} catch (err) {
const fallback = await buildBrowserIndex(DEFAULT_CODE, "sample_users.py");
setCode(DEFAULT_CODE);
setFilename("sample_users.py");
setClientIndex(fallback);
setStats(fallback.stats);
setResult(null);
setError("");
setStatus("idle");
}
}
async function search() {
setStatus("searching");
setError("");
try {
const response = await fetch("/api/coderag/search", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
query,
top_k: topK,
per_source: perSource,
sources
})
});
const payload = await response.json();
if (!response.ok) throw new Error(payload.error || "Search failed");
setResult(payload);
setStats(payload.stats);
setStatus("idle");
} catch (err) {
const fallback = clientIndex || await buildBrowserIndex(code, filename || "user_index.py");
setClientIndex(fallback);
const payload = await searchBrowserIndex(fallback, query, topK, perSource, sources);
setResult(payload);
setStats(payload.stats);
setError("");
setStatus("idle");
}
}
function toggleSource(source) {
setSources((current) => {
if (current.includes(source) && current.length > 1) {
return current.filter((item) => item !== source);
}
if (!current.includes(source)) return [...current, source];
return current;
});
}
return (
CodeRAG Retrieval Console
{hit.file_path} {hit.start_line ? `:${hit.start_line}-${hit.end_line}` : ""}
{hit.matched_text || "No text stored for this vector."}
{hit.matched_scope && {hit.matched_scope}} {hit.matched_evidence?.length > 0 && ({item}
))}
{value}