import { parser as goParser } from "@lezer/go"; import { highlightTree, tagHighlighter, tags } from "@lezer/highlight"; import { parser as jsParser } from "@lezer/javascript"; import { parser as pythonParser } from "@lezer/python"; import { parser as rustParser } from "@lezer/rust"; import { extensionOf } from "./languages.js"; const PARSERS = { py: () => pythonParser, pyi: () => pythonParser, js: () => jsParser, mjs: () => jsParser, cjs: () => jsParser, jsx: () => jsParser.configure({ dialect: "jsx" }), ts: () => jsParser.configure({ dialect: "ts" }), tsx: () => jsParser.configure({ dialect: "ts jsx" }), rs: () => rustParser, go: () => goParser, }; const tokenClasses = tagHighlighter([ { tag: tags.comment, class: "tok-com" }, { tag: [tags.string, tags.regexp, tags.character], class: "tok-str" }, { tag: [tags.keyword, tags.bool, tags.null, tags.atom, tags.self], class: "tok-kw" }, { tag: tags.number, class: "tok-num" }, ]); export function tokenizeLines(text, filename) { const lines = text.split("\n"); const spans = lines.map(() => []); const factory = PARSERS[extensionOf(filename)]; if (!factory) { return spans; } const lineStarts = []; let offset = 0; for (const line of lines) { lineStarts.push(offset); offset += line.length + 1; } let lineIndex = 0; highlightTree(factory().parse(text), tokenClasses, (from, to, className) => { while (lineIndex + 1 < lineStarts.length && lineStarts[lineIndex + 1] <= from) { lineIndex += 1; } for (let index = lineIndex; index < lines.length && lineStarts[index] < to; index += 1) { const start = Math.max(from, lineStarts[index]) - lineStarts[index]; const end = Math.min(to - lineStarts[index], lines[index].length); if (end > start) { spans[index].push({ from: start, to: end, className }); } } }); return spans; }