Spaces:
Sleeping
Sleeping
File size: 1,898 Bytes
161cb23 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | import { parser as goParser } from "@lezer/go";
import { highlightTree, tagHighlighter, tags } from "@lezer/highlight";
import { parser as jsParser } from "@lezer/javascript";
import { parser as pythonParser } from "@lezer/python";
import { parser as rustParser } from "@lezer/rust";
import { extensionOf } from "./languages.js";
const PARSERS = {
py: () => pythonParser,
pyi: () => pythonParser,
js: () => jsParser,
mjs: () => jsParser,
cjs: () => jsParser,
jsx: () => jsParser.configure({ dialect: "jsx" }),
ts: () => jsParser.configure({ dialect: "ts" }),
tsx: () => jsParser.configure({ dialect: "ts jsx" }),
rs: () => rustParser,
go: () => goParser,
};
const tokenClasses = tagHighlighter([
{ tag: tags.comment, class: "tok-com" },
{ tag: [tags.string, tags.regexp, tags.character], class: "tok-str" },
{ tag: [tags.keyword, tags.bool, tags.null, tags.atom, tags.self], class: "tok-kw" },
{ tag: tags.number, class: "tok-num" },
]);
export function tokenizeLines(text, filename) {
const lines = text.split("\n");
const spans = lines.map(() => []);
const factory = PARSERS[extensionOf(filename)];
if (!factory) {
return spans;
}
const lineStarts = [];
let offset = 0;
for (const line of lines) {
lineStarts.push(offset);
offset += line.length + 1;
}
let lineIndex = 0;
highlightTree(factory().parse(text), tokenClasses, (from, to, className) => {
while (lineIndex + 1 < lineStarts.length && lineStarts[lineIndex + 1] <= from) {
lineIndex += 1;
}
for (let index = lineIndex; index < lines.length && lineStarts[index] < to; index += 1) {
const start = Math.max(from, lineStarts[index]) - lineStarts[index];
const end = Math.min(to - lineStarts[index], lines[index].length);
if (end > start) {
spans[index].push({ from: start, to: end, className });
}
}
});
return spans;
}
|