File size: 1,898 Bytes
161cb23
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
import { parser as goParser } from "@lezer/go";
import { highlightTree, tagHighlighter, tags } from "@lezer/highlight";
import { parser as jsParser } from "@lezer/javascript";
import { parser as pythonParser } from "@lezer/python";
import { parser as rustParser } from "@lezer/rust";
import { extensionOf } from "./languages.js";

const PARSERS = {
  py: () => pythonParser,
  pyi: () => pythonParser,
  js: () => jsParser,
  mjs: () => jsParser,
  cjs: () => jsParser,
  jsx: () => jsParser.configure({ dialect: "jsx" }),
  ts: () => jsParser.configure({ dialect: "ts" }),
  tsx: () => jsParser.configure({ dialect: "ts jsx" }),
  rs: () => rustParser,
  go: () => goParser,
};

const tokenClasses = tagHighlighter([
  { tag: tags.comment, class: "tok-com" },
  { tag: [tags.string, tags.regexp, tags.character], class: "tok-str" },
  { tag: [tags.keyword, tags.bool, tags.null, tags.atom, tags.self], class: "tok-kw" },
  { tag: tags.number, class: "tok-num" },
]);

export function tokenizeLines(text, filename) {
  const lines = text.split("\n");
  const spans = lines.map(() => []);
  const factory = PARSERS[extensionOf(filename)];
  if (!factory) {
    return spans;
  }
  const lineStarts = [];
  let offset = 0;
  for (const line of lines) {
    lineStarts.push(offset);
    offset += line.length + 1;
  }
  let lineIndex = 0;
  highlightTree(factory().parse(text), tokenClasses, (from, to, className) => {
    while (lineIndex + 1 < lineStarts.length && lineStarts[lineIndex + 1] <= from) {
      lineIndex += 1;
    }
    for (let index = lineIndex; index < lines.length && lineStarts[index] < to; index += 1) {
      const start = Math.max(from, lineStarts[index]) - lineStarts[index];
      const end = Math.min(to - lineStarts[index], lines[index].length);
      if (end > start) {
        spans[index].push({ from: start, to: end, className });
      }
    }
  });
  return spans;
}