Spaces:
Running
Running
File size: 3,895 Bytes
1b38872 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | // Core extraction logic, shared by index.html and the Node parity test (tests/js_parity.mjs).
// Same word/sentence splitting as skillex/predict.py (matches SkillSpan's training tokens).
const WORD = /[\p{L}\p{N}_\p{M}]+(?:['’.\-][\p{L}\p{N}_\p{M}]+)*|\S/gu;
export const splitWords = (text) => [...text.matchAll(WORD)].map((m) => [m.index, m.index + m[0].length]);
export function splitSentences(text, words) {
const sents = []; let cur = [];
words.forEach(([a, b], i) => {
if (cur.length && text.slice(words[cur.at(-1)][1], a).includes("\n")) { sents.push(cur); cur = []; }
cur.push(i);
if ([".", "!", "?"].includes(text.slice(a, b))) { sents.push(cur); cur = []; }
});
if (cur.length) sents.push(cur);
return sents;
}
// Viterbi with the learned CRF transitions (identical to skillex.decode.viterbi_bio).
export function viterbi(emit, trans, start) {
const T = emit.length; if (!T) return [];
let best = [0, 1, 2].map((j) => start[j] + emit[0][j]);
const back = [];
for (let t = 1; t < T; t++) {
const bt = [], nb = [];
for (let j = 0; j < 3; j++) {
let arg = 0, val = -Infinity;
for (let i = 0; i < 3; i++) { const v = best[i] + trans[i][j]; if (v > val) { val = v; arg = i; } }
bt.push(arg); nb.push(val + emit[t][j]);
}
back.push(bt); best = nb;
}
let k = best.indexOf(Math.max(...best)); const path = [k];
for (let t = T - 1; t > 0; t--) { k = back[t - 1][k]; path.push(k); }
return path.reverse();
}
const MAX_SUBWORDS = 510, BATCH = 16, TYPES = ["SKILL", "KNOWLEDGE"];
// Split a sentence (list of global word ids) into chunks that fit the model.
function encodeSentence(tok, CLS, SEP, text, words, sent) {
const chunks = []; let ids = [CLS], first = [], wids = [];
for (const w of sent) {
// Leading space = SentencePiece word-start marker "▁". Python's is_split_into_words adds it;
// transformers.js does not (without it, IDs differ for every word: 0/897 matched).
const sub = tok.encode(" " + text.slice(...words[w]), { add_special_tokens: false });
if (ids.length + sub.length > MAX_SUBWORDS && wids.length) {
chunks.push({ ids: [...ids, SEP], first, wids }); ids = [CLS]; first = []; wids = [];
}
if (!sub.length) continue;
first.push(ids.length); wids.push(w); ids.push(...sub.slice(0, MAX_SUBWORDS - 1));
}
if (wids.length) chunks.push({ ids: [...ids, SEP], first, wids });
return chunks;
}
export async function extract({ ort, tok, session, crfs }, text) {
const [CLS, SEP] = tok.encode("", { add_special_tokens: true });
const words = splitWords(text);
const chunks = splitSentences(text, words).flatMap((s) => encodeSentence(tok, CLS, SEP, text, words, s));
const spans = [];
for (let b = 0; b < chunks.length; b += BATCH) {
const batch = chunks.slice(b, b + BATCH), L = Math.max(...batch.map((c) => c.ids.length));
const ids = new BigInt64Array(batch.length * L), am = new BigInt64Array(batch.length * L);
batch.forEach((c, r) => c.ids.forEach((v, j) => { ids[r * L + j] = BigInt(v); am[r * L + j] = 1n; }));
const out = await session.run({
input_ids: new ort.Tensor("int64", ids, [batch.length, L]),
attention_mask: new ort.Tensor("int64", am, [batch.length, L]),
});
const lg = out.logits.data;
batch.forEach((c, r) => {
TYPES.forEach((type, h) => {
const emit = c.first.map((p) => [0, 1, 2].map((k) => lg[(r * L + p) * 6 + h * 3 + k]));
const path = viterbi(emit, crfs[h].transitions, crfs[h].start);
let s = -1;
for (let i = 0; i <= path.length; i++) {
const tag = i < path.length ? path[i] : 0; // 0=O 1=B 2=I
if (s >= 0 && tag !== 2) { spans.push({ type, start: words[c.wids[s]][0], end: words[c.wids[i - 1]][1] }); s = -1; }
if (tag === 1 || (tag === 2 && s < 0)) s = i;
}
});
});
}
return spans;
}
|