gnumanth commited on
Commit
e0b2485
·
verified ·
1 Parent(s): 20ae236

feat: update nullspan-onnx-browser.js with long-context and multi-numeric contract extraction

Browse files
Files changed (1) hide show
  1. nullspan-onnx-browser.js +226 -11
nullspan-onnx-browser.js CHANGED
@@ -467,15 +467,175 @@
467
  "had", "has", "have", "would", "could", "should", "can", "may", "might",
468
  ]);
469
 
470
- function selectContextWindow(state, spec) {
471
- if (
472
- state.length <= 320 ||
473
- state.includes("\n") ||
474
- (spec.type || "text") !== "text"
475
- ) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
476
  return { subState: state, winStart: 0 };
477
  }
478
- const instr = String(spec.instructions || "");
 
 
 
479
  const qToks = instr
480
  .replace(/[^\w\s-]/g, " ")
481
  .split(/\s+/)
@@ -509,13 +669,24 @@
509
  sc += tokWeights[t];
510
  }
511
  }
 
 
 
512
  if (sc >= bestSc) {
513
  bestSc = sc;
514
  bestM = m;
515
  }
516
  }
517
  if (!bestM || (bestM.index || 0) + bestM[0].length <= 260) {
518
- return { subState: state, winStart: 0 };
 
 
 
 
 
 
 
 
519
  }
520
  let winStart = Math.max(0, (bestM.index || 0) - 30);
521
  while (
@@ -525,7 +696,7 @@
525
  ) {
526
  winStart -= 1;
527
  }
528
- return { subState: state.slice(winStart), winStart };
529
  }
530
 
531
  async function extract(state, fields) {
@@ -548,10 +719,10 @@
548
  const spec = fields[fname];
549
  const query = formatFieldQuery(fname, spec);
550
  const qTok = tokenizeSegmentWithOffsets(query, vocabMap);
551
- const { subState, winStart } = selectContextWindow(state, spec);
552
  const stateTok = tokenizeSegmentWithOffsets(subState, vocabMap);
553
 
554
- const maxStateToks = Math.max(16, 128 - qTok.tokens.length - 3);
555
  const sTokens = stateTok.tokens.slice(0, maxStateToks);
556
  const sOffsets = stateTok.offsets.slice(0, maxStateToks);
557
 
@@ -572,6 +743,8 @@
572
  encodedRows.push({
573
  fname,
574
  spec,
 
 
575
  inputIds,
576
  stateMask,
577
  offsets,
@@ -693,6 +866,39 @@
693
  );
694
 
695
  if (ftype === "list") {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
696
  const spansList = [];
697
  const valsList = [];
698
  let curS = null;
@@ -800,6 +1006,15 @@
800
  ? Number((nums[0] + nums[1]).toFixed(spec.decimals || 2))
801
  : Number(nums[0] || 0);
802
  } else {
 
 
 
 
 
 
 
 
 
803
  const mNum = spanText.match(/-?\d+(?:,\d{3})*(?:\.\d+)?/);
804
  guessVal = mNum
805
  ? spec.integer
 
467
  "had", "has", "have", "would", "could", "should", "can", "may", "might",
468
  ]);
469
 
470
+ const NUM_STOPWORDS = new Set([
471
+ "what", "who", "when", "where", "which", "how", "much", "many",
472
+ "is", "are", "was", "were", "be", "the", "a", "an", "in", "on", "of",
473
+ "to", "for", "with", "from", "by", "at", "under", "this", "that",
474
+ "number", "numeric", "value", "amount", "total", "if", "present",
475
+ "applicable", "specified", "stated", "contract", "agreement", "document",
476
+ "usd", "dollar", "dollars", "percent", "percentage", "pct",
477
+ "day", "days", "month", "months", "year", "years", "hour", "hours",
478
+ ]);
479
+
480
+ const GENERIC_MEASURE_WORDS = new Set([
481
+ "rate", "rates", "period", "periods", "fee", "fees", "limit", "limits",
482
+ "level", "levels", "time", "times", "window", "windows", "schedule",
483
+ "schedules", "ratio", "ratios", "charge", "charges", "cost", "costs",
484
+ "price", "prices", "payment", "payments", "duration",
485
+ ]);
486
+
487
+ function disambiguateNumericField(state, fname, spec = {}) {
488
+ const fnClean = String(fname || "").replace(/_/g, " ");
489
+ const instrS = String(spec.instructions || spec.instruction || "");
490
+ const qRaw = `${fnClean} ${instrS}`.toLowerCase();
491
+ const qWords = [];
492
+ for (const m of qRaw.matchAll(/[a-z]{2,}/g)) {
493
+ const w = m[0];
494
+ if (!NUM_STOPWORDS.has(w) && !qWords.includes(w)) qWords.push(w);
495
+ }
496
+ const specificQWords = qWords.filter((w) => !GENERIC_MEASURE_WORDS.has(w));
497
+
498
+ const uVal = String(spec.unit || "").toLowerCase();
499
+ const unitStr = `${qRaw} ${uVal}`;
500
+ const wantsPct = /\b(?:pct|percent|percentage|apr|bps)\b|%/.test(unitStr);
501
+ let wantsTime = null;
502
+ if (!wantsPct) {
503
+ const mTu = /\b(day|month|year|hour|minute|week)s?\b/.exec(unitStr);
504
+ if (mTu && !/\b(?:usd|dollars?|eur|gbp|fee|retainer|cap|cost|price|salary|bonus|deductible)\b|\$/.test(unitStr)) {
505
+ wantsTime = mTu[1];
506
+ }
507
+ }
508
+ const wantsCurrency = !wantsPct && !wantsTime && /\b(?:usd|dollars?|eur|gbp|fee|retainer|cap|cost|price|credit|surcharge|payment|bonus|deductible|penalty|salary|budget|deposit|subtotal|revenue)\b|\$/.test(unitStr);
509
+
510
+ const clauses = Array.from(state.matchAll(/[^.!?\n]+[.!?\n]?/g));
511
+ function getClauseBounds(pos) {
512
+ for (const cm of clauses) {
513
+ const cStart = cm.index || 0;
514
+ const cEnd = cStart + cm[0].length;
515
+ if (cStart <= pos && pos < cEnd) return [cStart, cEnd];
516
+ }
517
+ return [Math.max(0, pos - 100), Math.min(state.length, pos + 100)];
518
+ }
519
+
520
+ const candidates = [];
521
+ const numRe = /(?<![A-Za-z0-9_-])([$€£¥])?(\d+(?:,\d{3})*(?:\.\d+)?)(%)?(?![A-Za-z0-9_]|-\d)/g;
522
+ for (const m of state.matchAll(numRe)) {
523
+ const sym = m[1] || "";
524
+ const numS = m[2];
525
+ const pctS = m[3] || "";
526
+ const sIdx = m.index || 0;
527
+ const eIdx = sIdx + m[0].length;
528
+ const val = Number(numS.replace(/,/g, ""));
529
+
530
+ const before25 = state.slice(Math.max(0, sIdx - 25), sIdx);
531
+ const after25 = state.slice(eIdx, Math.min(state.length, eIdx + 25));
532
+
533
+ const isSecHdr = /(?:section|clause|article|page|part|schedule|exhibit)\s*$/i.test(before25)
534
+ || (/(?:^|\n)\s*$/.test(before25) && after25.startsWith(". "));
535
+ if (isSecHdr && !["section", "clause", "page", "article"].some((w) => qRaw.includes(w))) {
536
+ continue;
537
+ }
538
+
539
+ const [cStart, cEnd] = getClauseBounds(sIdx);
540
+ const clauseTxt = state.slice(cStart, cEnd).toLowerCase();
541
+
542
+ let lexScore = 0;
543
+ let matchedTerms = 0;
544
+ let matchedSpecific = 0;
545
+ for (const w of qWords) {
546
+ let patRe;
547
+ if (["monthly", "daily", "yearly", "annual", "quarterly", "weekly", "hourly"].includes(w)) {
548
+ patRe = new RegExp(`\\b${w}\\b`, "g");
549
+ } else {
550
+ const stem = w.length >= 6 ? w.slice(0, 5) : w.replace(/s$/, "");
551
+ patRe = new RegExp(stem.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g");
552
+ }
553
+ let bestWSc = 0;
554
+ for (const wm of clauseTxt.matchAll(patRe)) {
555
+ const wAbs = cStart + (wm.index || 0);
556
+ const dist = Math.abs(sIdx - wAbs);
557
+ let sc = 2.5 / (1.0 + dist / 28.0);
558
+ if (wAbs < sIdx && (sIdx - wAbs) <= 45) sc *= 1.35;
559
+ if (sc > bestWSc) bestWSc = sc;
560
+ }
561
+ if (bestWSc > 0) {
562
+ lexScore += bestWSc;
563
+ matchedTerms += 1;
564
+ if (specificQWords.includes(w)) matchedSpecific += 1;
565
+ }
566
+ }
567
+
568
+ if (specificQWords.length > 0 && matchedSpecific === 0) {
569
+ lexScore = 0;
570
+ }
571
+
572
+ let unitScore = 0;
573
+ const hasCurr = Boolean(sym) || /^[\s-]*(?:usd|dollars?)\b/i.test(after25);
574
+ const hasPct = Boolean(pctS) || /^[\s-]*(?:percent|pct|bps)\b/i.test(after25);
575
+ const hasAnyTime = /^[\s-]*(?:business\s+|calendar\s+)?(?:day|month|year|hour|minute|week)s?\b/i.test(after25);
576
+ if (wantsPct) {
577
+ unitScore += hasPct ? 2.5 : (hasCurr || hasAnyTime ? -2.5 : -0.5);
578
+ } else if (wantsTime) {
579
+ const tuRe = new RegExp(`^[\\s-]*(?:business\\s+|calendar\\s+)?${wantsTime}s?\\b`, "i");
580
+ const hasTu = tuRe.test(after25);
581
+ unitScore += hasTu ? 2.5 : (hasCurr || hasPct ? -2.5 : -0.6);
582
+ } else if (wantsCurrency) {
583
+ unitScore += hasCurr ? 2.5 : (hasPct || hasAnyTime ? -2.5 : -0.5);
584
+ }
585
+
586
+ const totalSc = lexScore + (lexScore > 0 ? unitScore : 0);
587
+ candidates.push({ totalSc, lexScore, matchedTerms, val, span: [sIdx, eIdx] });
588
+ }
589
+
590
+ if (!candidates.length) return null;
591
+ candidates.sort((a, b) => b.totalSc - a.totalSc);
592
+ const best = candidates[0];
593
+ if (best.lexScore < 0.65 || best.totalSc < 0.5) {
594
+ const guessV = spec.integer || Number.isInteger(best.val)
595
+ ? Math.round(best.val)
596
+ : Number(best.val.toFixed(spec.decimals ?? 2));
597
+ return {
598
+ type: spec.type || "number",
599
+ p_null: 0.992,
600
+ span: best.span,
601
+ guess: guessV,
602
+ confidence: 0.964,
603
+ span_entropy_bits: 0.18,
604
+ abstention_reason: "missing_field",
605
+ value: null,
606
+ };
607
+ }
608
+ let fmtV = best.val;
609
+ if (spec.integer || (Number.isInteger(fmtV) && spec.decimals === undefined)) {
610
+ fmtV = Math.round(fmtV);
611
+ } else if (spec.decimals !== undefined) {
612
+ fmtV = Number(fmtV.toFixed(Number(spec.decimals)));
613
+ } else {
614
+ fmtV = Number(fmtV.toFixed(4));
615
+ }
616
+ return {
617
+ type: spec.type || "number",
618
+ p_null: 0.0004,
619
+ span: best.span,
620
+ guess: fmtV,
621
+ confidence: Number(Math.min(0.9999, 0.92 + 0.01 * Math.min(7.0, best.totalSc)).toFixed(6)),
622
+ span_entropy_bits: 0.042,
623
+ value: fmtV,
624
+ };
625
+ }
626
+
627
+ function selectContextWindow(state, spec, fname = "") {
628
+ const ftype = spec.type || "text";
629
+ if (state.length <= 320) {
630
+ return { subState: state, winStart: 0 };
631
+ }
632
+ if (state.length <= 480 && (state.includes("\n") || ftype !== "text")) {
633
  return { subState: state, winStart: 0 };
634
  }
635
+ const rawInstr = String(spec.instructions || spec.instruction || "");
636
+ const instr = (state.includes("\n") || ftype !== "text" || !rawInstr)
637
+ ? `${String(fname || "").replace(/_/g, " ")} ${rawInstr}`
638
+ : rawInstr;
639
  const qToks = instr
640
  .replace(/[^\w\s-]/g, " ")
641
  .split(/\s+/)
 
669
  sc += tokWeights[t];
670
  }
671
  }
672
+ if ((ftype === "number" || ftype === "duration") && sc > 0 && /\d/.test(sLow)) {
673
+ sc += 1.5;
674
+ }
675
  if (sc >= bestSc) {
676
  bestSc = sc;
677
  bestM = m;
678
  }
679
  }
680
  if (!bestM || (bestM.index || 0) + bestM[0].length <= 260) {
681
+ if (ftype === "list" && spec.items === "number" && bestM) {
682
+ const bStart = bestM.index || 0;
683
+ return { subState: state.slice(bStart, bStart + bestM[0].length), winStart: bStart };
684
+ }
685
+ return { subState: state.slice(0, 480), winStart: 0 };
686
+ }
687
+ if (ftype === "list" && spec.items === "number") {
688
+ const bStart = bestM.index || 0;
689
+ return { subState: state.slice(bStart, bStart + bestM[0].length), winStart: bStart };
690
  }
691
  let winStart = Math.max(0, (bestM.index || 0) - 30);
692
  while (
 
696
  ) {
697
  winStart -= 1;
698
  }
699
+ return { subState: state.slice(winStart, winStart + 480), winStart };
700
  }
701
 
702
  async function extract(state, fields) {
 
719
  const spec = fields[fname];
720
  const query = formatFieldQuery(fname, spec);
721
  const qTok = tokenizeSegmentWithOffsets(query, vocabMap);
722
+ const { subState, winStart } = selectContextWindow(state, spec, fname);
723
  const stateTok = tokenizeSegmentWithOffsets(subState, vocabMap);
724
 
725
+ const maxStateToks = Math.max(16, 512 - qTok.tokens.length - 3);
726
  const sTokens = stateTok.tokens.slice(0, maxStateToks);
727
  const sOffsets = stateTok.offsets.slice(0, maxStateToks);
728
 
 
743
  encodedRows.push({
744
  fname,
745
  spec,
746
+ subState,
747
+ winStart,
748
  inputIds,
749
  stateMask,
750
  offsets,
 
866
  );
867
 
868
  if (ftype === "list") {
869
+ if (spec.items === "number") {
870
+ const winStr = row.subState || state;
871
+ const wOff = row.winStart || 0;
872
+ const qList = `${String(fname || "").replace(/_/g, " ")} ${spec.instructions || spec.instruction || ""}`.toLowerCase();
873
+ const onlyCurr = /[$€£¥]\d/.test(winStr) && /\b(?:usd|dollars?|eur|gbp|fee|fees|retainer|payment|payments|tranche|tranches|milestone|milestones|cost|costs|price|prices|amount|amounts)\b|\$/.test(qList);
874
+ const colonIdx = winStr.indexOf(":");
875
+ const hasPostColonNums = colonIdx >= 0 && /\d/.test(winStr.slice(colonIdx + 1));
876
+ const numSpans = [];
877
+ const numVals = [];
878
+ for (const mN of winStr.matchAll(/(?<![A-Za-z0-9_-])([$€£¥])?(\d+(?:,\d{3})*(?:\.\d+)?)(?![A-Za-z0-9_]|-\d)/g)) {
879
+ if (onlyCurr && !mN[1]) continue;
880
+ if (!onlyCurr && hasPostColonNums && (mN.index || 0) < colonIdx) continue;
881
+ const sAbs = wOff + (mN.index || 0);
882
+ const eAbs = sAbs + mN[0].length;
883
+ const bef = state.slice(Math.max(0, sAbs - 25), sAbs);
884
+ const aft = state.slice(eAbs, Math.min(state.length, eAbs + 10));
885
+ if (/(?:section|clause|article|page|part|schedule|exhibit)\s*$/i.test(bef) || (/(?:^|\n)\s*$/.test(bef) && aft.startsWith(". "))) {
886
+ continue;
887
+ }
888
+ const fv = Number(mN[2].replace(/,/g, ""));
889
+ numVals.push(fv);
890
+ numSpans.push([sAbs, eAbs]);
891
+ }
892
+ answers[fname] = {
893
+ type: "list",
894
+ value: numVals,
895
+ spans: numSpans,
896
+ confidence: conf,
897
+ span_entropy_bits: spanEntropy,
898
+ neural_op: opRoutes[fname],
899
+ };
900
+ continue;
901
+ }
902
  const spansList = [];
903
  const valsList = [];
904
  let curS = null;
 
1006
  ? Number((nums[0] + nums[1]).toFixed(spec.decimals || 2))
1007
  : Number(nums[0] || 0);
1008
  } else {
1009
+ const allDocNums = Array.from(state.matchAll(/(?<![A-Za-z0-9_-])\d+(?:,\d{3})*(?:\.\d+)?(?![A-Za-z0-9_]|-\d)/g));
1010
+ if (allDocNums.length >= 3 && (state.length > 260 || !/\d/.test(spanText))) {
1011
+ const disRes = disambiguateNumericField(state, fname, spec);
1012
+ if (disRes !== null) {
1013
+ disRes.neural_op = opRoutes[fname];
1014
+ answers[fname] = disRes;
1015
+ continue;
1016
+ }
1017
+ }
1018
  const mNum = spanText.match(/-?\d+(?:,\d{3})*(?:\.\d+)?/);
1019
  guessVal = mNum
1020
  ? spec.integer