beyoru commited on
Commit
883df0c
·
verified ·
1 Parent(s): cf7aa5a

Upload gemma_extract.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. gemma_extract.py +25 -5
gemma_extract.py CHANGED
@@ -91,6 +91,15 @@ def snap(text, a, b):
91
  return a, b
92
 
93
 
 
 
 
 
 
 
 
 
 
94
  def merge(spans):
95
  out = []
96
  for a, b in sorted(spans):
@@ -291,7 +300,7 @@ def decode(out, enc, state_text, null_threshold=0.5, en=None):
291
  k = j + 1
292
  while k < len(tg) and tg[k] == 2:
293
  k += 1
294
- spans.append((off[j][0], off[k - 1][1], pr[j, tg[j]].item()))
295
  j = k
296
  else:
297
  j += 1
@@ -306,7 +315,7 @@ def decode(out, enc, state_text, null_threshold=0.5, en=None):
306
  joint = ps[:, None] * pe[None, :] * band
307
  best = int(joint.argmax())
308
  s, e = divmod(best, S)
309
- cs, ce = off[s][0], off[e][1]
310
  r = {"null": p_null > null_threshold, "p_null": p_null, "guess": state_text[cs:ce].strip(),
311
  "confidence": joint.view(-1)[best].item() * (1 - p_null), "spans": [[cs, ce]]}
312
  if ty == "number":
@@ -387,6 +396,17 @@ _VI = _re.compile(r"[ăâđêôơưạảấầẩẫậắằẳẵặẹẻẽ
387
  SUPPORTED = {"text", "list", "number", "date"}
388
 
389
 
 
 
 
 
 
 
 
 
 
 
 
390
  class GemmaExtract:
391
  """Typed field extraction in one forward pass, no text generation.
392
 
@@ -431,8 +451,8 @@ class GemmaExtract:
431
  for k, v in fields.items():
432
  if v.get("type") not in SUPPORTED:
433
  raise ValueError("field %s: type must be one of %s" % (k, sorted(SUPPORTED)))
434
- lang = "vi" if len(_VI.findall(state)) > 3 else "en"
435
- thr_all = self.cfg["null_threshold"][lang] if null_threshold is None else null_threshold
436
  en = {"en": True, "vi": False}.get(number_style)
437
  answers, usage = {}, {"state_tokens": 0, "passes": 0}
438
  queue = self._passes({k: {"type": v["type"], "instructions": v.get("instructions") or k} for k, v in fields.items()})
@@ -456,4 +476,4 @@ class GemmaExtract:
456
  usage["state_tokens"] = max(usage["state_tokens"], e["state"][1] - e["state"][0])
457
  usage["passes"] += 1
458
  usage["truncated"] = usage["state_tokens"] < len(self.tok(state, add_special_tokens=False)["input_ids"])
459
- return {"answers": answers, "usage": usage}
 
91
  return a, b
92
 
93
 
94
+ def trim(text, a, b):
95
+ """Bo khoang trang hai dau span (offset token Gemma kem dau cach dung truoc)."""
96
+ while a < b and text[a].isspace():
97
+ a += 1
98
+ while b > a and text[b - 1].isspace():
99
+ b -= 1
100
+ return a, b
101
+
102
+
103
  def merge(spans):
104
  out = []
105
  for a, b in sorted(spans):
 
300
  k = j + 1
301
  while k < len(tg) and tg[k] == 2:
302
  k += 1
303
+ spans.append((*trim(state_text, off[j][0], off[k - 1][1]), pr[j, tg[j]].item()))
304
  j = k
305
  else:
306
  j += 1
 
315
  joint = ps[:, None] * pe[None, :] * band
316
  best = int(joint.argmax())
317
  s, e = divmod(best, S)
318
+ cs, ce = trim(state_text, off[s][0], off[e][1])
319
  r = {"null": p_null > null_threshold, "p_null": p_null, "guess": state_text[cs:ce].strip(),
320
  "confidence": joint.view(-1)[best].item() * (1 - p_null), "spans": [[cs, ce]]}
321
  if ty == "number":
 
396
  SUPPORTED = {"text", "list", "number", "date"}
397
 
398
 
399
+ def detect_language(text):
400
+ """'vi' neu co dau tieng Viet; 'en' neu chu cai gan nhu toan ASCII; con lai 'default' (nguong null cao hon:
401
+ do tren XQuAD 7 ngon ngu khac, nguong 0.4 tra null nham 3-8%, 0.8 con 0.3-3%)."""
402
+ if len(_VI.findall(text)) > 3:
403
+ return "vi"
404
+ letters = [c for c in text if c.isalpha()]
405
+ if letters and sum(c.isascii() for c in letters) / len(letters) > 0.98:
406
+ return "en"
407
+ return "default"
408
+
409
+
410
  class GemmaExtract:
411
  """Typed field extraction in one forward pass, no text generation.
412
 
 
451
  for k, v in fields.items():
452
  if v.get("type") not in SUPPORTED:
453
  raise ValueError("field %s: type must be one of %s" % (k, sorted(SUPPORTED)))
454
+ lang = detect_language(state)
455
+ thr_all = self.cfg["null_threshold"].get(lang, self.cfg["null_threshold"]["default"]) if null_threshold is None else null_threshold
456
  en = {"en": True, "vi": False}.get(number_style)
457
  answers, usage = {}, {"state_tokens": 0, "passes": 0}
458
  queue = self._passes({k: {"type": v["type"], "instructions": v.get("instructions") or k} for k, v in fields.items()})
 
476
  usage["state_tokens"] = max(usage["state_tokens"], e["state"][1] - e["state"][0])
477
  usage["passes"] += 1
478
  usage["truncated"] = usage["state_tokens"] < len(self.tok(state, add_special_tokens=False)["input_ids"])
479
+ return {"answers": answers, "usage": usage, "language": lang}