thaki-AI's picture
EvidenceLink-DART-Reranker-KO: DART correction-to-original linking reranker (prereg_E_v2_4 PASS_V24)
039683c verified
Raw History Blame Contribute Delete
9.7 kB
#!/usr/bin/env python3
"""EvidenceLink-DART-Reranker-KO β€” inference helper (self-contained, Apache-2.0).
Task: given one DART *correction* filing (query) and a candidate set of original filings from the
same issuer, pick the original filing the correction amends, or return NONE.
This file reproduces the exact input format used in training and in the sealed evaluation:
* cp949-aware decoding of OpenDART document.xml (decoder v2)
* field extraction (series/round, principal, conversion price, coupon, yield, refixing floor)
* masked cards (first-submission date and 14-digit receipt numbers are masked)
* a one-line "[μš”μ•½]" summary prefixed to every card (survives longest_first truncation)
* pointwise cross-encoder scoring, max_length 1024, then argmax + frozen NONE threshold
Usage:
python evidencelink.py --selftest # synthetic example, no network besides the model
python evidencelink.py --model . --example examples/synthetic_query.json
"""
from __future__ import annotations
import argparse, io, json, math, re, sys, zipfile
from pathlib import Path
NONE_THRESHOLD = -8.4375 # frozen on dev before the sealed evaluation (prereg_E_v2_4)
MAX_LEN = 1024
CARD_FORM_CHARS = 2200
CARD_CORR_CHARS = 1200
# ── decoder v2: OpenDART document.xml (cp949 before mid-2022, utf-8 after) ────────────────────────
_DECL = re.compile(rb'<\?xml[^>]*encoding=["\']([A-Za-z0-9_\-]+)["\']')
def decode_member(raw: bytes) -> tuple[str, str]:
"""declared encoding -> utf-8 strict -> cp949 -> utf-8 replace."""
m = _DECL.search(raw[:200]); cands = ([m.group(1).decode("ascii").lower()] if m else []) + ["utf-8", "cp949"]
for enc in cands:
try:
enc2 = "cp949" if enc in ("euc-kr", "euckr", "ks_c_5601-1987") else enc
return raw.decode(enc2), enc2
except (UnicodeDecodeError, LookupError):
continue
return raw.decode("utf-8", errors="replace"), "utf-8-replace"
def dart_zip_to_text(b: bytes) -> str:
"""OpenDART `document.xml` API response (zip bytes) -> plain text with ' | ' cell separators."""
z = zipfile.ZipFile(io.BytesIO(b)); parts = []
for n in z.namelist():
t, _ = decode_member(z.read(n))
t = re.sub(r"</(TD|TH|TE|TU)>", " | ", t, flags=re.I)
t = re.sub(r"</(TR|P|TITLE|SECTION-\d)>|<BR\s*/?>", "\n", t, flags=re.I)
t = re.sub(r"<[^>]+>", "", t)
parts.append(re.sub(r"[ \tΒ ]+", " ", t))
return "\n".join(x.strip() for x in "\n".join(parts).splitlines() if x.strip())
# ── fields / cards ───────────────────────────────────────────────────────────────────────────────
FIELD_LABELS = {
"principal_krw": r"(\d+\.\s*)?μ‚¬μ±„μ˜\s*ꢌ면\s*(\(μ „μžλ“±λ‘\))?\s*총앑\s*\(원\)",
"coupon_rate_pct": r"ν‘œλ©΄μ΄μžμœ¨\s*\(%\)",
"redemption_yield_pct": r"만기이자율\s*\(%\)",
"maturity_date": r"(\d+\.\s*)?μ‚¬μ±„λ§ŒκΈ°μΌ",
"conversion_price_krw": r"μ „ν™˜κ°€μ•‘\s*\(원/μ£Ό\)",
"refixing_floor_krw": r"μ΅œμ €\s*μ‘°μ •κ°€μ•‘\s*\(원\)",
"issue_date": r"(\d+\.\s*)?λ‚©μž…μΌ",
}
_EMPTY = {"", "-", "--", "ν•΄λ‹Ήμ‚¬ν•­μ—†μŒ", "해당사항 μ—†μŒ", "N/A", "μ—†μŒ"}
_FORM_START = re.compile(r"^1\.\s*μ‚¬μ±„μ˜\s*μ’…λ₯˜\s*\|?\s*$")
_FIRST = re.compile(r"졜초\s*제좜일\s*:?[\s|]*(\d{4})\s*(?:λ…„|[.\-/])\s*(\d{1,2})\s*(?:μ›”|[.\-/])\s*(\d{1,2})")
_SERIES = re.compile(r"^1\.\s*μ‚¬μ±„μ˜\s*μ’…λ₯˜\s*\|?\s*\n\s*회\s*μ°¨\s*\|?\s*\n\s*\(?제?\s*(\d+)", re.M)
_RCEPT = re.compile(r"(?<!\d)\d{14}(?!\d)")
_SUM_F = (("series", "회차"), ("principal_krw", "κΆŒλ©΄μ΄μ•‘"), ("conversion_price_krw", "μ „ν™˜κ°€μ•‘"), ("coupon_rate_pct", "ν‘œλ©΄μ΄μžμœ¨"),
("redemption_yield_pct", "만기이자율"), ("refixing_floor_krw", "μ΅œμ €μ‘°μ •κ°€μ•‘"))
def form_text(text: str) -> str:
lines = text.splitlines(); idx = [i for i, l in enumerate(lines) if _FORM_START.match(l.strip())]
return "\n".join(lines[idx[-1]:]) if idx else text
def correction_section(text: str) -> str:
lines = text.splitlines(); s = next((i for i, l in enumerate(lines) if "정정사항" in l), None)
e = next((i for i, l in enumerate(lines) if _FORM_START.match(l.strip())), len(lines))
return "\n".join(lines[s:e]) if s is not None else ""
def _norm(f: str, v):
if v is None: return None
s = str(v).strip()
if s in _EMPTY: return None
if f.endswith("_date"):
n = re.findall(r"\d+", s)
if len(n) == 1 and len(n[0]) == 8: n = [n[0][:4], n[0][4:6], n[0][6:]]
return f"{int(n[0]):04d}-{int(n[1]):02d}-{int(n[2]):02d}" if len(n) >= 3 else None
if f.endswith("_krw"):
m = re.search(r"\d[\d,]*", s); return str(int(m.group().replace(",", ""))) if m else None
if f.endswith("_pct"):
m = re.search(r"-?\d+(?:\.\d+)?", s); return str(float(m.group())) if m else None
return s
def extract_fields(text: str) -> dict:
cells = [c.strip().rstrip("|").strip() for line in form_text(text).splitlines() for c in line.split(" | ")]
cells = [c for c in cells if c]; res: dict = {}
for f, pat in FIELD_LABELS.items():
rx = re.compile(rf"^{pat}$")
for i, c in enumerate(cells):
if rx.match(c):
v = cells[i + 1] if i + 1 < len(cells) else ""
res[f] = None if v in _EMPTY or rx.match(v) else _norm(f, v)
break
m = _SERIES.search(form_text(text)); res["series"] = str(int(m.group(1))) if m else None
return res
def mask(t: str) -> str:
return _RCEPT.sub("[RCEPT]", _FIRST.sub("졜초제좜일 : [MASK]", t))
def summary_line(f: dict) -> str:
return "[μš”μ•½] " + " | ".join(f"{lab} {f.get(k)}" for k, lab in _SUM_F if f.get(k) is not None)
def query_card(corp_name: str, text: str) -> str:
body = mask(f"νšŒμ‚¬: {corp_name}\n[정정사항]\n{correction_section(text)[:CARD_CORR_CHARS]}\n[μ„œμ‹]\n{form_text(text)[:CARD_FORM_CHARS]}")
return summary_line(extract_fields(text)) + "\n" + body
def candidate_card(corp_name: str, rcept_dt: str, text: str) -> str:
body = mask(f"νšŒμ‚¬: {corp_name}\nμ ‘μˆ˜μΌ: {rcept_dt}\n{form_text(text)[:CARD_FORM_CHARS]}")
return summary_line(extract_fields(text)) + "\n" + body
# ── scoring ──────────────────────────────────────────────────────────────────────────────────────
def load(model_dir: str = "."):
import torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer
tok = AutoTokenizer.from_pretrained(model_dir)
m = AutoModelForSequenceClassification.from_pretrained(model_dir, torch_dtype=torch.float32).eval()
return tok, m
def score(tok, m, q_card: str, c_cards: list[str], batch: int = 8) -> list[float]:
import torch
out: list[float] = []
with torch.no_grad():
for i in range(0, len(c_cards), batch):
b = c_cards[i:i + batch]
x = tok([q_card] * len(b), b, truncation="longest_first", max_length=MAX_LEN, padding=True, return_tensors="pt")
x = {k: v.to(m.device) for k, v in x.items()}
out += m(**x).logits.view(-1).float().tolist()
return out
def link(tok, m, q_card: str, candidates: list[dict], thr: float = NONE_THRESHOLD) -> dict:
"""candidates: [{"id": ..., "card": ...}], ordered by receipt number descending (ties -> first wins)."""
s = score(tok, m, q_card, [c["card"] for c in candidates])
best, bs = None, -math.inf
for c, v in zip(candidates, s):
if v > bs: best, bs = c["id"], v
return {"pred": best if bs >= thr else None, "top_score": bs, "threshold": thr,
"scores": {c["id"]: v for c, v in zip(candidates, s)}}
def run_example(model_dir: str, path: str) -> dict:
ex = json.loads(Path(path).read_text(encoding="utf-8"))
q = query_card(ex["query"]["corp_name"], ex["query"]["text"])
cands = sorted(ex["candidates"], key=lambda c: c["id"], reverse=True)
cc = [{"id": c["id"], "card": candidate_card(c["corp_name"], c["rcept_dt"], c["text"])} for c in cands]
tok, m = load(model_dir)
r = link(tok, m, q, cc)
r["expected"] = ex.get("expected"); r["query_fields"] = extract_fields(ex["query"]["text"])
return r
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--model", default=str(Path(__file__).resolve().parent))
ap.add_argument("--example", default=str(Path(__file__).resolve().parent / "examples" / "synthetic_query.json"))
ap.add_argument("--selftest", action="store_true", help="decoder + card checks without the model")
a = ap.parse_args()
if a.selftest:
raw = "<?xml version=\"1.0\" encoding=\"euc-kr\"?><P>회차</P>".encode("cp949")
buf = io.BytesIO(); zipfile.ZipFile(buf, "w").writestr("d.xml", raw)
assert "회차" in dart_zip_to_text(buf.getvalue()), "cp949 decode failed"
ex = json.loads(Path(a.example).read_text(encoding="utf-8"))
q = query_card(ex["query"]["corp_name"], ex["query"]["text"])
assert q.startswith("[μš”μ•½] ") and not re.search(r"(?<!\d)\d{14}(?!\d)", q), "card format"
print(json.dumps({"selftest": "ok", "query_fields": extract_fields(ex["query"]["text"])}, ensure_ascii=False))
return 0
print(json.dumps(run_example(a.model, a.example), ensure_ascii=False, indent=1))
return 0
if __name__ == "__main__":
sys.exit(main())