#!/usr/bin/env python3
"""EvidenceLink-DART-Reranker-KO — inference helper (self-contained, Apache-2.0).
Task: given one DART *correction* filing (query) and a candidate set of original filings from the
same issuer, pick the original filing the correction amends, or return NONE.
This file reproduces the exact input format used in training and in the sealed evaluation:
* cp949-aware decoding of OpenDART document.xml (decoder v2)
* field extraction (series/round, principal, conversion price, coupon, yield, refixing floor)
* masked cards (first-submission date and 14-digit receipt numbers are masked)
* a one-line "[요약]" summary prefixed to every card (survives longest_first truncation)
* pointwise cross-encoder scoring, max_length 1024, then argmax + frozen NONE threshold
Usage:
python evidencelink.py --selftest # synthetic example, no network besides the model
python evidencelink.py --model . --example examples/synthetic_query.json
"""
from __future__ import annotations
import argparse, io, json, math, re, sys, zipfile
from pathlib import Path
NONE_THRESHOLD = -8.4375 # frozen on dev before the sealed evaluation (prereg_E_v2_4)
MAX_LEN = 1024
CARD_FORM_CHARS = 2200
CARD_CORR_CHARS = 1200
# ── decoder v2: OpenDART document.xml (cp949 before mid-2022, utf-8 after) ────────────────────────
_DECL = re.compile(rb'<\?xml[^>]*encoding=["\']([A-Za-z0-9_\-]+)["\']')
def decode_member(raw: bytes) -> tuple[str, str]:
"""declared encoding -> utf-8 strict -> cp949 -> utf-8 replace."""
m = _DECL.search(raw[:200]); cands = ([m.group(1).decode("ascii").lower()] if m else []) + ["utf-8", "cp949"]
for enc in cands:
try:
enc2 = "cp949" if enc in ("euc-kr", "euckr", "ks_c_5601-1987") else enc
return raw.decode(enc2), enc2
except (UnicodeDecodeError, LookupError):
continue
return raw.decode("utf-8", errors="replace"), "utf-8-replace"
def dart_zip_to_text(b: bytes) -> str:
"""OpenDART `document.xml` API response (zip bytes) -> plain text with ' | ' cell separators."""
z = zipfile.ZipFile(io.BytesIO(b)); parts = []
for n in z.namelist():
t, _ = decode_member(z.read(n))
t = re.sub(r"(TD|TH|TE|TU)>", " | ", t, flags=re.I)
t = re.sub(r"(TR|P|TITLE|SECTION-\d)>|
", "\n", t, flags=re.I)
t = re.sub(r"<[^>]+>", "", t)
parts.append(re.sub(r"[ \t ]+", " ", t))
return "\n".join(x.strip() for x in "\n".join(parts).splitlines() if x.strip())
# ── fields / cards ───────────────────────────────────────────────────────────────────────────────
FIELD_LABELS = {
"principal_krw": r"(\d+\.\s*)?사채의\s*권면\s*(\(전자등록\))?\s*총액\s*\(원\)",
"coupon_rate_pct": r"표면이자율\s*\(%\)",
"redemption_yield_pct": r"만기이자율\s*\(%\)",
"maturity_date": r"(\d+\.\s*)?사채만기일",
"conversion_price_krw": r"전환가액\s*\(원/주\)",
"refixing_floor_krw": r"최저\s*조정가액\s*\(원\)",
"issue_date": r"(\d+\.\s*)?납입일",
}
_EMPTY = {"", "-", "--", "해당사항없음", "해당사항 없음", "N/A", "없음"}
_FORM_START = re.compile(r"^1\.\s*사채의\s*종류\s*\|?\s*$")
_FIRST = re.compile(r"최초\s*제출일\s*:?[\s|]*(\d{4})\s*(?:년|[.\-/])\s*(\d{1,2})\s*(?:월|[.\-/])\s*(\d{1,2})")
_SERIES = re.compile(r"^1\.\s*사채의\s*종류\s*\|?\s*\n\s*회\s*차\s*\|?\s*\n\s*\(?제?\s*(\d+)", re.M)
_RCEPT = re.compile(r"(? str:
lines = text.splitlines(); idx = [i for i, l in enumerate(lines) if _FORM_START.match(l.strip())]
return "\n".join(lines[idx[-1]:]) if idx else text
def correction_section(text: str) -> str:
lines = text.splitlines(); s = next((i for i, l in enumerate(lines) if "정정사항" in l), None)
e = next((i for i, l in enumerate(lines) if _FORM_START.match(l.strip())), len(lines))
return "\n".join(lines[s:e]) if s is not None else ""
def _norm(f: str, v):
if v is None: return None
s = str(v).strip()
if s in _EMPTY: return None
if f.endswith("_date"):
n = re.findall(r"\d+", s)
if len(n) == 1 and len(n[0]) == 8: n = [n[0][:4], n[0][4:6], n[0][6:]]
return f"{int(n[0]):04d}-{int(n[1]):02d}-{int(n[2]):02d}" if len(n) >= 3 else None
if f.endswith("_krw"):
m = re.search(r"\d[\d,]*", s); return str(int(m.group().replace(",", ""))) if m else None
if f.endswith("_pct"):
m = re.search(r"-?\d+(?:\.\d+)?", s); return str(float(m.group())) if m else None
return s
def extract_fields(text: str) -> dict:
cells = [c.strip().rstrip("|").strip() for line in form_text(text).splitlines() for c in line.split(" | ")]
cells = [c for c in cells if c]; res: dict = {}
for f, pat in FIELD_LABELS.items():
rx = re.compile(rf"^{pat}$")
for i, c in enumerate(cells):
if rx.match(c):
v = cells[i + 1] if i + 1 < len(cells) else ""
res[f] = None if v in _EMPTY or rx.match(v) else _norm(f, v)
break
m = _SERIES.search(form_text(text)); res["series"] = str(int(m.group(1))) if m else None
return res
def mask(t: str) -> str:
return _RCEPT.sub("[RCEPT]", _FIRST.sub("최초제출일 : [MASK]", t))
def summary_line(f: dict) -> str:
return "[요약] " + " | ".join(f"{lab} {f.get(k)}" for k, lab in _SUM_F if f.get(k) is not None)
def query_card(corp_name: str, text: str) -> str:
body = mask(f"회사: {corp_name}\n[정정사항]\n{correction_section(text)[:CARD_CORR_CHARS]}\n[서식]\n{form_text(text)[:CARD_FORM_CHARS]}")
return summary_line(extract_fields(text)) + "\n" + body
def candidate_card(corp_name: str, rcept_dt: str, text: str) -> str:
body = mask(f"회사: {corp_name}\n접수일: {rcept_dt}\n{form_text(text)[:CARD_FORM_CHARS]}")
return summary_line(extract_fields(text)) + "\n" + body
# ── scoring ──────────────────────────────────────────────────────────────────────────────────────
def load(model_dir: str = "."):
import torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer
tok = AutoTokenizer.from_pretrained(model_dir)
m = AutoModelForSequenceClassification.from_pretrained(model_dir, torch_dtype=torch.float32).eval()
return tok, m
def score(tok, m, q_card: str, c_cards: list[str], batch: int = 8) -> list[float]:
import torch
out: list[float] = []
with torch.no_grad():
for i in range(0, len(c_cards), batch):
b = c_cards[i:i + batch]
x = tok([q_card] * len(b), b, truncation="longest_first", max_length=MAX_LEN, padding=True, return_tensors="pt")
x = {k: v.to(m.device) for k, v in x.items()}
out += m(**x).logits.view(-1).float().tolist()
return out
def link(tok, m, q_card: str, candidates: list[dict], thr: float = NONE_THRESHOLD) -> dict:
"""candidates: [{"id": ..., "card": ...}], ordered by receipt number descending (ties -> first wins)."""
s = score(tok, m, q_card, [c["card"] for c in candidates])
best, bs = None, -math.inf
for c, v in zip(candidates, s):
if v > bs: best, bs = c["id"], v
return {"pred": best if bs >= thr else None, "top_score": bs, "threshold": thr,
"scores": {c["id"]: v for c, v in zip(candidates, s)}}
def run_example(model_dir: str, path: str) -> dict:
ex = json.loads(Path(path).read_text(encoding="utf-8"))
q = query_card(ex["query"]["corp_name"], ex["query"]["text"])
cands = sorted(ex["candidates"], key=lambda c: c["id"], reverse=True)
cc = [{"id": c["id"], "card": candidate_card(c["corp_name"], c["rcept_dt"], c["text"])} for c in cands]
tok, m = load(model_dir)
r = link(tok, m, q, cc)
r["expected"] = ex.get("expected"); r["query_fields"] = extract_fields(ex["query"]["text"])
return r
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--model", default=str(Path(__file__).resolve().parent))
ap.add_argument("--example", default=str(Path(__file__).resolve().parent / "examples" / "synthetic_query.json"))
ap.add_argument("--selftest", action="store_true", help="decoder + card checks without the model")
a = ap.parse_args()
if a.selftest:
raw = "
회차
".encode("cp949") buf = io.BytesIO(); zipfile.ZipFile(buf, "w").writestr("d.xml", raw) assert "회차" in dart_zip_to_text(buf.getvalue()), "cp949 decode failed" ex = json.loads(Path(a.example).read_text(encoding="utf-8")) q = query_card(ex["query"]["corp_name"], ex["query"]["text"]) assert q.startswith("[요약] ") and not re.search(r"(?