#!/usr/bin/env python3 """EvidenceLink-DART-Reranker-KO — inference helper (self-contained, Apache-2.0). Task: given one DART *correction* filing (query) and a candidate set of original filings from the same issuer, pick the original filing the correction amends, or return NONE. This file reproduces the exact input format used in training and in the sealed evaluation: * cp949-aware decoding of OpenDART document.xml (decoder v2) * field extraction (series/round, principal, conversion price, coupon, yield, refixing floor) * masked cards (first-submission date and 14-digit receipt numbers are masked) * a one-line "[요약]" summary prefixed to every card (survives longest_first truncation) * pointwise cross-encoder scoring, max_length 1024, then argmax + frozen NONE threshold Usage: python evidencelink.py --selftest # synthetic example, no network besides the model python evidencelink.py --model . --example examples/synthetic_query.json """ from __future__ import annotations import argparse, io, json, math, re, sys, zipfile from pathlib import Path NONE_THRESHOLD = -8.4375 # frozen on dev before the sealed evaluation (prereg_E_v2_4) MAX_LEN = 1024 CARD_FORM_CHARS = 2200 CARD_CORR_CHARS = 1200 # ── decoder v2: OpenDART document.xml (cp949 before mid-2022, utf-8 after) ──────────────────────── _DECL = re.compile(rb'<\?xml[^>]*encoding=["\']([A-Za-z0-9_\-]+)["\']') def decode_member(raw: bytes) -> tuple[str, str]: """declared encoding -> utf-8 strict -> cp949 -> utf-8 replace.""" m = _DECL.search(raw[:200]); cands = ([m.group(1).decode("ascii").lower()] if m else []) + ["utf-8", "cp949"] for enc in cands: try: enc2 = "cp949" if enc in ("euc-kr", "euckr", "ks_c_5601-1987") else enc return raw.decode(enc2), enc2 except (UnicodeDecodeError, LookupError): continue return raw.decode("utf-8", errors="replace"), "utf-8-replace" def dart_zip_to_text(b: bytes) -> str: """OpenDART `document.xml` API response (zip bytes) -> plain text with ' | ' cell separators.""" z = zipfile.ZipFile(io.BytesIO(b)); parts = [] for n in z.namelist(): t, _ = decode_member(z.read(n)) t = re.sub(r"", " | ", t, flags=re.I) t = re.sub(r"|", "\n", t, flags=re.I) t = re.sub(r"<[^>]+>", "", t) parts.append(re.sub(r"[ \t ]+", " ", t)) return "\n".join(x.strip() for x in "\n".join(parts).splitlines() if x.strip()) # ── fields / cards ─────────────────────────────────────────────────────────────────────────────── FIELD_LABELS = { "principal_krw": r"(\d+\.\s*)?사채의\s*권면\s*(\(전자등록\))?\s*총액\s*\(원\)", "coupon_rate_pct": r"표면이자율\s*\(%\)", "redemption_yield_pct": r"만기이자율\s*\(%\)", "maturity_date": r"(\d+\.\s*)?사채만기일", "conversion_price_krw": r"전환가액\s*\(원/주\)", "refixing_floor_krw": r"최저\s*조정가액\s*\(원\)", "issue_date": r"(\d+\.\s*)?납입일", } _EMPTY = {"", "-", "--", "해당사항없음", "해당사항 없음", "N/A", "없음"} _FORM_START = re.compile(r"^1\.\s*사채의\s*종류\s*\|?\s*$") _FIRST = re.compile(r"최초\s*제출일\s*:?[\s|]*(\d{4})\s*(?:년|[.\-/])\s*(\d{1,2})\s*(?:월|[.\-/])\s*(\d{1,2})") _SERIES = re.compile(r"^1\.\s*사채의\s*종류\s*\|?\s*\n\s*회\s*차\s*\|?\s*\n\s*\(?제?\s*(\d+)", re.M) _RCEPT = re.compile(r"(? str: lines = text.splitlines(); idx = [i for i, l in enumerate(lines) if _FORM_START.match(l.strip())] return "\n".join(lines[idx[-1]:]) if idx else text def correction_section(text: str) -> str: lines = text.splitlines(); s = next((i for i, l in enumerate(lines) if "정정사항" in l), None) e = next((i for i, l in enumerate(lines) if _FORM_START.match(l.strip())), len(lines)) return "\n".join(lines[s:e]) if s is not None else "" def _norm(f: str, v): if v is None: return None s = str(v).strip() if s in _EMPTY: return None if f.endswith("_date"): n = re.findall(r"\d+", s) if len(n) == 1 and len(n[0]) == 8: n = [n[0][:4], n[0][4:6], n[0][6:]] return f"{int(n[0]):04d}-{int(n[1]):02d}-{int(n[2]):02d}" if len(n) >= 3 else None if f.endswith("_krw"): m = re.search(r"\d[\d,]*", s); return str(int(m.group().replace(",", ""))) if m else None if f.endswith("_pct"): m = re.search(r"-?\d+(?:\.\d+)?", s); return str(float(m.group())) if m else None return s def extract_fields(text: str) -> dict: cells = [c.strip().rstrip("|").strip() for line in form_text(text).splitlines() for c in line.split(" | ")] cells = [c for c in cells if c]; res: dict = {} for f, pat in FIELD_LABELS.items(): rx = re.compile(rf"^{pat}$") for i, c in enumerate(cells): if rx.match(c): v = cells[i + 1] if i + 1 < len(cells) else "" res[f] = None if v in _EMPTY or rx.match(v) else _norm(f, v) break m = _SERIES.search(form_text(text)); res["series"] = str(int(m.group(1))) if m else None return res def mask(t: str) -> str: return _RCEPT.sub("[RCEPT]", _FIRST.sub("최초제출일 : [MASK]", t)) def summary_line(f: dict) -> str: return "[요약] " + " | ".join(f"{lab} {f.get(k)}" for k, lab in _SUM_F if f.get(k) is not None) def query_card(corp_name: str, text: str) -> str: body = mask(f"회사: {corp_name}\n[정정사항]\n{correction_section(text)[:CARD_CORR_CHARS]}\n[서식]\n{form_text(text)[:CARD_FORM_CHARS]}") return summary_line(extract_fields(text)) + "\n" + body def candidate_card(corp_name: str, rcept_dt: str, text: str) -> str: body = mask(f"회사: {corp_name}\n접수일: {rcept_dt}\n{form_text(text)[:CARD_FORM_CHARS]}") return summary_line(extract_fields(text)) + "\n" + body # ── scoring ────────────────────────────────────────────────────────────────────────────────────── def load(model_dir: str = "."): import torch from transformers import AutoModelForSequenceClassification, AutoTokenizer tok = AutoTokenizer.from_pretrained(model_dir) m = AutoModelForSequenceClassification.from_pretrained(model_dir, torch_dtype=torch.float32).eval() return tok, m def score(tok, m, q_card: str, c_cards: list[str], batch: int = 8) -> list[float]: import torch out: list[float] = [] with torch.no_grad(): for i in range(0, len(c_cards), batch): b = c_cards[i:i + batch] x = tok([q_card] * len(b), b, truncation="longest_first", max_length=MAX_LEN, padding=True, return_tensors="pt") x = {k: v.to(m.device) for k, v in x.items()} out += m(**x).logits.view(-1).float().tolist() return out def link(tok, m, q_card: str, candidates: list[dict], thr: float = NONE_THRESHOLD) -> dict: """candidates: [{"id": ..., "card": ...}], ordered by receipt number descending (ties -> first wins).""" s = score(tok, m, q_card, [c["card"] for c in candidates]) best, bs = None, -math.inf for c, v in zip(candidates, s): if v > bs: best, bs = c["id"], v return {"pred": best if bs >= thr else None, "top_score": bs, "threshold": thr, "scores": {c["id"]: v for c, v in zip(candidates, s)}} def run_example(model_dir: str, path: str) -> dict: ex = json.loads(Path(path).read_text(encoding="utf-8")) q = query_card(ex["query"]["corp_name"], ex["query"]["text"]) cands = sorted(ex["candidates"], key=lambda c: c["id"], reverse=True) cc = [{"id": c["id"], "card": candidate_card(c["corp_name"], c["rcept_dt"], c["text"])} for c in cands] tok, m = load(model_dir) r = link(tok, m, q, cc) r["expected"] = ex.get("expected"); r["query_fields"] = extract_fields(ex["query"]["text"]) return r def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--model", default=str(Path(__file__).resolve().parent)) ap.add_argument("--example", default=str(Path(__file__).resolve().parent / "examples" / "synthetic_query.json")) ap.add_argument("--selftest", action="store_true", help="decoder + card checks without the model") a = ap.parse_args() if a.selftest: raw = "

회차

".encode("cp949") buf = io.BytesIO(); zipfile.ZipFile(buf, "w").writestr("d.xml", raw) assert "회차" in dart_zip_to_text(buf.getvalue()), "cp949 decode failed" ex = json.loads(Path(a.example).read_text(encoding="utf-8")) q = query_card(ex["query"]["corp_name"], ex["query"]["text"]) assert q.startswith("[요약] ") and not re.search(r"(?