Text Classification
Transformers
Safetensors
Korean
xlm-roberta
reranker
cross-encoder
dart
opendart
financial-disclosure
korean
record-linkage
convertible-bond
text-embeddings-inference
Instructions to use ThakiCloud/EvidenceLink-DART-Reranker-KO with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use ThakiCloud/EvidenceLink-DART-Reranker-KO with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="ThakiCloud/EvidenceLink-DART-Reranker-KO")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForSequenceClassification tokenizer = AutoTokenizer.from_pretrained("ThakiCloud/EvidenceLink-DART-Reranker-KO") model = AutoModelForSequenceClassification.from_pretrained("ThakiCloud/EvidenceLink-DART-Reranker-KO", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download evidencelink.py from ThakiCloud/EvidenceLink-DART-Reranker-KO: direct link, hf CLI and curl.
- Browser
- Download file 9.7 kB
-
https://huggingface.co/ThakiCloud/EvidenceLink-DART-Reranker-KO/resolve/main/evidencelink.py
- Command line
-
hf download hf://ThakiCloud/EvidenceLink-DART-Reranker-KO/evidencelink.py
-
curl -L -o evidencelink.py https://huggingface.co/ThakiCloud/EvidenceLink-DART-Reranker-KO/resolve/main/evidencelink.py
9.7 kB
| #!/usr/bin/env python3 | |
| """EvidenceLink-DART-Reranker-KO β inference helper (self-contained, Apache-2.0). | |
| Task: given one DART *correction* filing (query) and a candidate set of original filings from the | |
| same issuer, pick the original filing the correction amends, or return NONE. | |
| This file reproduces the exact input format used in training and in the sealed evaluation: | |
| * cp949-aware decoding of OpenDART document.xml (decoder v2) | |
| * field extraction (series/round, principal, conversion price, coupon, yield, refixing floor) | |
| * masked cards (first-submission date and 14-digit receipt numbers are masked) | |
| * a one-line "[μμ½]" summary prefixed to every card (survives longest_first truncation) | |
| * pointwise cross-encoder scoring, max_length 1024, then argmax + frozen NONE threshold | |
| Usage: | |
| python evidencelink.py --selftest # synthetic example, no network besides the model | |
| python evidencelink.py --model . --example examples/synthetic_query.json | |
| """ | |
| from __future__ import annotations | |
| import argparse, io, json, math, re, sys, zipfile | |
| from pathlib import Path | |
| NONE_THRESHOLD = -8.4375 # frozen on dev before the sealed evaluation (prereg_E_v2_4) | |
| MAX_LEN = 1024 | |
| CARD_FORM_CHARS = 2200 | |
| CARD_CORR_CHARS = 1200 | |
| # ββ decoder v2: OpenDART document.xml (cp949 before mid-2022, utf-8 after) ββββββββββββββββββββββββ | |
| _DECL = re.compile(rb'<\?xml[^>]*encoding=["\']([A-Za-z0-9_\-]+)["\']') | |
| def decode_member(raw: bytes) -> tuple[str, str]: | |
| """declared encoding -> utf-8 strict -> cp949 -> utf-8 replace.""" | |
| m = _DECL.search(raw[:200]); cands = ([m.group(1).decode("ascii").lower()] if m else []) + ["utf-8", "cp949"] | |
| for enc in cands: | |
| try: | |
| enc2 = "cp949" if enc in ("euc-kr", "euckr", "ks_c_5601-1987") else enc | |
| return raw.decode(enc2), enc2 | |
| except (UnicodeDecodeError, LookupError): | |
| continue | |
| return raw.decode("utf-8", errors="replace"), "utf-8-replace" | |
| def dart_zip_to_text(b: bytes) -> str: | |
| """OpenDART `document.xml` API response (zip bytes) -> plain text with ' | ' cell separators.""" | |
| z = zipfile.ZipFile(io.BytesIO(b)); parts = [] | |
| for n in z.namelist(): | |
| t, _ = decode_member(z.read(n)) | |
| t = re.sub(r"</(TD|TH|TE|TU)>", " | ", t, flags=re.I) | |
| t = re.sub(r"</(TR|P|TITLE|SECTION-\d)>|<BR\s*/?>", "\n", t, flags=re.I) | |
| t = re.sub(r"<[^>]+>", "", t) | |
| parts.append(re.sub(r"[ \tΒ ]+", " ", t)) | |
| return "\n".join(x.strip() for x in "\n".join(parts).splitlines() if x.strip()) | |
| # ββ fields / cards βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| FIELD_LABELS = { | |
| "principal_krw": r"(\d+\.\s*)?μ¬μ±μ\s*κΆλ©΄\s*(\(μ μλ±λ‘\))?\s*μ΄μ‘\s*\(μ\)", | |
| "coupon_rate_pct": r"νλ©΄μ΄μμ¨\s*\(%\)", | |
| "redemption_yield_pct": r"λ§κΈ°μ΄μμ¨\s*\(%\)", | |
| "maturity_date": r"(\d+\.\s*)?μ¬μ±λ§κΈ°μΌ", | |
| "conversion_price_krw": r"μ νκ°μ‘\s*\(μ/μ£Ό\)", | |
| "refixing_floor_krw": r"μ΅μ \s*μ‘°μ κ°μ‘\s*\(μ\)", | |
| "issue_date": r"(\d+\.\s*)?λ©μ μΌ", | |
| } | |
| _EMPTY = {"", "-", "--", "ν΄λΉμ¬νμμ", "ν΄λΉμ¬ν μμ", "N/A", "μμ"} | |
| _FORM_START = re.compile(r"^1\.\s*μ¬μ±μ\s*μ’ λ₯\s*\|?\s*$") | |
| _FIRST = re.compile(r"μ΅μ΄\s*μ μΆμΌ\s*:?[\s|]*(\d{4})\s*(?:λ |[.\-/])\s*(\d{1,2})\s*(?:μ|[.\-/])\s*(\d{1,2})") | |
| _SERIES = re.compile(r"^1\.\s*μ¬μ±μ\s*μ’ λ₯\s*\|?\s*\n\s*ν\s*μ°¨\s*\|?\s*\n\s*\(?μ ?\s*(\d+)", re.M) | |
| _RCEPT = re.compile(r"(?<!\d)\d{14}(?!\d)") | |
| _SUM_F = (("series", "νμ°¨"), ("principal_krw", "κΆλ©΄μ΄μ‘"), ("conversion_price_krw", "μ νκ°μ‘"), ("coupon_rate_pct", "νλ©΄μ΄μμ¨"), | |
| ("redemption_yield_pct", "λ§κΈ°μ΄μμ¨"), ("refixing_floor_krw", "μ΅μ μ‘°μ κ°μ‘")) | |
| def form_text(text: str) -> str: | |
| lines = text.splitlines(); idx = [i for i, l in enumerate(lines) if _FORM_START.match(l.strip())] | |
| return "\n".join(lines[idx[-1]:]) if idx else text | |
| def correction_section(text: str) -> str: | |
| lines = text.splitlines(); s = next((i for i, l in enumerate(lines) if "μ μ μ¬ν" in l), None) | |
| e = next((i for i, l in enumerate(lines) if _FORM_START.match(l.strip())), len(lines)) | |
| return "\n".join(lines[s:e]) if s is not None else "" | |
| def _norm(f: str, v): | |
| if v is None: return None | |
| s = str(v).strip() | |
| if s in _EMPTY: return None | |
| if f.endswith("_date"): | |
| n = re.findall(r"\d+", s) | |
| if len(n) == 1 and len(n[0]) == 8: n = [n[0][:4], n[0][4:6], n[0][6:]] | |
| return f"{int(n[0]):04d}-{int(n[1]):02d}-{int(n[2]):02d}" if len(n) >= 3 else None | |
| if f.endswith("_krw"): | |
| m = re.search(r"\d[\d,]*", s); return str(int(m.group().replace(",", ""))) if m else None | |
| if f.endswith("_pct"): | |
| m = re.search(r"-?\d+(?:\.\d+)?", s); return str(float(m.group())) if m else None | |
| return s | |
| def extract_fields(text: str) -> dict: | |
| cells = [c.strip().rstrip("|").strip() for line in form_text(text).splitlines() for c in line.split(" | ")] | |
| cells = [c for c in cells if c]; res: dict = {} | |
| for f, pat in FIELD_LABELS.items(): | |
| rx = re.compile(rf"^{pat}$") | |
| for i, c in enumerate(cells): | |
| if rx.match(c): | |
| v = cells[i + 1] if i + 1 < len(cells) else "" | |
| res[f] = None if v in _EMPTY or rx.match(v) else _norm(f, v) | |
| break | |
| m = _SERIES.search(form_text(text)); res["series"] = str(int(m.group(1))) if m else None | |
| return res | |
| def mask(t: str) -> str: | |
| return _RCEPT.sub("[RCEPT]", _FIRST.sub("μ΅μ΄μ μΆμΌ : [MASK]", t)) | |
| def summary_line(f: dict) -> str: | |
| return "[μμ½] " + " | ".join(f"{lab} {f.get(k)}" for k, lab in _SUM_F if f.get(k) is not None) | |
| def query_card(corp_name: str, text: str) -> str: | |
| body = mask(f"νμ¬: {corp_name}\n[μ μ μ¬ν]\n{correction_section(text)[:CARD_CORR_CHARS]}\n[μμ]\n{form_text(text)[:CARD_FORM_CHARS]}") | |
| return summary_line(extract_fields(text)) + "\n" + body | |
| def candidate_card(corp_name: str, rcept_dt: str, text: str) -> str: | |
| body = mask(f"νμ¬: {corp_name}\nμ μμΌ: {rcept_dt}\n{form_text(text)[:CARD_FORM_CHARS]}") | |
| return summary_line(extract_fields(text)) + "\n" + body | |
| # ββ scoring ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def load(model_dir: str = "."): | |
| import torch | |
| from transformers import AutoModelForSequenceClassification, AutoTokenizer | |
| tok = AutoTokenizer.from_pretrained(model_dir) | |
| m = AutoModelForSequenceClassification.from_pretrained(model_dir, torch_dtype=torch.float32).eval() | |
| return tok, m | |
| def score(tok, m, q_card: str, c_cards: list[str], batch: int = 8) -> list[float]: | |
| import torch | |
| out: list[float] = [] | |
| with torch.no_grad(): | |
| for i in range(0, len(c_cards), batch): | |
| b = c_cards[i:i + batch] | |
| x = tok([q_card] * len(b), b, truncation="longest_first", max_length=MAX_LEN, padding=True, return_tensors="pt") | |
| x = {k: v.to(m.device) for k, v in x.items()} | |
| out += m(**x).logits.view(-1).float().tolist() | |
| return out | |
| def link(tok, m, q_card: str, candidates: list[dict], thr: float = NONE_THRESHOLD) -> dict: | |
| """candidates: [{"id": ..., "card": ...}], ordered by receipt number descending (ties -> first wins).""" | |
| s = score(tok, m, q_card, [c["card"] for c in candidates]) | |
| best, bs = None, -math.inf | |
| for c, v in zip(candidates, s): | |
| if v > bs: best, bs = c["id"], v | |
| return {"pred": best if bs >= thr else None, "top_score": bs, "threshold": thr, | |
| "scores": {c["id"]: v for c, v in zip(candidates, s)}} | |
| def run_example(model_dir: str, path: str) -> dict: | |
| ex = json.loads(Path(path).read_text(encoding="utf-8")) | |
| q = query_card(ex["query"]["corp_name"], ex["query"]["text"]) | |
| cands = sorted(ex["candidates"], key=lambda c: c["id"], reverse=True) | |
| cc = [{"id": c["id"], "card": candidate_card(c["corp_name"], c["rcept_dt"], c["text"])} for c in cands] | |
| tok, m = load(model_dir) | |
| r = link(tok, m, q, cc) | |
| r["expected"] = ex.get("expected"); r["query_fields"] = extract_fields(ex["query"]["text"]) | |
| return r | |
| def main() -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--model", default=str(Path(__file__).resolve().parent)) | |
| ap.add_argument("--example", default=str(Path(__file__).resolve().parent / "examples" / "synthetic_query.json")) | |
| ap.add_argument("--selftest", action="store_true", help="decoder + card checks without the model") | |
| a = ap.parse_args() | |
| if a.selftest: | |
| raw = "<?xml version=\"1.0\" encoding=\"euc-kr\"?><P>νμ°¨</P>".encode("cp949") | |
| buf = io.BytesIO(); zipfile.ZipFile(buf, "w").writestr("d.xml", raw) | |
| assert "νμ°¨" in dart_zip_to_text(buf.getvalue()), "cp949 decode failed" | |
| ex = json.loads(Path(a.example).read_text(encoding="utf-8")) | |
| q = query_card(ex["query"]["corp_name"], ex["query"]["text"]) | |
| assert q.startswith("[μμ½] ") and not re.search(r"(?<!\d)\d{14}(?!\d)", q), "card format" | |
| print(json.dumps({"selftest": "ok", "query_fields": extract_fields(ex["query"]["text"])}, ensure_ascii=False)) | |
| return 0 | |
| print(json.dumps(run_example(a.model, a.example), ensure_ascii=False, indent=1)) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |