| """Benchmark images for Hindko Embeddings (model card), in the style of the Hindko Tokenizer charts. |
| |
| Every number is read from the files in eval/ (results.csv, selection.json, training.json, full_metrics.json and |
| training_loss.csv); nothing is typed in by hand except the parameter counts of the public models (PARAMS), the dev |
| standard error of about 0.01 from the training plan (shaded band in chart 16), and a few fixed counts in subtitles |
| and notes, all matching eval/: task sizes in charts 00-06 (HindkoEmbedBench v1.0 description), 14, 15 and 20; |
| run counts, steps, epochs, batch and block sizes, GPU and seed in charts 15-18 and 21, and the max_seq_length and |
| precision settings (512 or 256 tokens, float16 or float32) named in subtitles and legends. |
| Writes benchmarks/<nn>_<name>.png (3200 x 1800): HTML pages rendered by headless Chrome at 2x, with the Google |
| Fonts embedded so they always load. |
| |
| python make_charts.py # all charts |
| python make_charts.py 00,03 # only these |
| python make_charts.py --keep-html |
| """ |
| import base64 |
| import csv |
| import json |
| import math |
| import os |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import urllib.request |
|
|
| sys.dont_write_bytecode = True |
| HERE = os.path.dirname(os.path.abspath(__file__)) |
| EVAL = os.path.join(HERE, 'eval') |
| OUT = os.path.join(HERE, 'benchmarks') |
| WORK = os.path.join(HERE, '.render') |
| MAC_CHROME = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' |
| CHROME = (os.environ.get('CHROME') or shutil.which('chrome') or shutil.which('google-chrome') or shutil.which('google-chrome-stable') |
| or shutil.which('chromium') or shutil.which('chromium-browser') or (MAC_CHROME if os.path.exists(MAC_CHROME) else None) |
| or os.path.join(os.environ.get('PROGRAMFILES', ''), 'Google', 'Chrome', 'Application', 'chrome.exe')) |
| UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0 Safari/537.36'} |
| W, H = 1600, 900 |
|
|
| |
| R = {} |
| for r in csv.DictReader(open(os.path.join(EVAL, 'results.csv'), encoding='utf-8')): |
| R[(r['model_id'], r['split'])] = r |
| SEL = json.load(open(os.path.join(EVAL, 'selection.json'), encoding='utf-8')) |
|
|
| OURS = SEL['selected'] |
| RUNNER = SEL['runner_up'] |
| TFIDF = 'tfidf-char2-4' |
| R1 = 'r1-ft-e5-base' |
| R1S = 'r1-ft-e5-small' |
| BGE = 'BAAI/bge-m3' |
| E5L = 'intfloat/multilingual-e5-large' |
| POOL_ID = {'ft-e5-base': R1, 'ft-e5-small': R1S} |
| TASKS = ['HeadlineRetrieval', 'BookTitleRetrieval', 'PassageContinuation', 'VerseProseClassification', 'BookClustering'] |
| TASK_LABEL = {'overall': ('Overall', 'mean of 5'), 'HeadlineRetrieval': ('Headline', 'nDCG@10'), |
| 'BookTitleRetrieval': ('Book title', 'nDCG@10'), 'PassageContinuation': ('Continuation', 'nDCG@10'), |
| 'VerseProseClassification': ('Verse / prose', 'macro-F1'), 'BookClustering': ('Book clustering', 'V-measure')} |
| |
| SIZES = {'dev': '85 headline, 262 book-title and 416 continuation queries; 297 verse/prose and 355 clustering items', |
| 'test': '80 headline, 28 book-title and 360 continuation queries; 157 verse/prose and 176 clustering items'} |
| |
| |
| PARAMS = {'intfloat/multilingual-e5-small': 118, 'intfloat/multilingual-e5-base': 278, 'intfloat/multilingual-e5-large': 560, |
| 'BAAI/bge-m3': 568, 'sentence-transformers/LaBSE': 471, 'Qwen/Qwen3-Embedding-0.6B': 596, |
| 'sentence-transformers/paraphrase-multilingual-mpnet-base-v2': 278} |
|
|
| PUBLIC = [(m, s) for (m, s), r in R.items() if r['kind'] == 'public' and s == 'test'] |
|
|
|
|
| def v(mid, split, col='overall'): |
| return float(R[(mid, split)][col]) |
|
|
|
|
| def f3(x): |
| return '%.3f' % x |
|
|
|
|
| def f4(x): |
| return '%.4f' % x |
|
|
|
|
| def short(mid): |
| return mid.split('/')[-1] |
|
|
|
|
| |
| CSS = """ |
| :root{--ink:#0F172A;--muted:#5B6576;--grid:#E4E8EE;--accent:#2F6FDB;--accent-ink:#1D4FA8;--bar:#C9D0DA;--soft:#EAF1FC} |
| *{box-sizing:border-box} |
| html,body{margin:0;width:%(W)dpx;height:%(H)dpx;overflow:hidden;background:#FFFFFF;color:var(--ink);font-family:'IBM Plex Sans',sans-serif} |
| .page{position:absolute;inset:0;padding:56px 80px 0} |
| .eyebrow{font-size:17px;font-weight:600;letter-spacing:.16em;text-transform:uppercase;color:var(--accent)} |
| h1{font-family:'Source Serif 4',serif;font-weight:600;font-size:54px;line-height:1.08;letter-spacing:-.015em;margin:12px 0 10px} |
| .sub{font-size:23px;line-height:1.4;color:var(--muted);max-width:1400px;margin:0} |
| .foot{position:absolute;left:80px;right:80px;bottom:34px;display:flex;justify-content:space-between;gap:40px;font-size:16px;color:var(--muted);border-top:1px solid var(--grid);padding-top:14px} |
| .foot b{color:var(--ink);font-weight:600} |
| .bars{margin-top:30px} |
| .row{display:grid;grid-template-columns:380px 1fr 120px;align-items:center;column-gap:22px;height:%(rowh)dpx} |
| .lab{font-size:21px;line-height:1.15;text-align:right} |
| .lab small{display:block;font-size:15px;color:var(--muted)} |
| .trk{position:relative;height:%(barh)dpx} |
| .fill{position:absolute;left:0;top:0;bottom:0;background:var(--bar);border-radius:0 4px 4px 0} |
| .val{font-size:21px;font-variant-numeric:tabular-nums;color:var(--ink)} |
| .row.me .lab{font-weight:700}.row.me .fill{background:var(--accent)}.row.me .val{font-weight:700;color:var(--accent-ink)} |
| .row.ref .fill{background:#8E9AAB} |
| .gl{position:absolute;top:-8px;bottom:-8px;border-left:1px dashed var(--grid)} |
| table{border-collapse:collapse;width:100%%;margin-top:30px;font-variant-numeric:tabular-nums} |
| th{font-size:15px;font-weight:600;letter-spacing:.06em;text-transform:uppercase;color:var(--muted);text-align:right;padding:0 18px 12px;border-bottom:2px solid var(--ink);vertical-align:bottom} |
| th small{display:block;font-size:13px;letter-spacing:.02em;text-transform:none;font-weight:400} |
| th:first-child,td:first-child{text-align:left;padding-left:0} |
| td{font-size:22px;text-align:right;padding:13px 18px;border-bottom:1px solid var(--grid)} |
| td small{font-size:15px;color:var(--muted);margin-left:8px;font-weight:400} |
| td .best{border-bottom:3px solid currentColor;padding-bottom:1px} |
| tr.me td{font-weight:700;color:var(--accent-ink);background:var(--soft)} |
| tr.me td:first-child{padding-left:14px;border-radius:6px 0 0 6px} |
| .legend{display:flex;gap:34px;font-size:19px;margin-top:26px} |
| .legend span{display:inline-flex;align-items:center;gap:10px} |
| .legend i{display:inline-block;width:22px;height:16px;border-radius:3px} |
| .note{font-size:19px;line-height:1.45;color:var(--muted);max-width:1440px;margin-top:18px} |
| """ |
|
|
| HEAD = ('<!doctype html><html><head><meta charset="utf-8">' |
| '<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Sans:wght@400;600;700' |
| '&family=Source+Serif+4:opsz,wght@8..60,600&display=block">' |
| '<style>%s</style></head><body><div class="page">') |
|
|
|
|
| CSS2 = """ |
| .hm{margin-top:20px} |
| .hm th{padding:0 6px 10px;text-align:center;font-size:14px} |
| .hm th:first-child{text-align:left} |
| .hm td{padding:0 14px;border-bottom:2px solid #FFFFFF;text-align:center} |
| .hm td.nm{text-align:left;line-height:1.05;background:#FFFFFF;padding-left:14px} |
| .hm td.nm small{display:block;margin:1px 0 0;font-size:13px;line-height:1.15;font-weight:400;color:#5B6576} |
| .hm tr.me td.nm{box-shadow:inset 6px 0 0 #2F6FDB;padding-left:20px;color:#1D4FA8;background:#EAF1FC} |
| .ptitle{font-size:21px;font-weight:600;margin-top:10px} |
| .ptitle small{font-size:16px;font-weight:400;color:#5B6576;margin-left:8px} |
| .legend svg{display:inline-block} |
| .lm span{white-space:nowrap} |
| .lm svg{flex-shrink:0} |
| """ |
|
|
|
|
| def page(title, sub, body, foot_left, rowh=52, barh=30, eyebrow='Benchmarks'): |
| return (HEAD % (CSS % dict(W=W, H=H, rowh=rowh, barh=barh) + CSS2) + |
| '<div class="eyebrow">Hindko Embeddings · %s</div><h1>%s</h1><p class="sub">%s</p>%s' |
| '<div class="foot"><span>%s</span><span style="white-space:nowrap">huggingface.co/junaid008/hindko-embeddings</span></div></div></body></html>' |
| % (eyebrow, title, sub, body, foot_left)) |
|
|
|
|
| def foot(split, extra=''): |
| name = 'Test split, scored once after selection on dev' if split == 'test' else 'Dev (validation) split' |
| return '<b>HindkoEmbedBench v1.0 · %s</b>%s' % (name, extra) |
|
|
|
|
| def bars(items, vmax, grid=()): |
| """items: (label, sublabel, value, value_html, cls) with cls '', 'me' or 'ref'.""" |
| out = ['<div class="bars">'] |
| for lab, sublab, val, vh, cls in items: |
| gl = ''.join('<span class="gl" style="left:%.3f%%"></span>' % (100 * g / vmax) for g in grid) |
| out.append('<div class="row %s"><div class="lab">%s%s</div><div class="trk">%s<div class="fill" style="width:%.3f%%"></div></div>' |
| '<div class="val">%s</div></div>' % (cls, lab, '<small>%s</small>' % sublab if sublab else '', gl, 100 * val / vmax, vh)) |
| out.append('</div>') |
| return ''.join(out) |
|
|
|
|
| def legend(series): |
| return '<div class="legend">%s</div>' % ''.join('<span><i style="background:%s"></i>%s</span>' % (c, n) for n, c in series) |
|
|
|
|
| def grouped(groups, series, vmax=1.0, width=1440, height=470, ticks=(0.2, 0.4, 0.6, 0.8, 1.0), x0=60, label_fmt=f3, |
| bar_w=None, gap=10, lab_size=16, below=None, tick_fmt='%.1f', bottom=70): |
| """Vertical grouped columns. groups: [(title, subtitle)]; series: [(name, color, [values], bold)].""" |
| y0, y1 = 22, height - bottom |
| plot_w = width - x0 - 10 |
| gw = plot_w / len(groups) |
| n = len(series) |
| bw = bar_w or min(64, (gw - 50 - gap * (n - 1)) / n) |
| svg = ['<svg viewBox="0 0 %d %d" width="%d" height="%d" style="display:block;margin-top:14px;overflow:visible">' % (width, height, width, height)] |
| Y = lambda val: y1 - (y1 - y0) * val / vmax |
| for t in ticks: |
| svg.append('<line x1="%d" x2="%d" y1="%.1f" y2="%.1f" stroke="#E4E8EE"/><text x="%d" y="%.1f" font-size="15" fill="#5B6576" text-anchor="end">%s</text>' |
| % (x0, width - 10, Y(t), Y(t), x0 - 10, Y(t) + 5, tick_fmt % t)) |
| svg.append('<line x1="%d" x2="%d" y1="%d" y2="%d" stroke="#0F172A" stroke-width="1.5"/>' % (x0, width - 10, y1, y1)) |
| for gi, (gt, gs) in enumerate(groups): |
| gx = x0 + gi * gw |
| total = n * bw + (n - 1) * gap |
| bx0 = gx + (gw - total) / 2 |
| for si, (name, col, vals, bold) in enumerate(series): |
| val = vals[gi] |
| if val is None: |
| continue |
| x = bx0 + si * (bw + gap) |
| svg.append('<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" fill="%s" rx="3"/>' % (x, Y(val), bw, y1 - Y(val), col)) |
| svg.append('<text x="%.1f" y="%.1f" font-size="%d" font-weight="%d" fill="%s" text-anchor="middle">%s</text>' |
| % (x + bw / 2, Y(val) - 8, lab_size, 700 if bold else 400, '#1D4FA8' if bold else '#0F172A', label_fmt(val))) |
| svg.append('<text x="%.1f" y="%d" font-size="19" font-weight="600" fill="#0F172A" text-anchor="middle">%s</text>' % (gx + gw / 2, y1 + 28, gt)) |
| svg.append('<text x="%.1f" y="%d" font-size="15" fill="#5B6576" text-anchor="middle">%s</text>' % (gx + gw / 2, y1 + 50, gs)) |
| if below: |
| svg.append(below(gi, gx, gw, y1)) |
| svg.append('</svg>') |
| return ''.join(svg) |
|
|
|
|
| |
| def c00(): |
| rows = [(OURS, 'Hindko Embeddings', 'this model · 512 tokens'), |
| (TFIDF, 'TF-IDF char 2-4-gram', 'lexical reference'), |
| (RUNNER, 'Round-2 e5-base fine-tune', 'runner-up on dev · 512 tokens'), |
| (R1, 'Round-1 model', 'our e5-base fine-tune · 256 tokens'), |
| (R1S, 'Round-1 e5-small fine-tune', '256 tokens'), |
| (BGE, 'bge-m3', 'best public model · 512 tokens'), |
| (E5L, 'multilingual-e5-large', 'base model, untuned · 512 tokens')] |
| rows.sort(key=lambda r: -v(r[0], 'test')) |
| cols = ['overall'] + TASKS |
| best = {c: max(v(m, 'test', c) for m, _, _ in rows) for c in cols} |
| head = ''.join('<th>%s<small>%s</small></th>' % TASK_LABEL[c] for c in cols) |
| trs = [] |
| for m, name, sub in rows: |
| tds = ''.join('<td>%s</td>' % (('<span class="best">%s</span>' if v(m, 'test', c) == best[c] else '%s') % f3(v(m, 'test', c))) for c in cols) |
| trs.append('<tr%s><td>%s<small>%s</small></td>%s</tr>' % (' class="me"' if m == OURS else '', name, sub, tds)) |
| body = '<table><tr><th>Model</th>%s</tr>%s</table>' % (head, ''.join(trs)) |
| d_tf = v(OURS, 'test') - v(TFIDF, 'test') |
| d_bge = v(OURS, 'test') - v(BGE, 'test') |
| body += ('<p class="note">Overall: <b style="color:#0F172A">+%.3f</b> over TF-IDF and <b style="color:#0F172A">+%.3f</b> over bge-m3. ' |
| 'Ahead of TF-IDF on four of the five tasks; behind it, and behind bge-m3, on book-title retrieval (28 test queries).</p>' % (d_tf, d_bge)) |
| return page('Hindko Embeddings on the held-out test set', |
| 'Five automatic Hindko tasks; higher is better. The best value in each column is underlined.', body, |
| foot('test', ' · neural models at max_seq_length 512 unless noted')) |
|
|
|
|
| |
| def c01(): |
| items = [('Hindko Embeddings', 'this model · e5-large fine-tune', v(OURS, 'test'), f4(v(OURS, 'test')), 'me'), |
| ('Round-1 e5-small fine-tune', 'ours, round 1 · 256 tokens', v(R1S, 'test'), f4(v(R1S, 'test')), ''), |
| ('Round-1 model', 'ours, round 1 · e5-base · 256 tokens', v(R1, 'test'), f4(v(R1, 'test')), ''), |
| ('Round-2 e5-base fine-tune', 'ours, runner-up on dev', v(RUNNER, 'test'), f4(v(RUNNER, 'test')), ''), |
| ('TF-IDF char 2-4-gram', 'lexical reference', v(TFIDF, 'test'), f4(v(TFIDF, 'test')), 'ref')] |
| for m, s in PUBLIC: |
| items.append((short(m).replace('paraphrase-', ''), 'public, untuned', v(m, 'test'), f4(v(m, 'test')), '')) |
| items.sort(key=lambda x: -x[2]) |
| return page('Overall score on the test set', |
| 'Mean of the five task scores for every model tested. Public models run untuned at 512 tokens; higher is better.', |
| bars(items, 0.8, grid=(0.2, 0.4, 0.6)) + |
| '<div style="display:grid;grid-template-columns:380px 1fr 120px;column-gap:22px;margin-top:2px"><span></span>' |
| '<div style="position:relative;height:20px;font-size:15px;color:#5B6576">%s</div><span></span></div>' |
| % ''.join('<span style="position:absolute;left:%.3f%%;transform:translateX(-50%%)">%.1f</span>' % (100 * t / 0.8, t) for t in (0.0, 0.2, 0.4, 0.6, 0.8)), |
| foot('test', ' · max_seq_length 512, except the round-1 models (256) and TF-IDF'), rowh=47, barh=26) |
|
|
|
|
| |
| def c02(): |
| cols = ['overall'] + TASKS |
| series = [('Hindko Embeddings (this model)', '#2F6FDB', [v(OURS, 'test', c) for c in cols], True), |
| ('TF-IDF char 2-4-gram', '#8E9AAB', [v(TFIDF, 'test', c) for c in cols], False), |
| ('bge-m3, best public model', '#C9D0DA', [v(BGE, 'test', c) for c in cols], False)] |
| body = legend([(n, c) for n, c, _, _ in series]) + grouped([TASK_LABEL[c] for c in cols], series, height=500) |
| return page('Per-task scores on the test set', |
| 'Ahead of TF-IDF on four of five tasks. On book-title retrieval (only 28 test queries) TF-IDF and bge-m3 score higher.', |
| body, foot('test', ' · neural models at max_seq_length 512')) |
|
|
|
|
| |
| def c03(): |
| pool = sorted(SEL['dev_scores'].items(), key=lambda kv: -kv[1]) |
| best = pool[0][1] |
| lo, hi = 0.660, 0.720 |
| X = lambda val: 100 * (val - lo) / (hi - lo) |
| sub = {'r2-e5l': 'e5-large · 560M · 512 tokens', 'r2-e5b': 'e5-base · 278M · 512 tokens', |
| 'r2-e5b-plain': 'e5-base, no mined negatives or book blocks', 'r2-e5s': 'e5-small · 118M · 512 tokens', |
| 'r2-bgem3': 'bge-m3 · 568M · 512 tokens', 'ft-e5-base': 'round-1 model · e5-base · 256 tokens', |
| 'ft-e5-small': 'round-1 · e5-small · 256 tokens'} |
| tag = {SEL['selected']: 'selected', SEL['runner_up']: 'runner-up', 'ft-e5-base': 'incumbent', 'ft-e5-small': 'incumbent'} |
| rows = [] |
| for name, val in pool: |
| me = name == SEL['selected'] |
| dot = ('<div style="position:absolute;top:50%%;left:0;width:%.3f%%;height:2px;margin-top:-1px;background:%s"></div>' |
| '<div style="position:absolute;top:50%%;left:%.3f%%;width:18px;height:18px;margin:-9px 0 0 -9px;border-radius:50%%;background:%s"></div>' |
| % (X(val), '#9DB9EA' if me else '#D3D9E1', X(val), '#2F6FDB' if me else '#8E9AAB')) |
| rows.append('<div class="row%s" style="grid-template-columns:330px 1fr 230px;height:64px"><div class="lab">%s<small>%s</small></div>' |
| '<div class="trk" style="height:30px">%s</div><div class="val">%s<small style="font-size:15px;color:#5B6576;margin-left:10px;font-weight:400">%s</small></div></div>' |
| % (' me' if me else '', name, sub[name], dot, f4(val), tag.get(name, ''))) |
| left = 'calc(330px + 22px + (100%% - 330px - 230px - 44px)*%.5f)' |
| band = ('<div style="position:absolute;top:0;bottom:0;left:%s;width:calc((100%% - 330px - 230px - 44px)*%.5f);background:#EAF1FC;border-left:1px solid #9DB9EA;border-right:1px solid #9DB9EA"></div>' |
| % (left % (X(best - SEL['tie']) / 100), (X(best) - X(best - SEL['tie'])) / 100)) |
| tf = v(TFIDF, 'dev') |
| ref = ('<div style="position:absolute;top:0;bottom:0;left:%s;border-left:2px dashed #8E9AAB"></div>' % (left % (X(tf) / 100))) |
| ticks = ''.join('<span style="position:absolute;left:%s;transform:translateX(-50%%);font-size:15px;color:#5B6576">%.2f</span>' % (left % (X(t) / 100), t) |
| for t in (0.66, 0.67, 0.68, 0.69, 0.70, 0.71, 0.72)) |
| body = ('<div style="position:relative;margin-top:26px">%s%s%s</div><div style="position:relative;height:22px;margin-top:4px">%s</div>' |
| '<p class="note" style="margin-top:14px">Shaded: the pre-registered tie band (within %.3f of the best). Only the best model falls in it, so it was selected; ' |
| 'the runner-up is %.4f below. Dashed line: TF-IDF on dev, %s. Axis starts at 0.66.</p>' |
| % (band, ref, ''.join(rows), ticks, SEL['tie'], best - SEL['dev_scores'][SEL['runner_up']], f4(tf))) |
| return page('Selected on dev by a fixed rule', |
| 'Overall dev score of every model in the selection pool. Rule: highest dev score; within %.3f of it, the smaller model wins.' % SEL['tie'], |
| body, foot('dev', ' · round-2 models at 512 tokens in float16; incumbents at their round-1 setting')) |
|
|
|
|
| |
| def c04(): |
| cols = ['overall'] + TASKS |
| a = [v(R1, 'test', c) for c in cols] |
| b = [v(OURS, 'test', c) for c in cols] |
| series = [('Round-1 model (e5-base fine-tune, 256 tokens)', '#9DB9EA', a, False), |
| ('Hindko Embeddings, round 2 (e5-large fine-tune, 512 tokens)', '#2F6FDB', b, True)] |
|
|
| def below(gi, gx, gw, y1): |
| d = b[gi] - a[gi] |
| return ('<text x="%.1f" y="%d" font-size="17" font-weight="700" fill="%s" text-anchor="middle">%s</text>' |
| % (gx + gw / 2, y1 + 76, '#1D4FA8' if d >= 0 else '#B4442C', ('%+.3f' % d).replace('-', '−'))) |
| body = legend([(n, c) for n, c, _, _ in series]) + grouped([TASK_LABEL[c] for c in cols], series, height=500, below=below) |
| return page('Round 2 vs round 1', |
| 'Test scores, with the change below each group. Round 2 changed the base (e5-base to e5-large), the data (new short-query pairs, mined hard negatives, same-book batches) and the length (256 to 512 tokens).', |
| body, foot('test')) |
|
|
|
|
| |
| def c05(): |
| width, height = 1440, 520 |
| x0, x1, y0, y1 = 80, 1280, 20, height - 60 |
| xmax, ylo, yhi = 650, 0.40, 0.75 |
| X = lambda p: x0 + (x1 - x0) * p / xmax |
| Y = lambda s: y1 - (y1 - y0) * (s - ylo) / (yhi - ylo) |
| svg = ['<svg viewBox="0 0 %d %d" width="%d" height="%d" style="display:block;margin-top:26px;overflow:visible">' % (width, height, width, height)] |
| for t in (0.40, 0.45, 0.50, 0.55, 0.60, 0.65, 0.70, 0.75): |
| svg.append('<line x1="%d" x2="%d" y1="%.1f" y2="%.1f" stroke="#E4E8EE"/><text x="%d" y="%.1f" font-size="15" fill="#5B6576" text-anchor="end">%.2f</text>' % (x0, x1, Y(t), Y(t), x0 - 10, Y(t) + 5, t)) |
| for p in (0, 100, 200, 300, 400, 500, 600): |
| svg.append('<text x="%.1f" y="%d" font-size="15" fill="#5B6576" text-anchor="middle">%d</text>' % (X(p), y1 + 24, p)) |
| svg.append('<text x="%.1f" y="%d" font-size="16" fill="#5B6576" text-anchor="middle">parameters (millions)</text>' % ((x0 + x1) / 2, y1 + 50)) |
| svg.append('<text x="18" y="%d" font-size="16" fill="#5B6576" transform="rotate(-90 18 %d)" text-anchor="middle">dev overall score</text>' % ((y0 + y1) // 2, (y0 + y1) // 2)) |
| tf = v(TFIDF, 'dev') |
| svg.append('<line x1="%d" x2="%d" y1="%.1f" y2="%.1f" stroke="#8E9AAB" stroke-width="2" stroke-dasharray="7 6"/>' % (x0, x1, Y(tf), Y(tf))) |
| svg.append('<text x="%d" y="%.1f" font-size="16" fill="#5B6576" text-anchor="start">TF-IDF %s</text>' % (x0 + 8, Y(tf) - 8, f4(tf))) |
| e5 = ['intfloat/multilingual-e5-small', 'intfloat/multilingual-e5-base', 'intfloat/multilingual-e5-large'] |
| ft = {'intfloat/multilingual-e5-small': 'r2-e5s', 'intfloat/multilingual-e5-base': 'r2-e5b', 'intfloat/multilingual-e5-large': OURS, BGE: 'r2-bgem3'} |
| for ids, col, w in (([ft[m] for m in e5], '#9DB9EA', 3), (e5, '#C9D0DA', 3)): |
| pts = ' '.join('%.1f,%.1f' % (X(PARAMS[m if m in PARAMS else [k for k, x in ft.items() if x == m][0]]), Y(v(m, 'dev'))) for m in ids) |
| svg.append('<polyline points="%s" fill="none" stroke="%s" stroke-width="%d"/>' % (pts, col, w)) |
| for base, run in ft.items(): |
| svg.append('<line x1="%.1f" x2="%.1f" y1="%.1f" y2="%.1f" stroke="#C9D0DA" stroke-width="1.5" stroke-dasharray="3 4"/>' |
| % (X(PARAMS[base]), X(PARAMS[base]), Y(v(base, 'dev')), Y(v(run, 'dev')))) |
| |
| pts = [(OURS, 'e5-large fine-tune (this model)', -14, -14, 'end', 'me'), ('r2-bgem3', 'bge-m3 fine-tune', 14, 26, 'start', 'ft'), |
| ('r2-e5b', 'e5-base fine-tune', 0, -16, 'middle', 'ft'), ('r2-e5s', 'e5-small fine-tune', 0, -16, 'middle', 'ft'), |
| (E5L, 'e5-large', -14, -12, 'end', 'pub'), (BGE, 'bge-m3', 14, -10, 'start', 'pub'), |
| ('intfloat/multilingual-e5-base', 'e5-base', 14, -10, 'start', 'pub'), ('intfloat/multilingual-e5-small', 'e5-small', 0, 28, 'middle', 'pub'), |
| ('sentence-transformers/LaBSE', 'LaBSE', -14, 6, 'end', 'pub'), ('Qwen/Qwen3-Embedding-0.6B', 'Qwen3-Embedding-0.6B', 14, 6, 'start', 'pub'), |
| ('sentence-transformers/paraphrase-multilingual-mpnet-base-v2', 'mpnet-base-v2', 14, 6, 'start', 'pub')] |
| inv = {r: b for b, r in ft.items()} |
| for mid, lab, dx, dy, anc, kind in pts: |
| p = PARAMS[mid] if mid in PARAMS else PARAMS[inv[mid]] |
| s = v(mid, 'dev') |
| col = {'me': '#2F6FDB', 'ft': '#6F95DE', 'pub': '#8E9AAB'}[kind] |
| r = 11 if kind == 'me' else 8 |
| svg.append('<circle cx="%.1f" cy="%.1f" r="%d" fill="%s" stroke="#FFFFFF" stroke-width="2"/>' % (X(p), Y(s), r, col)) |
| svg.append('<text x="%.1f" y="%.1f" font-size="%d" font-weight="%d" fill="%s" text-anchor="%s" stroke="#FFFFFF" stroke-width="5" paint-order="stroke">%s ' |
| '<tspan font-weight="400" fill="#5B6576">%s</tspan></text>' |
| % (X(p) + dx, Y(s) + dy, 18 if kind == 'me' else 16, 700 if kind == 'me' else 600, '#1D4FA8' if kind == 'me' else '#0F172A', anc, lab, f4(s))) |
| svg.append('</svg>') |
| body = (legend([('fine-tuned on Hindko (round 2)', '#6F95DE'), ('public model, untuned', '#8E9AAB'), ('TF-IDF reference', '#8E9AAB')]).replace( |
| '<i style="background:#8E9AAB"></i>TF-IDF', '<i style="background:none;border-top:2px dashed #8E9AAB;height:0;width:28px"></i>TF-IDF') + ''.join(svg)) |
| return page('Fine-tuning matters more than size', |
| 'Dev score against model size. Every fine-tuned model scores above every untuned one; among the fine-tuned e5 models the larger ones score slightly higher.', |
| body, foot('dev', ' · neural models at max_seq_length 512 in float16; TF-IDF is lexical')) |
|
|
|
|
| |
| def c06(): |
| mets = [('BT_R1', 'Recall@1'), ('BT_R10', 'Recall@10'), ('BT_MRR', 'MRR@10'), ('BookTitleRetrieval', 'nDCG@10')] |
| panels = [] |
| for split, title in (('dev', 'Dev: 262 queries'), ('test', 'Test: 28 queries from 5 books')): |
| series = [('this model', '#2F6FDB', [v(OURS, split, c) for c, _ in mets], True), |
| ('TF-IDF', '#8E9AAB', [v(TFIDF, split, c) for c, _ in mets], False), |
| ('bge-m3', '#C9D0DA', [v(BGE, split, c) for c, _ in mets], False)] |
| svg = grouped([(lab, '') for _, lab in mets], series, vmax=0.8, width=700, height=430, ticks=(0.2, 0.4, 0.6, 0.8), x0=50, |
| bar_w=44, gap=6, lab_size=15) |
| panels.append('<div><div style="font-size:22px;font-weight:600;margin-top:6px">%s</div>%s</div>' % (title, svg)) |
| body = (legend([('Hindko Embeddings (this model)', '#2F6FDB'), ('TF-IDF char 2-4-gram', '#8E9AAB'), ('bge-m3 (untuned, 512 tokens)', '#C9D0DA')]) + |
| '<div style="display:grid;grid-template-columns:1fr 1fr;gap:40px">%s</div>' % ''.join(panels) + |
| '<p class="note" style="margin-top:0">The test set has only 28 queries (57 relevant passages), so its numbers are noisy; the dev gap to TF-IDF points the same way.</p>') |
| return page('Weak spot: finding a passage from its book title', |
| 'BookTitleRetrieval: a table-of-contents title (often the first line of a poem) must retrieve its passage. TF-IDF beats this model on nDCG@10 on both dev and test.', |
| body, foot('test', '; dev panel on the dev split · neural models at max_seq_length 512')) |
|
|
|
|
| |
| |
| |
| FM = json.load(open(os.path.join(EVAL, 'full_metrics.json'), encoding='utf-8')) |
| M, M256 = {}, {} |
| for _key, _res in FM['results'].items(): |
| (M256 if _key.startswith('baseline256/') else M)[(_res['model_id'], _res['split'])] = _res |
| TRAIN = FM['train'] |
| BENCH = FM['benchmark'] |
| TDATA = FM['train_data'] |
| TRJ = json.load(open(os.path.join(EVAL, 'training.json'), encoding='utf-8')) |
| LOSS = {} |
| for _r in csv.DictReader(open(os.path.join(EVAL, 'training_loss.csv'), encoding='utf-8')): |
| LOSS.setdefault(_r['model'], []).append((int(_r['step']), _r['dataset_type'], float(_r['loss']))) |
|
|
| R2B, R2S, R2G, R2P = 'r2-e5b', 'r2-e5s', 'r2-bgem3', 'r2-e5b-plain' |
| E5B, E5S = 'intfloat/multilingual-e5-base', 'intfloat/multilingual-e5-small' |
| QWEN, LABSE = 'Qwen/Qwen3-Embedding-0.6B', 'sentence-transformers/LaBSE' |
| MPNET = 'sentence-transformers/paraphrase-multilingual-mpnet-base-v2' |
| PUBS = [BGE, E5L, E5B, E5S, QWEN, LABSE, MPNET] |
| CANDS = [OURS, R2B, R2P, R2S, R2G] |
| COLS = ['overall'] + TASKS |
| NAME = {OURS: 'Hindko Embeddings (this model)', R2B: 'Round-2 e5-base', R2S: 'Round-2 e5-small', R2G: 'Round-2 bge-m3', |
| R2P: 'Round-2 e5-base, no hard negatives', R1: 'Round-1 e5-base fine-tune', R1S: 'Round-1 e5-small fine-tune', |
| TFIDF: 'TF-IDF char 2-4-gram'} |
| SHORT = {OURS: 'This model', R2B: 'Round-2 e5-base', R2S: 'Round-2 e5-small', R2G: 'Round-2 bge-m3', |
| R2P: 'Round-2 e5-base, no hard neg.', R1: 'Round-1 e5-base', R1S: 'Round-1 e5-small', TFIDF: 'TF-IDF', |
| BGE: 'bge-m3', E5L: 'e5-large (untuned)', E5B: 'e5-base (untuned)', E5S: 'e5-small (untuned)', |
| QWEN: 'Qwen3-Embedding-0.6B', LABSE: 'LaBSE', MPNET: 'mpnet-base-v2'} |
| |
| COL = {OURS: '#2F6FDB', R2B: '#EB6834', R2S: '#1BAF7A', R2G: '#EDA100', R2P: '#E87BA4', R1: '#9DB9EA', R1S: '#8FD6BC', |
| TFIDF: '#8E9AAB', BGE: '#C9D0DA', E5L: '#4A5568'} |
| PUBCOL = '#E1E5EB' |
| TYPES = ['title', 'verse_ict', 'prose_q', 'continuation', 'neighbour'] |
| TYPECOL = {'title': '#6250D6', 'verse_ict': '#E34948', 'prose_q': '#B07AA1', 'continuation': '#4E9FA5', 'neighbour': '#008300'} |
| INK, MUTED, GRID, POS, NEG = '#0F172A', '#5B6576', '#E4E8EE', '#1D4FA8', '#B4442C' |
|
|
|
|
| def col(mid): |
| return COL.get(mid, PUBCOL) |
|
|
|
|
| def nm(mid): |
| return NAME.get(mid, mid) |
|
|
|
|
| def sc(mid, split, task='overall', metric='main', src=None, tab=None): |
| r = (M if tab is None else tab)[(mid, split)] |
| if task == 'overall': |
| return r['overall'] |
| t = r['tasks'][task] |
| if src: |
| return t['by_source'][src]['ndcg_at_10' if metric == 'main' else metric] |
| return t[metric] |
|
|
|
|
| def dlt(d, nd=4): |
| return ('%+.*f' % (nd, d)).replace('-', '−') |
|
|
|
|
| def thou(n): |
| return '{:,}'.format(n) |
|
|
|
|
| def mix(a, b, t): |
| a, b = [int(a[i:i + 2], 16) for i in (1, 3, 5)], [int(b[i:i + 2], 16) for i in (1, 3, 5)] |
| return '#%02X%02X%02X' % tuple(round(x + (y - x) * t) for x, y in zip(a, b)) |
|
|
|
|
| def tx(x, y, s, size=16, anchor='start', weight=400, fill=INK, halo=False, extra=''): |
| h = ' stroke="#FFFFFF" stroke-width="5" stroke-linejoin="round" paint-order="stroke"' if halo else '' |
| return ('<text x="%.1f" y="%.1f" font-size="%s" font-weight="%d" fill="%s" text-anchor="%s"%s%s>%s</text>' |
| % (x, y, size, weight, fill, anchor, h, extra, s)) |
|
|
|
|
| def ln(x1, y1, x2, y2, stroke=GRID, w=1, extra=''): |
| return '<line x1="%.1f" y1="%.1f" x2="%.1f" y2="%.1f" stroke="%s" stroke-width="%s"%s/>' % (x1, y1, x2, y2, stroke, w, extra) |
|
|
|
|
| def rect(x, y, w, h, fill, rx=3, extra=''): |
| return '<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" fill="%s" rx="%s"%s/>' % (x, y, max(w, 0), max(h, 0), fill, rx, extra) |
|
|
|
|
| def svgw(w, h, parts, mt=14): |
| return ('<svg viewBox="0 0 %d %d" width="%d" height="%d" style="display:block;margin-top:%dpx;overflow:visible">%s</svg>' |
| % (w, h, w, h, mt, ''.join(parts))) |
|
|
|
|
| def mark(shape, x, y, r, fill, stroke='#FFFFFF', sw=2): |
| a = ' fill="%s" stroke="%s" stroke-width="%s"' % (fill, stroke, sw) |
| if shape == 'circle': |
| return '<circle cx="%.1f" cy="%.1f" r="%.1f"%s/>' % (x, y, r, a) |
| if shape == 'square': |
| return '<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" rx="2"%s/>' % (x - r * .88, y - r * .88, r * 1.76, r * 1.76, a) |
| if shape == 'diamond': |
| return '<polygon points="%.1f,%.1f %.1f,%.1f %.1f,%.1f %.1f,%.1f"%s/>' % (x, y - r * 1.2, x + r * 1.2, y, x, y + r * 1.2, x - r * 1.2, y, a) |
| return '<polygon points="%.1f,%.1f %.1f,%.1f %.1f,%.1f"%s/>' % (x, y - r * 1.15, x + r * 1.1, y + r * .8, x - r * 1.1, y + r * .8, a) |
|
|
|
|
| def legend_marks(items, mt=22): |
| """items: (label, colour, shape, stroke) -> legend row with the same marker shapes as the plot.""" |
| out = [] |
| for lab, c, shape, stroke in items: |
| if shape == 'line': |
| sym = '<svg width="34" height="18"><line x1="2" y1="9" x2="32" y2="9" stroke="%s" stroke-width="4" stroke-linecap="round"/></svg>' % c |
| elif shape == 'dash': |
| sym = '<svg width="34" height="18"><line x1="2" y1="9" x2="32" y2="9" stroke="%s" stroke-width="2.5" stroke-dasharray="6 5"/></svg>' % c |
| else: |
| sym = '<svg width="24" height="24">%s</svg>' % mark(shape, 12, 12, 8, c, stroke, 1.5 if stroke != '#FFFFFF' else 0) |
| out.append('<span>%s%s</span>' % (sym, lab)) |
| return '<div class="legend lm" style="margin-top:%dpx">%s</div>' % (mt, ''.join(out)) |
|
|
|
|
| def legend2(series, size=18, gap=26): |
| return legend(series).replace('<div class="legend"', '<div class="legend lm" style="gap:%dpx;font-size:%dpx"' % (gap, size)) |
|
|
|
|
| def foot2(what, extra=''): |
| return '<b>HindkoEmbedBench v1.0 · %s</b>%s' % (what, extra) |
|
|
|
|
| def mavg(vals, w): |
| h = w // 2 |
| return [sum(vals[max(0, i - h):i + h + 1]) / len(vals[max(0, i - h):i + h + 1]) for i in range(len(vals))] |
|
|
|
|
| def spread(ys, gap, lo=-1e9, hi=1e9): |
| """Push label positions apart (keeping their order) so neighbours are at least `gap` apart.""" |
| order = sorted(range(len(ys)), key=lambda i: ys[i]) |
| out = list(ys) |
| for a, b in zip(order, order[1:]): |
| out[b] = max(out[b], out[a] + gap) |
| over = out[order[-1]] - hi |
| if over > 0: |
| for i in order: |
| out[i] -= over |
| for a, b in zip(order[::-1], order[::-1][1:]): |
| out[b] = min(out[b], out[a] - gap) |
| under = lo - out[order[0]] |
| if under > 0: |
| for i in order: |
| out[i] += under |
| return out |
|
|
|
|
| def hbar_rows(rows, x_lab, x0, x1, vmax, y, rowh, barh, fmt, lab_size=16, sub_size=14, name_w=None, val_size=16): |
| """Horizontal bars. rows: (label, sub, value, colour, bold, value_text). Returns svg parts and the next y.""" |
| parts = [] |
| for lab, sub, val, c, bold, vt in rows: |
| cy = y + rowh / 2 |
| if lab: |
| parts.append(tx(x_lab, cy + (0 if sub else 6), lab, lab_size, 'end', 700 if bold else 400, POS if bold else INK)) |
| if sub: |
| parts.append(tx(x_lab, cy + 17, sub, sub_size, 'end', fill=MUTED)) |
| w = (x1 - x0) * val / vmax |
| parts.append(rect(x0, cy - barh / 2, w, barh, c, 3)) |
| parts.append(tx(x0 + w + 8, cy + val_size * .36, vt if vt is not None else fmt(val), val_size, 'start', 700 if bold else 400, POS if bold else INK)) |
| y += rowh |
| return parts, y |
|
|
|
|
| |
| ROWSUB = {OURS: 'e5-large fine-tune · 512 tokens', R2B: 'runner-up on dev · 512', R2P: 'no mined negatives or same-book blocks · 512', |
| R2S: '512 tokens', R2G: '512 tokens', R1: 'round-1 model · 256 tokens', R1S: 'round 1 · 256 tokens', |
| TFIDF: 'lexical reference'} |
|
|
|
|
| def heat(split, mids, rowh, fs, colw=168): |
| vals = {m: [sc(m, split, c) for c in COLS] for m in mids} |
| mids = sorted(mids, key=lambda m: -vals[m][0]) |
| lo = [min(vals[m][i] for m in mids) for i in range(len(COLS))] |
| hi = [max(vals[m][i] for m in mids) for i in range(len(COLS))] |
| head = ''.join('<th style="width:%dpx">%s<small>%s</small></th>' % ((colw,) + TASK_LABEL[c]) for c in COLS) |
| trs = [] |
| for m in mids: |
| tds = [] |
| for i, val in enumerate(vals[m]): |
| t = (val - lo[i]) / (hi[i] - lo[i]) |
| st = 'background:%s;height:%dpx;font-size:%dpx;color:#0F172A' % (mix('#F4F6F9', '#7EA5E8', t), rowh, fs) |
| if val == hi[i]: |
| st += ';box-shadow:inset 0 0 0 3px #0F172A;font-weight:700' |
| tds.append('<td style="%s">%s</td>' % (st, f4(val))) |
| if m in NAME: |
| name, sub = NAME[m], ROWSUB[m] |
| else: |
| org, short_id = m.split('/') |
| name, sub = short_id, '%s · untuned · 512' % org |
| trs.append('<tr%s><td class="nm" style="font-size:%dpx">%s<small>%s</small></td>%s</tr>' |
| % (' class="me"' if m == OURS else '', fs, name, sub, ''.join(tds))) |
| return '<table class="hm"><tr><th>Model</th>%s</tr>%s</table>' % (head, ''.join(trs)) |
|
|
|
|
| def c07(): |
| mids = [OURS, R2B, R1, R1S, TFIDF] + PUBS |
| return page('Every model on every task: test split', |
| 'Test split, max_seq_length 512 (round-1 models 256). Shading runs from lowest to highest within each column; best outlined.', |
| heat('test', mids, 44, 19), foot('test', ' · neural models at max_seq_length 512 unless noted')) |
|
|
|
|
| def c08(): |
| mids = CANDS + [R1, R1S, TFIDF] + PUBS |
| return page('Every model on every task: dev split', |
| 'Dev split, max_seq_length 512 (round-1 models 256). Shading runs from lowest to highest within each column; best outlined.', |
| heat('dev', mids, 36, 16), foot('dev', ' · all five round-2 candidates, both round-1 models, TF-IDF and seven public models')) |
|
|
|
|
| |
| def c09(): |
| models = [(OURS, 'circle'), (TFIDF, 'diamond'), (BGE, 'square'), (E5L, 'triangle')] |
| stroke = {OURS: '#FFFFFF', TFIDF: '#FFFFFF', BGE: '#7C8796', E5L: '#FFFFFF'} |
| width, rowh, lane, top = 1440, 84, 17, 30 |
| x0, x1, lo, hi = 250, 1100, 0.3, 1.0 |
| X = lambda val: x0 + (x1 - x0) * (val - lo) / (hi - lo) |
| bottom = top + rowh * len(COLS) |
| p = [] |
| for t in (0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0): |
| p.append(ln(X(t), top - 4, X(t), bottom, GRID)) |
| p.append(tx(X(t), bottom + 24, '%.1f' % t, 15, 'middle', fill=MUTED)) |
| p.append(tx(1180, top - 10, 'Lead over the best other', 15, 'start', 600, MUTED)) |
| for ri, c in enumerate(COLS): |
| yc = top + rowh * ri + rowh / 2 |
| lab, metric = TASK_LABEL[c] |
| p.append(tx(0, yc - 1, lab, 21, weight=600)) |
| p.append(tx(0, yc + 21, metric, 15, fill=MUTED)) |
| if ri: |
| p.append(ln(0, top + rowh * ri, 1440, top + rowh * ri, GRID)) |
| for li, (m, shape) in enumerate(models): |
| y = yc + (li - 1.5) * lane |
| val = sc(m, 'test', c) |
| me = m == OURS |
| p.append(ln(x0, y, X(val), y, mix(col(m), '#FFFFFF', .55) if not me else '#9DB9EA', 2.5 if me else 1.5)) |
| p.append(mark(shape, X(val), y, 8.5 if me else 7, col(m), stroke[m], 1.5 if m == BGE else 1.5)) |
| p.append(tx(X(val) + 14, y + 5.5, f4(val), 16 if me else 15, 'start', 700 if me else 400, POS if me else INK)) |
| others = max((sc(m, 'test', c), m) for m, _ in models[1:]) |
| d = sc(OURS, 'test', c) - others[0] |
| p.append(tx(1180, yc + 4, dlt(d), 24, 'start', 700, POS if d >= 0 else NEG)) |
| p.append(tx(1290, yc + 4, 'vs ' + SHORT[others[1]], 16, 'start', fill=MUTED)) |
| body = (legend_marks([('Hindko Embeddings (this model)', COL[OURS], 'circle', '#FFFFFF'), ('TF-IDF char 2-4-gram', COL[TFIDF], 'diamond', '#FFFFFF'), |
| ('BAAI/bge-m3, best public model', COL[BGE], 'square', '#7C8796'), ('multilingual-e5-large, the untuned base model', COL[E5L], 'triangle', '#FFFFFF')], mt=18) |
| + svgw(width, bottom + 34, p, 10)) |
| return page('Task profile against the references', |
| 'Test split, max_seq_length 512; one lane per model, axis from 0.3. This model leads on every task except book titles.', |
| body, foot('test', ' · neural models at max_seq_length 512')) |
|
|
|
|
| |
| def c10(): |
| mids = [OURS, R2B, R1, R1S, TFIDF] + PUBS |
| width, height = 1440, 560 |
| x0, x1, y0, y1 = 90, 1000, 10, 500 |
| dlo, dhi, tlo, thi = 0.40, 0.75, 0.40, 0.76 |
| X = lambda d: x0 + (x1 - x0) * (d - dlo) / (dhi - dlo) |
| Y = lambda t: y1 - (y1 - y0) * (t - tlo) / (thi - tlo) |
| p = [] |
| for t in (0.40, 0.45, 0.50, 0.55, 0.60, 0.65, 0.70, 0.75): |
| p.append(ln(x0, Y(t), x1, Y(t), GRID) + tx(x0 - 10, Y(t) + 5, '%.2f' % t, 15, 'end', fill=MUTED)) |
| p.append(ln(X(t), y0, X(t), y1, GRID) + tx(X(t), y1 + 24, '%.2f' % t, 15, 'middle', fill=MUTED)) |
| p.append(tx((x0 + x1) / 2, y1 + 50, 'dev overall score', 16, 'middle', fill=MUTED)) |
| p.append(tx(22, (y0 + y1) / 2, 'test overall score', 16, 'middle', fill=MUTED, extra=' transform="rotate(-90 22 %d)"' % ((y0 + y1) / 2))) |
| p.append(ln(X(0.40), Y(0.40), X(0.75), Y(0.75), '#8E9AAB', 2, ' stroke-dasharray="7 6"')) |
| ang = math.degrees(math.atan2(Y(0.40) - Y(0.75), X(0.75) - X(0.40))) |
| p.append(tx(X(0.452), Y(0.452) - 12, 'test = dev', 15, 'start', fill=MUTED, extra=' transform="rotate(%.1f %.1f %.1f)"' % (-ang, X(0.452), Y(0.452) - 12))) |
| up = sum(1 for m in mids if sc(m, 'test') > sc(m, 'dev')) |
| p.append(tx(X(0.415), Y(0.735), 'Above the line: higher on test than on dev', 16, 'start', 600)) |
| p.append(tx(X(0.415), Y(0.735) + 22, '%d of the %d models' % (up, len(mids)), 16, 'start', fill=MUTED)) |
| |
| near = {BGE: (14, -4, 'start'), E5L: (14, 4, 'start'), E5B: (14, 30, 'middle'), QWEN: (10, 76, 'middle'), |
| E5S: (-14, -10, 'end'), LABSE: (-14, -2, 'end'), MPNET: (14, -4, 'start')} |
| column = {OURS: 30, R1S: 84, R2B: 138, R1: 192, TFIDF: 246} |
| for m in mids: |
| d, t = sc(m, 'dev'), sc(m, 'test') |
| me = m == OURS |
| name = SHORT[m] if m not in (E5B, E5S) else SHORT[m].replace(' (untuned)', '') |
| vals = 'dev %s · test %s' % (f4(d), f4(t)) |
| if m in column: |
| ly = column[m] |
| p.append(ln(X(d), Y(t), 1024, ly - 5, '#AEB7C3', 1.2)) |
| p.append(tx(1032, ly, nm(m) if m != OURS else 'Hindko Embeddings (this model)', 17, 'start', 700 if me else 600, POS if me else INK)) |
| p.append(tx(1032, ly + 20, vals + (' (%s)' % dlt(t - d)), 14, 'start', fill=MUTED)) |
| else: |
| dx, dy, anc = near[m] |
| if dy > 40: |
| p.append(ln(X(d), Y(t) + 9, X(d) + dx, Y(t) + dy - 16, '#AEB7C3', 1.2)) |
| p.append(tx(X(d) + dx, Y(t) + dy, name, 16, anc, 600, INK, True)) |
| p.append(tx(X(d) + dx, Y(t) + dy + 18, vals, 13, anc, fill=MUTED, halo=True)) |
| for m in mids[::-1]: |
| st = '#7C8796' if col(m) in (PUBCOL, COL[BGE]) else '#FFFFFF' |
| p.append(mark('circle', X(sc(m, 'dev')), Y(sc(m, 'test')), 11 if m == OURS else 8, col(m), st, 1.5 if st != '#FFFFFF' else 2)) |
| body = (legend_marks([(SHORT[m], col(m), 'circle', '#FFFFFF') for m in (OURS, R2B, R1, R1S, TFIDF)] + |
| [(SHORT[E5L], COL[E5L], 'circle', '#FFFFFF'), ('bge-m3', COL[BGE], 'circle', '#7C8796'), |
| ('other public models', PUBCOL, 'circle', '#7C8796')], mt=14) |
| .replace('style="margin-top:14px"', 'style="margin-top:14px;gap:24px;font-size:18px"') |
| + svgw(width, height, p, 6)) |
| return page('Dev score against test score', |
| 'Overall score on dev (x) and test (y) for every model scored on both. max_seq_length 512, round-1 models 256.', |
| body, foot2('Dev and test splits', ' · neural models at max_seq_length 512 unless noted')) |
|
|
|
|
| |
| def c11(): |
| mids = sorted(PUBS, key=lambda m: -sc(m, 'test')) |
| width, top, rowh = 1440, 44, 60 |
| name_w = 350 |
| panels = [('test', 'Test split', 380, 850), ('dev', 'Dev split', 950, 1420)] |
| lo, hi = 0.40, 0.66 |
| p = [] |
| height = top + rowh * len(mids) + 40 |
| for split, title, a, b in panels: |
| X = lambda v, a=a, b=b: a + (b - a - 120) * (v - lo) / (hi - lo) |
| p.append(tx(a, 18, title, 21, 'start', 600)) |
| for t in (0.40, 0.45, 0.50, 0.55, 0.60, 0.65): |
| p.append(ln(X(t), top - 6, X(t), top + rowh * len(mids), GRID)) |
| p.append(tx(X(t), top + rowh * len(mids) + 24, '%.2f' % t, 15, 'middle', fill=MUTED)) |
| p.append(tx(b, 18, 'change', 15, 'end', 600, MUTED)) |
| for i, m in enumerate(mids): |
| y = top + rowh * i + rowh / 2 |
| s256, s512 = sc(m, split, tab=M256), sc(m, split) |
| up = s512 >= s256 |
| p.append(ln(X(s256), y, X(s512), y, '#AEB7C3', 3)) |
| p.append(mark('circle', X(s256), y, 8, '#FFFFFF', '#8E9AAB', 2.5)) |
| p.append(mark('circle', X(s512), y, 8.5, '#334155', '#FFFFFF', 2)) |
| lo_v, hi_v = (s256, s512) if up else (s512, s256) |
| p.append(tx(X(lo_v) - 14, y + 5, f4(lo_v), 15, 'end', 700 if lo_v == s512 else 400, INK if lo_v == s512 else MUTED, True)) |
| p.append(tx(X(hi_v) + 14, y + 5, f4(hi_v), 15, 'start', 700 if hi_v == s512 else 400, INK if hi_v == s512 else MUTED, True)) |
| p.append(tx(b, y + 6, dlt(s512 - s256), 18, 'end', 700, POS if up else NEG)) |
| for i, m in enumerate(mids): |
| y = top + rowh * i + rowh / 2 |
| org, sid = m.split('/') |
| p.append(tx(name_w - 10, y - 1, sid, 17, 'end', 600)) |
| p.append(tx(name_w - 10, y + 18, org, 14, 'end', fill=MUTED)) |
| if i: |
| p.append(ln(0, top + rowh * i, width, top + rowh * i, '#F0F2F5')) |
| n_up = {s: sum(1 for m in PUBS if sc(m, s) > sc(m, s, tab=M256)) for s in ('test', 'dev')} |
| body = (legend_marks([('256 tokens, float32 (round 1)', '#FFFFFF', 'circle', '#8E9AAB'), ('512 tokens, float16 (re-run for round 2)', '#334155', 'circle', '#FFFFFF')], mt=16) |
| + svgw(width, height, p, 8) |
| + '<p class="note" style="margin-top:0">Length and precision changed together, so their effects cannot be separated. ' |
| 'Higher at 512: %d of 7 models on test, %d of 7 on dev. The other charts compare public models at 512.</p>' % (n_up['test'], n_up['dev'])) |
| return page('Public models at 256 and at 512 tokens', |
| 'Untuned public models, overall score: round 1 (max_seq_length 256, float32) and the re-run for round 2 (512, float16).', |
| body, foot2('Dev and test splits', ' · untuned public models only')) |
|
|
|
|
| |
| def c12(): |
| models = [OURS, R2B, R1, TFIDF, BGE] |
| panels = [] |
| for split in ('test', 'dev'): |
| n = {s: M[(OURS, split)]['tasks']['HeadlineRetrieval']['by_source'][s]['n_queries'] for s in ('newspaper', 'web')} |
| groups = [('Newspaper headlines', '%d queries' % n['newspaper']), ('Web page titles', '%d queries' % n['web'])] |
| series = [(nm(m), col(m), [sc(m, split, 'HeadlineRetrieval', src=s) for s in ('newspaper', 'web')], m == OURS) for m in models] |
| svg = grouped(groups, series, vmax=1.0, width=690, height=455, x0=50, bar_w=46, gap=8, lab_size=15) |
| panels.append('<div><div class="ptitle">%s<small>HeadlineRetrieval nDCG@10 · %d queries in all</small></div>%s</div>' |
| % ('Test' if split == 'test' else 'Dev', sum(n.values()), svg)) |
| body = (legend2([(nm(m) + ('' if m in (OURS, TFIDF) else ' (512)' if m != R1 else ' (256)'), col(m)) for m in models]) |
| + '<div style="display:grid;grid-template-columns:1fr 1fr;gap:60px">%s</div>' % ''.join(panels)) |
| return page('Headline retrieval by query source', |
| 'nDCG@10 for newspaper headlines and web page titles, test and dev. max_seq_length 512 (round-1 model 256). ' |
| 'The web subsets are tiny: one query can move them by up to %.3f (test) and %.3f (dev).' |
| % tuple(1 / M[(OURS, s_)]['tasks']['HeadlineRetrieval']['by_source']['web']['n_queries'] for s_ in ('test', 'dev')), |
| body, foot2('Test split (left) and dev split (right)', ' · key models')) |
|
|
|
|
| |
| def c13(): |
| models = [OURS, R2B, R1, TFIDF, BGE, E5L] |
| srcs = ['book', 'newspaper', 'web'] |
| bs = M[(OURS, 'test')]['tasks']['PassageContinuation'] |
| groups = [('%s passages' % s.capitalize() if s != 'web' else 'Web pages', '%d queries' % bs['by_source'][s]['n_queries']) for s in srcs] |
| groups.append(('All', '%d queries' % bs['n_queries'])) |
| series = [(nm(m), col(m), [sc(m, 'test', 'PassageContinuation', src=s) for s in srcs] + [sc(m, 'test', 'PassageContinuation')], m == OURS) |
| for m in models] |
| body = (legend2([(SHORT[m] if m in (TFIDF, BGE, E5L) else nm(m), col(m)) for m in models], 17, 22) |
| + grouped(groups, series, vmax=1.0, width=1440, height=500, bar_w=42, gap=9, lab_size=15)) |
| lead = [s for s in srcs if all(sc(OURS, 'test', 'PassageContinuation', src=s) > sc(m, 'test', 'PassageContinuation', src=s) for m in models[1:])] |
| lost = [s for s in srcs if s not in lead] |
| beat = {s: [SHORT[m] for m in models[1:] if sc(m, 'test', 'PassageContinuation', src=s) > sc(OURS, 'test', 'PassageContinuation', src=s)] for s in lost} |
| return page('Passage continuation by source', |
| 'PassageContinuation nDCG@10 on the test split by source, max_seq_length 512 (round-1 model 256). This model leads on %s passages; %s.' |
| % (' and '.join(lead), '; '.join('on the %d %s queries %s score%s higher' % (bs['by_source'][s]['n_queries'], s, ' and '.join(beat[s]), '' if len(beat[s]) > 1 else 's') |
| for s in lost)), |
| body, foot('test', ' · the first half of a passage retrieves its second half')) |
|
|
|
|
| |
| def c14(): |
| models = [OURS, TFIDF, BGE] |
| mets = [('recall_at_1', 'Recall@1'), ('mrr_at_10', 'MRR@10'), ('ndcg_at_10', 'nDCG@10'), ('recall_at_10', 'Recall@10')] |
| panels = [] |
| for task, title in (('HeadlineRetrieval', 'Headline'), ('BookTitleRetrieval', 'Book title'), ('PassageContinuation', 'Continuation')): |
| r = M[(OURS, 'test')]['tasks'][task] |
| sub = '%d queries' % r['n_queries'] + (' (57 relevant)' if task == 'BookTitleRetrieval' else '') |
| p, y = [], 8 |
| for key, lab in mets: |
| p.append(tx(0, y + 46, lab, 18, 'start', 600)) |
| rows = [('', '', sc(m, 'test', task, key), col(m), m == OURS, None) for m in models] |
| parts, y = hbar_rows(rows, 0, 108, 390, 1.0, y, 30, 22, f3, val_size=15) |
| p += parts |
| y += 20 |
| for t in (0.0, 0.5, 1.0): |
| p.insert(0, ln(108 + 282 * t, 0, 108 + 282 * t, y - 14, GRID)) |
| p.append(tx(108 + 282 * t, y + 6, '%.1f' % t, 14, 'middle', fill=MUTED)) |
| panels.append('<div><div class="ptitle">%s<small>%s</small></div>%s</div>' % (title, sub, svgw(450, y + 10, p, 12))) |
| body = (legend([(nm(m) + (' (512)' if m == BGE else ''), col(m)) for m in models]) |
| + '<div style="display:grid;grid-template-columns:repeat(3,1fr);gap:45px">%s</div>' % ''.join(panels)) |
| return page('Retrieval at different depths', |
| 'Test split, max_seq_length 512. Recall@1 asks for the right document first; Recall@10 for it anywhere in the top ten. ' |
| 'Book titles can have several relevant passages, so Recall@1 is below hit@1 there.', |
| body, foot('test', ' · neural models at max_seq_length 512')) |
|
|
|
|
| |
| def c15(): |
| models = [OURS, R2B, R1, R1S, TFIDF, BGE, E5L] |
| width, top, rowh, lane = 1440, 52, 58, 12 |
| panels = [('VerseProseClassification', 'Verse / prose classification', [('f1_macro', 'f1_macro_std', 'macro-F1', 'circle'), ('accuracy', 'accuracy_std', 'accuracy', 'diamond')], |
| 0.92, 1.00, (0.92, 0.94, 0.96, 0.98, 1.00), 330, 630, 650), |
| ('BookClustering', 'Book clustering', [('v_measure', 'v_measure_std', 'V-measure', 'circle'), ('ari', None, 'ARI', 'diamond')], |
| 0.2, 1.0, (0.2, 0.4, 0.6, 0.8, 1.0), 900, 1200, 1220)] |
| bottom = top + rowh * len(models) |
| p = [] |
| for task, title, mets, lo, hi, ticks, a, b, vx in panels: |
| X = lambda v, a=a, b=b, lo=lo, hi=hi: a + (b - a) * (v - lo) / (hi - lo) |
| p.append(tx(a, 18, title, 21, 'start', 600)) |
| p.append(tx(vx, top - 12, mets[0][2], 14, 'start', 600, MUTED) + tx(vx + 150, top - 12, mets[1][2], 14, 'start', 600, MUTED)) |
| for t in ticks: |
| p.append(ln(X(t), top - 4, X(t), bottom, GRID)) |
| p.append(tx(X(t), bottom + 22, '%.2f' % t, 14, 'middle', fill=MUTED)) |
| for i, m in enumerate(models): |
| yc = top + rowh * i + rowh / 2 |
| r = M[(m, 'test')]['tasks'][task] |
| me = m == OURS |
| for li, (k, ks, lab, shape) in enumerate(mets): |
| y = yc + (li - .5) * 2 * lane |
| v, s = r[k], (r[ks] if ks else None) |
| if s: |
| p.append(ln(X(max(lo, v - s)), y, X(min(hi, v + s)), y, mix(col(m), INK, .25) if m != BGE else '#7C8796', 2)) |
| for e in (max(lo, v - s), min(hi, v + s)): |
| p.append(ln(X(e), y - 5, X(e), y + 5, mix(col(m), INK, .25) if m != BGE else '#7C8796', 2)) |
| p.append(mark(shape, X(v), y, 7 if me else 6, col(m), '#7C8796' if m == BGE else '#FFFFFF', 1.5)) |
| txt = f3(v) + (' ± %s' % f3(s) if s is not None else '') |
| p.append(tx(vx + (0 if li == 0 else 150), yc + 6, txt, 15, 'start', 700 if me else 400, POS if me else INK)) |
| for i, m in enumerate(models): |
| yc = top + rowh * i + rowh / 2 |
| p.append(tx(300, yc + 6, nm(m) if m not in (BGE, E5L) else {BGE: 'BAAI/bge-m3', E5L: 'multilingual-e5-large'}[m], 17, 'end', 700 if m == OURS else 400, POS if m == OURS else INK)) |
| if i: |
| p.append(ln(0, top + rowh * i, width, top + rowh * i, '#F0F2F5')) |
| body = (legend_marks([('main metric (macro-F1, V-measure)', '#8E9AAB', 'circle', '#FFFFFF'), ('second metric (accuracy, ARI)', '#8E9AAB', 'diamond', '#FFFFFF'), |
| ('whiskers: ± one standard deviation over 10 runs', '#5B6576', 'line', '')], mt=16) |
| + svgw(width, bottom + 30, p, 6) |
| + '<p class="note" style="margin-top:0">The verse class of the test set comes almost entirely from one book. TF-IDF gives the same ' |
| 'classification score in all 10 test runs (std 0). ARI has no spread recorded.</p>') |
| return page('Classification and clustering in detail', |
| 'Test split, max_seq_length 512 (round-1 models 256). 10 logistic-regression runs on 157 passages; 10 k-means runs on 176 passages from 5 books. ' |
| 'Note the different axis ranges.', |
| body, foot('test', ' · neural models at max_seq_length 512 unless noted')) |
|
|
|
|
| |
| def c16(): |
| width, top, rowh = 1440, 56, 66 |
| a, b, lim = 640, 1240, 0.015 |
| X = lambda d: (a + b) / 2 + (b - a) / 2 * d / lim |
| bottom = top + rowh * len(COLS) |
| p = [rect(X(-0.01), top - 6, X(0.01) - X(-0.01), bottom - top + 6, '#F2F5F9', 0)] |
| p.append(tx(300, top - 16, 'with', 15, 'end', 600, MUTED)) |
| p.append(tx(300, top + 0, 'negatives + blocks', 13, 'end', fill=MUTED)) |
| p.append(tx(440, top - 16, 'without', 15, 'end', 600, MUTED)) |
| p.append(tx(440, top + 0, '(plain)', 13, 'end', fill=MUTED)) |
| p.append(tx((a + b) / 2, top - 16, 'with minus without', 15, 'middle', 600, MUTED)) |
| for t in (-0.015, -0.01, -0.005, 0, 0.005, 0.01, 0.015): |
| p.append(ln(X(t), top - 6, X(t), bottom, INK if t == 0 else GRID, 1.5 if t == 0 else 1)) |
| p.append(tx(X(t), bottom + 22, ('%+.3f' % t if t else '0').replace('-', '−'), 14, 'middle', fill=MUTED)) |
| for i, c in enumerate(COLS): |
| y = top + rowh * i + rowh / 2 |
| lab, metric = TASK_LABEL[c] |
| w_, wo = sc(R2B, 'dev', c), sc(R2P, 'dev', c) |
| d = w_ - wo |
| p.append(tx(0, y - 1, lab, 20, 'start', 600)) |
| p.append(tx(0, y + 19, metric, 14, 'start', fill=MUTED)) |
| p.append(tx(300, y + 7, f4(w_), 19, 'end', 600)) |
| p.append(tx(440, y + 7, f4(wo), 19, 'end')) |
| x0, x1 = sorted((X(0), X(d))) |
| p.append(rect(x0, y - 15, x1 - x0, 30, '#6F95DE' if d >= 0 else '#D98A75', 3)) |
| p.append(tx(X(d) + (10 if d >= 0 else -10), y + 7, dlt(d), 19, 'start' if d >= 0 else 'end', 700, POS if d >= 0 else NEG)) |
| if i: |
| p.append(ln(0, top + rowh * i, width, top + rowh * i, '#F0F2F5')) |
| body = (svgw(width, bottom + 34, p, 22) |
| + '<p class="note" style="margin-top:10px">Shaded: ±0.01, about one standard error of the dev overall score (estimate from the training plan); ' |
| 'single tasks are noisier. One seed per run, so none of these differences is established. The recipe gate (fixed in advance) ' |
| 'kept mined negatives and same-book blocks for the larger runs.</p>') |
| return page('Ablation: mined hard negatives and same-book batches', |
| 'Dev split, max_seq_length 512. Round-2 e5-base trained twice on the same pairs: with mined hard negatives and same-book blocks of 32, ' |
| 'and without both (in-batch negatives only).', |
| body, foot('dev', ' · Round-2 e5-base vs Round-2 e5-base, no hard negatives')) |
|
|
|
|
| |
| def c17(): |
| width, height = 1440, 520 |
| x0, x1, y0, y1, ymax = 60, 1100, 10, 470, 6.5 |
| X = lambda s: x0 + (x1 - x0) * s / 321 |
| Y = lambda v: y1 - (y1 - y0) * v / ymax |
| warm = TRAIN[OURS]['training_args']['warmup_steps'] |
| p = [rect(X(0), y0, X(warm) - X(0), y1 - y0, '#F2F5F9', 0), tx(X(warm / 2), y0 + 18, 'warm-up', 14, 'middle', fill=MUTED)] |
| for t in range(0, 7): |
| p.append(ln(x0, Y(t), x1, Y(t), GRID) + tx(x0 - 10, Y(t) + 5, str(t), 15, 'end', fill=MUTED)) |
| for s in (0, 50, 100, 150, 200, 250, 300, 321): |
| p.append(tx(X(s), y1 + 24, str(s), 15, 'middle', fill=MUTED)) |
| ep = TRAIN[OURS]['steps'] // TRAIN[OURS]['training_args']['num_train_epochs'] |
| p.append(ln(X(ep - .5), y0, X(ep - .5), y1, INK, 1.5, ' stroke-dasharray="6 5"')) |
| p.append(tx(X(ep - .5) + 8, y0 + 18, 'epoch 2 starts (step %d)' % ep, 15, 'start', 600)) |
| p.append(tx((x0 + x1) / 2, y1 + 48, 'optimizer step (batch of 256 rows from one dataset)', 15, 'middle', fill=MUTED)) |
| order = [R2P, R2S, R2G, R2B, OURS] |
| ends = {} |
| for m in order: |
| vals = [v for _, _, v in sorted(LOSS[m])] |
| p.append('<polyline points="%s" fill="none" stroke="%s" stroke-width="1" stroke-opacity="0.13"/>' |
| % (' '.join('%.1f,%.1f' % (X(i), Y(v)) for i, v in enumerate(vals)), col(m))) |
| for m in order: |
| vals = [v for _, _, v in sorted(LOSS[m])] |
| ma = mavg(vals, 15) |
| ends[m] = ma[-1] |
| p.append('<polyline points="%s" fill="none" stroke="%s" stroke-width="%s" stroke-linejoin="round"/>' |
| % (' '.join('%.1f,%.1f' % (X(i), Y(v)) for i, v in enumerate(ma)), col(m), 4 if m == OURS else 2.5)) |
| ys = spread([Y(ends[m]) for m in order], 48, y0 + 10, y1) |
| for m, ly in zip(order, ys): |
| last = [v for _, _, v in sorted(LOSS[m])][-15:] |
| p.append(ln(X(321) + 4, Y(ends[m]), 1122, ly - 7, col(m), 2)) |
| p.append(rect(1122, ly - 13, 18, 12, col(m), 2)) |
| p.append(tx(1148, ly - 2, nm(m), 16, 'start', 700 if m == OURS else 600, POS if m == OURS else INK)) |
| p.append(tx(1148, ly + 17, 'last 15 steps %.3f · all %.3f' % (sum(last) / len(last), sum(v for _, _, v in LOSS[m]) / len(LOSS[m])), 13, 'start', fill=MUTED)) |
| body = (legend_marks([('15-step moving average', '#5B6576', 'line', ''), ('loss at every step (faint)', '#C9D0DA', 'line', '')], mt=14) |
| + svgw(width, height, p, 6)) |
| return page('Training loss of the five round-2 runs', |
| 'CachedMultipleNegativesRankingLoss per optimizer step, max_seq_length 512, 2 epochs of 161 steps. Each step is one dataset, so the raw ' |
| 'loss jumps with the dataset; runs without mined negatives have fewer candidates per anchor and a lower loss.', |
| body, foot2('Round-2 training on the train split', ' · one NVIDIA A100-SXM4-80GB · seed 20261001'), eyebrow='Training') |
|
|
|
|
| |
| def c18(): |
| width, height = 920, 520 |
| x0, x1, y0, y1, ymax = 50, 760, 10, 470, 6.5 |
| X = lambda s: x0 + (x1 - x0) * s / 321 |
| Y = lambda v: y1 - (y1 - y0) * v / ymax |
| p = [] |
| for t in range(0, 7): |
| p.append(ln(x0, Y(t), x1, Y(t), GRID) + tx(x0 - 10, Y(t) + 5, str(t), 15, 'end', fill=MUTED)) |
| for s in (0, 100, 200, 321): |
| p.append(tx(X(s), y1 + 24, str(s), 15, 'middle', fill=MUTED)) |
| p.append(ln(X(160.5), y0, X(160.5), y1, INK, 1.5, ' stroke-dasharray="6 5"') + tx(X(160.5) + 8, y0 + 16, 'epoch 2', 14, 'start', 600)) |
| p.append(tx((x0 + x1) / 2, y1 + 48, 'optimizer step', 15, 'middle', fill=MUTED)) |
| ends = {} |
| for ty in TYPES: |
| pts = [(s, v) for s, d, v in sorted(LOSS[OURS]) if d == ty] |
| ma = mavg([v for _, v in pts], 5) |
| ends[ty] = (pts[-1][0], ma[-1], len(pts), sum(v for _, v in pts[-5:]) / 5) |
| p.append(''.join('<circle cx="%.1f" cy="%.1f" r="3" fill="%s" fill-opacity="0.45"/>' % (X(s), Y(v), TYPECOL[ty]) for s, v in pts)) |
| p.append('<polyline points="%s" fill="none" stroke="%s" stroke-width="3" stroke-linejoin="round"/>' |
| % (' '.join('%.1f,%.1f' % (X(s), Y(v)) for (s, _), v in zip(pts, ma)), TYPECOL[ty])) |
| ys = spread([Y(ends[ty][1]) for ty in TYPES], 46, y0 + 10, y1) |
| for ty, ly in zip(TYPES, ys): |
| s, v, n, last5 = ends[ty] |
| p.append(ln(X(s) + 4, Y(v), 776, ly - 6, '#AEB7C3', 1)) |
| p.append(tx(782, ly - 1, ty, 17, 'start', 700)) |
| p.append(tx(782, ly + 17, 'last 5: %.2f · %d steps' % (last5, n), 13, 'start', fill=MUTED)) |
| chart = svgw(width, height, p, 6) |
| cand = [OURS, R2B, R2S, R2G, R2P] |
| head = ''.join('<th style="padding:0 6px 8px;text-align:right;text-transform:none;letter-spacing:0;font-size:14px;color:#0F172A">' |
| '<i style="display:block;margin:0 0 5px auto;width:22px;height:5px;border-radius:2px;background:%s"></i>%s</th>' % (TYPECOL[t], t) for t in TYPES) |
| rows = [] |
| for m in cand: |
| ml = TRAIN[m]['mean_loss_per_dataset'] |
| me = m == OURS |
| rows.append('<tr%s><td style="text-align:left;padding:9px 6px 9px 0;font-size:16px">%s</td>%s</tr>' |
| % (' class="me"' if me else '', SHORT[m] if m != R2P else 'Round-2 e5-base,<br>no hard negatives', |
| ''.join('<td style="padding:9px 6px;font-size:17px">%.2f</td>' % ml[t] for t in TYPES))) |
| table = ('<div><div class="ptitle" style="margin-top:4px">Mean loss per dataset<small>all steps</small></div>' |
| '<table style="margin-top:12px;width:100%%"><tr><th style="text-align:left;padding:0 6px 8px 0">Run</th>%s</tr>%s</table>' |
| '<p class="note" style="font-size:16px;margin-top:12px">Title and verse rows also carry mined negatives from the same book, and their loss stays the highest. ' |
| 'Without mined negatives (last row) the mean title loss is %.2f, against %.2f with them (Round-2 e5-base).</p></div>' |
| % (head, ''.join(rows), TRAIN[R2P]['mean_loss_per_dataset']['title'], TRAIN[R2B]['mean_loss_per_dataset']['title'])) |
| body = ('<div style="display:grid;grid-template-columns:920px 1fr;gap:30px;margin-top:10px"><div>%s%s</div>%s</div>' |
| % (legend_marks([('5-point moving average per dataset', '#5B6576', 'line', ''), ('loss at each step of that dataset', '#AEB7C3', 'circle', '#FFFFFF')], mt=4), chart, table)) |
| return page('This model\'s loss by training dataset', |
| 'Hindko Embeddings (round-2 e5-large), max_seq_length 512: loss at each step, split by the dataset the batch came from. ' |
| 'Right: the mean loss of each dataset for all five round-2 runs.', |
| body, foot2('Round-2 training on the train split', ' · batches of 256 rows from one dataset each'), eyebrow='Training') |
|
|
|
|
| |
| def c19(): |
| ts = TDATA['rows_by_type_source'] |
| tv = TDATA['rows_by_ds_type_variety'] |
| rows = [('title', 'title_any', 'table-of-contents title → passage'), ('title', 'verse_first', 'first line of a poem → the rest'), |
| ('verse_ict', 'verse_line', 'one verse line → the poem without it'), ('prose_q', 'headline', 'headline or page title → article'), |
| ('prose_q', 'prose_sent', 'one sentence → its surrounding text'), ('continuation', 'continuation', 'first half → second half'), |
| ('neighbour', 'neighbour', 'a chunk → the next chunk')] |
| p = [tx(0, 18, 'By pair type', 21, 'start', 600), tx(150, 18, 'coloured by dataset', 15, 'start', fill=MUTED)] |
| y, x0, x1, vmax = 40, 330, 640, 15000 |
| for ds, ty, desc in rows: |
| n = TDATA['rows_by_ds_type']['%s/%s' % (ds, ty)] |
| src = [(s, ts.get('%s/%s' % (ty, s), 0)) for s in ('book', 'newspaper', 'web')] |
| var = [(v, tv.get('%s/%s/%s' % (ds, ty, v), 0)) for v in ('hindko', 'urdu', 'mixed')] |
| cy = y + 22 |
| p.append(tx(x0 - 14, cy - 2, ty + ' <tspan fill="%s" font-weight="400" font-size="14">(%s)</tspan>' % (MUTED, ds), 18, 'end', 600)) |
| p.append(tx(x0 - 14, cy + 17, desc, 14, 'end', fill=MUTED)) |
| w = (x1 - x0) * n / vmax |
| p.append(rect(x0, cy - 13, w, 24, TYPECOL[ds], 3)) |
| p.append(tx(x0 + w + 8, cy + 6, thou(n), 17, 'start', 700)) |
| p.append(tx(x0, cy + 30, ' · '.join('%s %s' % (s, thou(c)) for s, c in src if c) + ' | ' + |
| ' · '.join('%s %s' % (v, thou(c)) for v, c in var if c), 13, 'start', fill=MUTED)) |
| y += 66 |
| total = TDATA['n_rows'] |
| p.append(tx(x0 - 14, y + 18, 'Total', 18, 'end', 700) + tx(x0, y + 18, '%s rows · %s distinct texts' % (thou(total), thou(TDATA['n_texts'])), 18, 'start', 700)) |
| left = svgw(800, y + 30, p, 4) |
|
|
| def summary(title, items): |
| q = [tx(0, 18, title, 21, 'start', 600)] |
| yy = 40 |
| mx = max(c for _, c, _ in items) |
| for lab, c, colr in items: |
| q.append(tx(150, yy + 22, lab, 17, 'end')) |
| w = 200 * c / mx |
| q.append(rect(162, yy + 6, w, 22, colr, 3)) |
| q.append(tx(162 + w + 8, yy + 23, '%s <tspan fill="%s" font-size="14">%.1f%%</tspan>' % (thou(c), MUTED, 100 * c / total), 16, 'start', 600)) |
| yy += 34 |
| return svgw(560, yy + 4, q, 4) |
| by_src = {s: sum(c for k, c in ts.items() if k.endswith('/' + s)) for s in ('book', 'newspaper', 'web')} |
| by_var = {v: sum(c for k, c in tv.items() if k.endswith('/' + v)) for v in ('hindko', 'urdu', 'mixed')} |
| by_ds = TDATA['rows_by_ds'] |
| assert sum(by_src.values()) == sum(by_var.values()) == sum(by_ds.values()) == total |
| right = (summary('By dataset', [(d, by_ds[d], TYPECOL[d]) for d in TYPES]) + |
| summary('By source', [(s, by_src[s], '#7D8899') for s in ('book', 'newspaper', 'web')]) + |
| summary('By variety of the record', [(v, by_var[v], '#7D8899') for v in ('hindko', 'urdu', 'mixed')])) |
| body = '<div style="display:grid;grid-template-columns:800px 1fr;gap:60px;margin-top:14px"><div>%s</div><div>%s</div></div>' % (left, right) |
| return page('What the model was trained on', |
| 'The %s round-2 training rows (anchor and positive, from the train split only), by pair type, source and variety. ' |
| 'Under each bar: rows by source | by variety.' % thou(total), |
| body, foot2('Training pairs, train split', ' · leak-checked against every dev and test text'), eyebrow='Training') |
|
|
|
|
| |
| def c20(): |
| B = lambda task, split: BENCH['%s.%s' % (task, split)] |
| hq = {s: B('HeadlineRetrieval', s)['query_sources'] for s in ('dev', 'test')} |
| rows = [('Headline', 'queries', 'dev: news %d + web %d · test: news %d + web %d' % (hq['dev']['newspaper'], hq['dev']['web'], hq['test']['newspaper'], hq['test']['web']), |
| B('HeadlineRetrieval', 'dev')['sizes']['queries'], B('HeadlineRetrieval', 'test')['sizes']['queries']), |
| ('', 'documents', 'retrieval corpus, shared with book titles', B('HeadlineRetrieval', 'dev')['sizes']['corpus'], B('HeadlineRetrieval', 'test')['sizes']['corpus']), |
| ('Book title', 'queries', 'dev: %d books · test: %d books' % (B('BookTitleRetrieval', 'dev')['n_books'], B('BookTitleRetrieval', 'test')['n_books']), |
| B('BookTitleRetrieval', 'dev')['sizes']['queries'], B('BookTitleRetrieval', 'test')['sizes']['queries']), |
| ('', 'relevant passages', '', B('BookTitleRetrieval', 'dev')['relevant_docs_total'], B('BookTitleRetrieval', 'test')['relevant_docs_total']), |
| ('Continuation', 'queries = documents', '', B('PassageContinuation', 'dev')['sizes']['queries'], B('PassageContinuation', 'test')['sizes']['queries']), |
| ('Verse / prose', 'eval passages', 'verse + prose: %d + %d (dev) · %d + %d (test)' % (B('VerseProseClassification', 'dev')['eval']['verse']['n'], B('VerseProseClassification', 'dev')['eval']['prose']['n'], |
| B('VerseProseClassification', 'test')['eval']['verse']['n'], B('VerseProseClassification', 'test')['eval']['prose']['n']), |
| B('VerseProseClassification', 'dev')['sizes']['eval'], B('VerseProseClassification', 'test')['sizes']['eval']), |
| ('', 'training pool', 'from the train split, same for both', B('VerseProseClassification', 'dev')['sizes']['train'], B('VerseProseClassification', 'test')['sizes']['train']), |
| ('Book clustering', 'passages', '', B('BookClustering', 'dev')['sizes']['items'], B('BookClustering', 'test')['sizes']['items']), |
| ('', 'books (clusters)', '', B('BookClustering', 'dev')['n_books'], B('BookClustering', 'test')['n_books'])] |
| width, top, rowh = 1440, 30, 56 |
| x0, x1, vmax = 470, 1240, 3000 |
| X = lambda v: x0 + (x1 - x0) * math.log10(v) / math.log10(vmax) |
| bottom = top + rowh * len(rows) |
| p = [tx(x1 + 30, top + rowh * len(rows) + 22, 'count, log scale', 14, 'start', 600, MUTED)] |
| for t in (1, 3, 10, 30, 100, 300, 1000, 3000): |
| p.append(ln(X(t), top - 6, X(t), bottom, GRID) + tx(X(t), bottom + 22, thou(t), 14, 'middle', fill=MUTED)) |
| for i, (task, what, note, dv, tv) in enumerate(rows): |
| y = top + rowh * i + rowh / 2 |
| if task: |
| p.append(tx(0, y + 1, task, 19, 'start', 600)) |
| if i: |
| p.append(ln(0, top + rowh * i, width, top + rowh * i, GRID)) |
| p.append(tx(x0 - 30, y + (0 if note else 6), what, 17, 'end')) |
| if note: |
| p.append(tx(x0 - 30, y + 18, note, 13, 'end', fill=MUTED)) |
| p.append(ln(X(min(dv, tv)), y, X(max(dv, tv)), y, '#AEB7C3', 3)) |
| p.append(mark('circle', X(dv), y, 8.5, '#334155', '#FFFFFF', 2)) |
| p.append(mark('diamond', X(tv), y, 8, '#E3A33B', '#FFFFFF', 1.5)) |
| if dv == tv: |
| p.append(tx(X(dv) + 16, y + 6, '%s (dev and test)' % thou(dv), 16, 'start', 600)) |
| else: |
| hn, hv, lname, lv = ('dev', dv, 'test', tv) if dv > tv else ('test', tv, 'dev', dv) |
| p.append(tx(X(hv) + 16, y + 6, '%s %s' % (hn, thou(hv)), 16, 'start', 600)) |
| p.append(tx(X(lv) - 16, y + 6, '%s %s' % (lname, thou(lv)), 16, 'end', 600)) |
| body = (legend_marks([('dev split (model selection)', '#334155', 'circle', '#FFFFFF'), ('test split (scored once)', '#E3A33B', 'diamond', '#FFFFFF')], mt=12) |
| + svgw(width, bottom + 30, p, 4)) |
| return page('How big each benchmark task is', |
| 'Dev and test splits (sizes do not depend on max_seq_length). The test book side is small: 28 book-title queries, 5 books.', |
| body, foot2('Dev and test splits', ' · built automatically from corpus metadata; group-disjoint from the training data')) |
|
|
|
|
| |
| def c21(): |
| runs = sorted(CANDS, key=lambda m: -sc(m, 'dev')) |
| params = {OURS: PARAMS[E5L], R2B: PARAMS[E5B], R2P: PARAMS[E5B], R2S: PARAMS[E5S], R2G: PARAMS[BGE]} |
| width, top, rowh = 1440, 56, 78 |
| panels = [('Training time', 'minutes', lambda m: TRAIN[m]['seconds'] / 60, 30, lambda v: '%.1f min' % v), |
| ('Seconds per step', '322 steps', lambda m: TRAIN[m]['s_per_step'], 6, lambda v: '%.2f s' % v), |
| ('Peak GPU memory', 'allocated, GB', lambda m: TRAIN[m]['mem_alloc_gb'], 30, lambda v: '%.1f GB' % v)] |
| p = [] |
| bottom = top + rowh * len(runs) |
| for pi, (title, unit, f, vmax, fmt) in enumerate(panels): |
| a = 340 + pi * 280 |
| p.append(tx(a, 18, title, 20, 'start', 600) + tx(a, 40, unit, 14, 'start', fill=MUTED)) |
| for i, m in enumerate(runs): |
| y = top + rowh * i + rowh / 2 |
| v = f(m) |
| w = 180 * v / vmax |
| p.append(rect(a, y - 13, w, 26, col(m), 3)) |
| p.append(tx(a + w + 8, y + 6, fmt(v), 16, 'start', 700 if m == OURS else 400, POS if m == OURS else INK)) |
| a, b, lo, hi = 1200, 1420, 0.695, 0.715 |
| X = lambda v: a + (b - a) * (v - lo) / (hi - lo) |
| p.append(tx(a, 18, 'Dev overall', 20, 'start', 600) + tx(a, 40, 'axis 0.695 to 0.715', 14, 'start', fill=MUTED)) |
| for t in (0.695, 0.705, 0.715): |
| p.append(ln(X(t), top - 4, X(t), bottom, GRID) + tx(X(t), bottom + 20, '%.3f' % t, 13, 'middle', fill=MUTED)) |
| for i, m in enumerate(runs): |
| y = top + rowh * i + rowh / 2 |
| v = sc(m, 'dev') |
| p.append(mark('circle', X(v), y - 6, 9 if m == OURS else 7.5, col(m), '#FFFFFF', 2)) |
| p.append(tx(X(v), y + 22, f4(v), 15, 'middle', 700 if m == OURS else 400, POS if m == OURS else INK)) |
| for i, m in enumerate(runs): |
| y = top + rowh * i + rowh / 2 |
| me = m == OURS |
| p.append(tx(310, y - 2, nm(m), 18, 'end', 700 if me else 600, POS if me else INK)) |
| p.append(tx(310, y + 18, '%dM params · mini-batch %d · lr %s' % (params[m], TRAIN[m]['mini'], ('%g' % TRAIN[m]['lr']).replace('e-05', 'e-5')), |
| 14, 'end', fill=MUTED)) |
| if i: |
| p.append(ln(0, top + rowh * i, width, top + rowh * i, '#F0F2F5')) |
| gpu = TRJ['runs'][OURS]['gpu'] |
| body = (svgw(width, bottom + 30, p, 22) |
| + '<p class="note" style="margin-top:10px">Same data, 2 epochs, batch 256 and max_seq_length 512 for every run. The gradient cache splits each batch of 256 ' |
| 'into mini-batches, so peak memory depends on the mini-batch size as well as on the model. Peak memory reserved for this model: %.1f GB.</p>' |
| % TRAIN[OURS]['mem_reserved_gb']) |
| return page('What each round-2 run cost', |
| 'Training time, speed and peak GPU memory of the five round-2 runs on one %s, next to their dev score ' |
| '(dev split, max_seq_length 512).' % gpu, |
| body, foot2('Round-2 training runs', ' · dev scores from the dev split'), eyebrow='Training') |
|
|
|
|
| |
| def c22(): |
| a = [sc(E5L, 'test', c) for c in COLS] |
| r1 = [sc(R1, 'test', c) for c in COLS] |
| b = [sc(OURS, 'test', c) for c in COLS] |
| series = [('multilingual-e5-large, untuned (512 tokens)', COL[E5L], a, False), |
| ('Round-1 e5-base fine-tune (256 tokens)', COL[R1], r1, False), |
| ('Hindko Embeddings, this model (e5-large fine-tune, 512 tokens)', COL[OURS], b, True)] |
|
|
| def below(gi, gx, gw, y1): |
| d1 = b[gi] - a[gi] |
| d2 = b[gi] - r1[gi] |
| return ''.join('<text x="%.1f" y="%d" font-size="16" font-weight="700" fill="%s" text-anchor="middle">%s <tspan font-weight="400" fill="#5B6576">%s</tspan></text>' |
| % (gx + gw / 2, y1 + yy, POS if d >= 0 else NEG, dlt(d, 3), lab) for d, lab, yy in ((d1, 'vs untuned', 76), (d2, 'vs round 1', 98))) |
| body = (legend([(n, c) for n, c, _, _ in series]) + |
| grouped([TASK_LABEL[c] for c in COLS], series, height=515, below=below, bar_w=56, gap=8, lab_size=15, bottom=108)) |
| return page('From the base model to this model', |
| 'Test split. Untuned multilingual-e5-large at 512 tokens, the round-1 model (e5-base, 256 tokens) and this model (e5-large, 512 tokens). ' |
| 'The round-1 model has a different base, so the middle step is a reference, not an ancestor.', |
| body, foot('test', ' · changes below each group are this model minus the other two')) |
|
|
|
|
| |
| def c23(): |
| tasks = [('HeadlineRetrieval', 'Headline', 'queries + shared corpus'), ('BookTitleRetrieval', 'Book title', 'queries only'), |
| ('PassageContinuation', 'Continuation', 'queries + documents'), ('VerseProseClassification', 'Verse / prose', 'train pool + eval'), |
| ('BookClustering', 'Clustering', 'passages not yet encoded'), ('all', 'All texts', 'every text of the run')] |
| width, top, rowh = 1440, 44, 70 |
| panels = [('test', 'Test split', 330, 840), ('dev', 'Dev split', 910, 1420)] |
| p = [] |
| shares = {} |
| for split, title, a, b in panels: |
| p.append(tx(a, 18, title, 21, 'start', 600)) |
| vmax = 0.6 |
| X = lambda v, a=a: a + 300 * v / vmax |
| for t in (0, 0.2, 0.4, 0.6): |
| p.append(ln(X(t), top - 4, X(t), top + rowh * len(tasks), GRID) + tx(X(t), top + rowh * len(tasks) + 20, '%d%%' % (100 * t), 13, 'middle', fill=MUTED)) |
| for i, (task, _, _) in enumerate(tasks): |
| y = top + rowh * i + rowh / 2 |
| for j, (m, msl) in enumerate(((R1, 256), (OURS, 512))): |
| tk = M[(m, split)]['tasks'] |
| assert M[(m, split)]['msl'] == msl |
| if task == 'all': |
| enc = sum(tk[t]['texts_encoded'] for t in TASKS) |
| tr = sum(tk[t]['texts_truncated'] for t in TASKS) |
| else: |
| enc, tr = tk[task]['texts_encoded'], tk[task]['texts_truncated'] |
| v = tr / enc if enc else 0 |
| shares[(split, task, msl)] = (tr, enc) |
| yy = y - 12 + j * 25 |
| p.append(rect(X(0), yy - 10, X(v) - X(0), 20, col(m), 3)) |
| p.append(tx(X(v) + 8, yy + 6, '%.1f%% <tspan fill="%s" font-weight="400" font-size="13">%s of %s</tspan>' % (100 * v, MUTED, thou(tr), thou(enc)), |
| 15, 'start', 700 if msl == 512 else 600, POS if msl == 512 else INK)) |
| for i, (task, lab, sub) in enumerate(tasks): |
| y = top + rowh * i + rowh / 2 |
| p.append(tx(300, y - 1, lab, 19, 'end', 700 if task == 'all' else 600)) |
| p.append(tx(300, y + 18, sub, 14, 'end', fill=MUTED)) |
| if i: |
| p.append(ln(0, top + rowh * i, width, top + rowh * i, '#F0F2F5')) |
| body = (legend([('256 tokens: Round-1 e5-base fine-tune', COL[R1]), ('512 tokens: Hindko Embeddings (this model)', COL[OURS])]) |
| + svgw(width, top + rowh * len(tasks) + 30, p, 10) |
| + '<p class="note" style="margin-top:0;font-size:17px">Same XLM-RoBERTa tokenizer and prefixes in both runs, so only max_seq_length differs.%s ' |
| 'Each text is encoded once per run and counted under the first task that needs it.</p>' |
| % (' At 512 no continuation text is cut.' if all(shares[(s, 'PassageContinuation', 512)][0] == 0 for s in ('dev', 'test')) else '')) |
| share = lambda s, msl: 100 * shares[(s, 'all', msl)][0] / shares[(s, 'all', msl)][1] |
| return page('Why the length went from 256 to 512 tokens', |
| 'Share of texts the model cuts off. All texts: %.1f%% at 256 vs %.1f%% at 512 tokens on dev, %.1f%% vs %.1f%% on test.' |
| % (share('dev', 256), share('dev', 512), share('test', 256), share('test', 512)), |
| body, foot2('Dev and test splits', ' · texts_truncated / texts_encoded from the evaluation runs')) |
|
|
|
|
| |
| def c24(): |
| mids = [OURS, R2B, R1, R1S, TFIDF] + PUBS |
| rank = {} |
| for c in COLS: |
| vals = {m: round(sc(m, 'test', c), 4) for m in mids} |
| for m in mids: |
| rank[(m, c)] = 1 + sum(1 for o in mids if vals[o] > vals[m]) |
| width, top, rowh = 1440, 56, 44 |
| xs = [420 + i * 150 for i in range(len(COLS))] |
| Y = lambda r: top + (r - 1) * rowh |
| p = [] |
| for x, c in zip(xs, COLS): |
| lab, metric = TASK_LABEL[c] |
| p.append(tx(x, 14, lab, 18, 'middle', 600) + tx(x, 34, metric, 13, 'middle', fill=MUTED)) |
| for r in range(1, len(mids) + 1): |
| p.append(ln(xs[0] - 30, Y(r), xs[-1] + 30, Y(r), '#F0F2F5')) |
| order = [m for m in mids if m not in (OURS,)][::-1] + [OURS] |
| for m in order: |
| me = m == OURS |
| c_ = col(m) if m not in PUBS or m == BGE else '#D5DAE1' |
| w = 5 if me else (3 if m in (TFIDF, BGE, R2B, R1, R1S) else 2) |
| p.append('<polyline points="%s" fill="none" stroke="%s" stroke-width="%s" stroke-linejoin="round"%s/>' |
| % (' '.join('%.1f,%.1f' % (x, Y(rank[(m, c)])) for x, c in zip(xs, COLS)), c_ if m != BGE else '#AEB7C3', w, '' if me or m not in PUBS else ' stroke-opacity="0.9"')) |
| for m in order: |
| me = m == OURS |
| fill = col(m) if m in COL else PUBCOL |
| ink = '#FFFFFF' if m in (OURS, E5L) else INK |
| for x, c in zip(xs, COLS): |
| r = rank[(m, c)] |
| p.append(mark('circle', x, Y(r), 15 if me else 13, fill, '#FFFFFF' if fill not in (PUBCOL, COL[BGE]) else '#8E9AAB', 2 if fill not in (PUBCOL, COL[BGE]) else 1.5)) |
| p.append(tx(x, Y(r) + 5, str(r), 14 if me else 13, 'middle', 700, ink)) |
| for m in mids: |
| me = m == OURS |
| r0, r5 = rank[(m, 'overall')], rank[(m, COLS[-1])] |
| lab = nm(m) if m in NAME else m.split('/')[1] |
| p.append(tx(xs[0] - 36, Y(r0) + 6, lab, 17 if me else 16, 'end', 700 if me else 400, POS if me else INK)) |
| p.append(tx(xs[-1] + 36, Y(r5) + 6, SHORT[m], 16 if me else 15, 'start', 700 if me else 400, POS if me else MUTED)) |
| body = svgw(width, Y(len(mids)) + 24, p, 18) |
| firsts = sum(1 for c in COLS if rank[(OURS, c)] == 1) |
| return page('Rank of every model on every task', |
| 'Test split, max_seq_length 512 (round-1 models 256); 1 = best of the %d models. This model is first in %d of the 6 columns, ' |
| 'overall included; its lowest rank, %d, is on book titles.' % (len(mids), firsts, rank[(OURS, 'BookTitleRetrieval')]), |
| body, foot('test', ' · left labels in overall order, right labels in clustering order')) |
|
|
|
|
| CHARTS = [('00_scorecard', c00), ('01_test_overall_all_models', c01), ('02_test_per_task', c02), ('03_dev_candidates', c03), |
| ('04_round1_vs_round2', c04), ('05_size_vs_score', c05), ('06_booktitle_detail', c06), |
| ('07_task_heatmap_test', c07), ('08_task_heatmap_dev', c08), ('09_task_profile', c09), ('10_dev_vs_test', c10), |
| ('11_context_256_vs_512', c11), ('12_headline_by_source', c12), ('13_continuation_by_source', c13), |
| ('14_retrieval_depth', c14), ('15_secondary_metrics', c15), ('16_hard_negative_ablation', c16), ('17_training_loss', c17), |
| ('18_loss_by_task_type', c18), ('19_training_data_mix', c19), ('20_benchmark_sizes', c20), ('21_training_cost', c21), |
| ('22_improvement_path', c22), ('23_truncation', c23), ('24_rank_summary', c24)] |
|
|
|
|
| FONT_CSS = {} |
|
|
|
|
| def embed_fonts(html): |
| for m in re.finditer(r'<link[^>]+href="(https://fonts.googleapis.com/css2?[^"]+)"[^>]*>', html): |
| url = m.group(1).replace('&', '&') |
| if url not in FONT_CSS: |
| css = urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=30).read().decode() |
| for u in sorted(set(re.findall(r'url\((https://fonts.gstatic.com/[^)]+)\)', css))): |
| data = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=60).read() |
| css = css.replace(u, 'data:font/woff2;base64,' + base64.b64encode(data).decode()) |
| FONT_CSS[url] = css |
| html = html.replace(m.group(0), '<style>' + FONT_CSS[url] + '</style>') |
| return html |
|
|
|
|
| def main(): |
| args = [a for a in sys.argv[1:] if not a.startswith('--')] |
| keep = '--keep-html' in sys.argv |
| only = args[0].split(',') if args else None |
| os.makedirs(OUT, exist_ok=True) |
| os.makedirs(WORK, exist_ok=True) |
| env = dict(os.environ, TEMP=WORK, TMP=WORK) |
| try: |
| for name, fn in CHARTS: |
| if only and name[:2] not in only: |
| continue |
| html = os.path.join(WORK, name + '.html') |
| open(html, 'w', encoding='utf-8', newline='\n').write(embed_fonts(fn())) |
| png = os.path.join(OUT, name + '.png') |
| subprocess.run([CHROME, '--headless=new', '--disable-gpu', '--hide-scrollbars', '--force-device-scale-factor=2', |
| '--user-data-dir=' + os.path.join(WORK, 'chrome'), '--no-first-run', '--disable-crash-reporter', |
| '--window-size=%d,%d' % (W, H), '--virtual-time-budget=20000', '--screenshot=' + png, |
| 'file:///' + html.replace('\\', '/')], check=True, capture_output=True, timeout=300, env=env) |
| print(name) |
| finally: |
| if not keep: |
| shutil.rmtree(WORK, ignore_errors=True) |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
