HindkoEmbedBench v1.0 · %s%s' % (what, extra) def mavg(vals, w): h = w // 2 return [sum(vals[max(0, i - h):i + h + 1]) / len(vals[max(0, i - h):i + h + 1]) for i in range(len(vals))] def spread(ys, gap, lo=-1e9, hi=1e9): """Push label positions apart (keeping their order) so neighbours are at least `gap` apart.""" order = sorted(range(len(ys)), key=lambda i: ys[i]) out = list(ys) for a, b in zip(order, order[1:]): out[b] = max(out[b], out[a] + gap) over = out[order[-1]] - hi if over > 0: for i in order: out[i] -= over for a, b in zip(order[::-1], order[::-1][1:]): out[b] = min(out[b], out[a] - gap) under = lo - out[order[0]] if under > 0: for i in order: out[i] += under return out def hbar_rows(rows, x_lab, x0, x1, vmax, y, rowh, barh, fmt, lab_size=16, sub_size=14, name_w=None, val_size=16): """Horizontal bars. rows: (label, sub, value, colour, bold, value_text). Returns svg parts and the next y.""" parts = [] for lab, sub, val, c, bold, vt in rows: cy = y + rowh / 2 if lab: parts.append(tx(x_lab, cy + (0 if sub else 6), lab, lab_size, 'end', 700 if bold else 400, POS if bold else INK)) if sub: parts.append(tx(x_lab, cy + 17, sub, sub_size, 'end', fill=MUTED)) w = (x1 - x0) * val / vmax parts.append(rect(x0, cy - barh / 2, w, barh, c, 3)) parts.append(tx(x0 + w + 8, cy + val_size * .36, vt if vt is not None else fmt(val), val_size, 'start', 700 if bold else 400, POS if bold else INK)) y += rowh return parts, y # ------------------------------------------------------------------------------------------------ 07 / 08 heatmaps ROWSUB = {OURS: 'e5-large fine-tune · 512 tokens', R2B: 'runner-up on dev · 512', R2P: 'no mined negatives or same-book blocks · 512', R2S: '512 tokens', R2G: '512 tokens', R1: 'round-1 model · 256 tokens', R1S: 'round 1 · 256 tokens', TFIDF: 'lexical reference'} def heat(split, mids, rowh, fs, colw=168): vals = {m: [sc(m, split, c) for c in COLS] for m in mids} mids = sorted(mids, key=lambda m: -vals[m][0]) lo = [min(vals[m][i] for m in mids) for i in range(len(COLS))] hi = [max(vals[m][i] for m in mids) for i in range(len(COLS))] head = ''.join('%s%s' % ((colw,) + TASK_LABEL[c]) for c in COLS) trs = [] for m in mids: tds = [] for i, val in enumerate(vals[m]): t = (val - lo[i]) / (hi[i] - lo[i]) st = 'background:%s;height:%dpx;font-size:%dpx;color:#0F172A' % (mix('#F4F6F9', '#7EA5E8', t), rowh, fs) if val == hi[i]: st += ';box-shadow:inset 0 0 0 3px #0F172A;font-weight:700' tds.append('%s' % (st, f4(val))) if m in NAME: name, sub = NAME[m], ROWSUB[m] else: org, short_id = m.split('/') name, sub = short_id, '%s · untuned · 512' % org trs.append('%s%s%s' % (' class="me"' if m == OURS else '', fs, name, sub, ''.join(tds))) return '%s%s

Model

' % (head, ''.join(trs)) def c07(): mids = [OURS, R2B, R1, R1S, TFIDF] + PUBS return page('Every model on every task: test split', 'Test split, max_seq_length 512 (round-1 models 256). Shading runs from lowest to highest within each column; best outlined.', heat('test', mids, 44, 19), foot('test', ' · neural models at max_seq_length 512 unless noted')) def c08(): mids = CANDS + [R1, R1S, TFIDF] + PUBS return page('Every model on every task: dev split', 'Dev split, max_seq_length 512 (round-1 models 256). Shading runs from lowest to highest within each column; best outlined.', heat('dev', mids, 36, 16), foot('dev', ' · all five round-2 candidates, both round-1 models, TF-IDF and seven public models')) # ------------------------------------------------------------------------------------------------ 09 task profile def c09(): models = [(OURS, 'circle'), (TFIDF, 'diamond'), (BGE, 'square'), (E5L, 'triangle')] stroke = {OURS: '#FFFFFF', TFIDF: '#FFFFFF', BGE: '#7C8796', E5L: '#FFFFFF'} width, rowh, lane, top = 1440, 84, 17, 30 x0, x1, lo, hi = 250, 1100, 0.3, 1.0 X = lambda val: x0 + (x1 - x0) * (val - lo) / (hi - lo) bottom = top + rowh * len(COLS) p = [] for t in (0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0): p.append(ln(X(t), top - 4, X(t), bottom, GRID)) p.append(tx(X(t), bottom + 24, '%.1f' % t, 15, 'middle', fill=MUTED)) p.append(tx(1180, top - 10, 'Lead over the best other', 15, 'start', 600, MUTED)) for ri, c in enumerate(COLS): yc = top + rowh * ri + rowh / 2 lab, metric = TASK_LABEL[c] p.append(tx(0, yc - 1, lab, 21, weight=600)) p.append(tx(0, yc + 21, metric, 15, fill=MUTED)) if ri: p.append(ln(0, top + rowh * ri, 1440, top + rowh * ri, GRID)) for li, (m, shape) in enumerate(models): y = yc + (li - 1.5) * lane val = sc(m, 'test', c) me = m == OURS p.append(ln(x0, y, X(val), y, mix(col(m), '#FFFFFF', .55) if not me else '#9DB9EA', 2.5 if me else 1.5)) p.append(mark(shape, X(val), y, 8.5 if me else 7, col(m), stroke[m], 1.5 if m == BGE else 1.5)) p.append(tx(X(val) + 14, y + 5.5, f4(val), 16 if me else 15, 'start', 700 if me else 400, POS if me else INK)) others = max((sc(m, 'test', c), m) for m, _ in models[1:]) d = sc(OURS, 'test', c) - others[0] p.append(tx(1180, yc + 4, dlt(d), 24, 'start', 700, POS if d >= 0 else NEG)) p.append(tx(1290, yc + 4, 'vs ' + SHORT[others[1]], 16, 'start', fill=MUTED)) body = (legend_marks([('Hindko Embeddings (this model)', COL[OURS], 'circle', '#FFFFFF'), ('TF-IDF char 2-4-gram', COL[TFIDF], 'diamond', '#FFFFFF'), ('BAAI/bge-m3, best public model', COL[BGE], 'square', '#7C8796'), ('multilingual-e5-large, the untuned base model', COL[E5L], 'triangle', '#FFFFFF')], mt=18) + svgw(width, bottom + 34, p, 10)) return page('Task profile against the references', 'Test split, max_seq_length 512; one lane per model, axis from 0.3. This model leads on every task except book titles.', body, foot('test', ' · neural models at max_seq_length 512')) # ------------------------------------------------------------------------------------------------ 10 dev vs test def c10(): mids = [OURS, R2B, R1, R1S, TFIDF] + PUBS width, height = 1440, 560 x0, x1, y0, y1 = 90, 1000, 10, 500 dlo, dhi, tlo, thi = 0.40, 0.75, 0.40, 0.76 X = lambda d: x0 + (x1 - x0) * (d - dlo) / (dhi - dlo) Y = lambda t: y1 - (y1 - y0) * (t - tlo) / (thi - tlo) p = [] for t in (0.40, 0.45, 0.50, 0.55, 0.60, 0.65, 0.70, 0.75): p.append(ln(x0, Y(t), x1, Y(t), GRID) + tx(x0 - 10, Y(t) + 5, '%.2f' % t, 15, 'end', fill=MUTED)) p.append(ln(X(t), y0, X(t), y1, GRID) + tx(X(t), y1 + 24, '%.2f' % t, 15, 'middle', fill=MUTED)) p.append(tx((x0 + x1) / 2, y1 + 50, 'dev overall score', 16, 'middle', fill=MUTED)) p.append(tx(22, (y0 + y1) / 2, 'test overall score', 16, 'middle', fill=MUTED, extra=' transform="rotate(-90 22 %d)"' % ((y0 + y1) / 2))) p.append(ln(X(0.40), Y(0.40), X(0.75), Y(0.75), '#8E9AAB', 2, ' stroke-dasharray="7 6"')) ang = math.degrees(math.atan2(Y(0.40) - Y(0.75), X(0.75) - X(0.40))) p.append(tx(X(0.452), Y(0.452) - 12, 'test = dev', 15, 'start', fill=MUTED, extra=' transform="rotate(%.1f %.1f %.1f)"' % (-ang, X(0.452), Y(0.452) - 12))) up = sum(1 for m in mids if sc(m, 'test') > sc(m, 'dev')) p.append(tx(X(0.415), Y(0.735), 'Above the line: higher on test than on dev', 16, 'start', 600)) p.append(tx(X(0.415), Y(0.735) + 22, '%d of the %d models' % (up, len(mids)), 16, 'start', fill=MUTED)) # label placement: (dx, dy, anchor) next to the point, or a leader line to the right-hand column (column y) near = {BGE: (14, -4, 'start'), E5L: (14, 4, 'start'), E5B: (14, 30, 'middle'), QWEN: (10, 76, 'middle'), E5S: (-14, -10, 'end'), LABSE: (-14, -2, 'end'), MPNET: (14, -4, 'start')} column = {OURS: 30, R1S: 84, R2B: 138, R1: 192, TFIDF: 246} for m in mids: d, t = sc(m, 'dev'), sc(m, 'test') me = m == OURS name = SHORT[m] if m not in (E5B, E5S) else SHORT[m].replace(' (untuned)', '') vals = 'dev %s · test %s' % (f4(d), f4(t)) if m in column: ly = column[m] p.append(ln(X(d), Y(t), 1024, ly - 5, '#AEB7C3', 1.2)) p.append(tx(1032, ly, nm(m) if m != OURS else 'Hindko Embeddings (this model)', 17, 'start', 700 if me else 600, POS if me else INK)) p.append(tx(1032, ly + 20, vals + (' (%s)' % dlt(t - d)), 14, 'start', fill=MUTED)) else: dx, dy, anc = near[m] if dy > 40: p.append(ln(X(d), Y(t) + 9, X(d) + dx, Y(t) + dy - 16, '#AEB7C3', 1.2)) p.append(tx(X(d) + dx, Y(t) + dy, name, 16, anc, 600, INK, True)) p.append(tx(X(d) + dx, Y(t) + dy + 18, vals, 13, anc, fill=MUTED, halo=True)) for m in mids[::-1]: st = '#7C8796' if col(m) in (PUBCOL, COL[BGE]) else '#FFFFFF' p.append(mark('circle', X(sc(m, 'dev')), Y(sc(m, 'test')), 11 if m == OURS else 8, col(m), st, 1.5 if st != '#FFFFFF' else 2)) body = (legend_marks([(SHORT[m], col(m), 'circle', '#FFFFFF') for m in (OURS, R2B, R1, R1S, TFIDF)] + [(SHORT[E5L], COL[E5L], 'circle', '#FFFFFF'), ('bge-m3', COL[BGE], 'circle', '#7C8796'), ('other public models', PUBCOL, 'circle', '#7C8796')], mt=14) .replace('style="margin-top:14px"', 'style="margin-top:14px;gap:24px;font-size:18px"') + svgw(width, height, p, 6)) return page('Dev score against test score', 'Overall score on dev (x) and test (y) for every model scored on both. max_seq_length 512, round-1 models 256.', body, foot2('Dev and test splits', ' · neural models at max_seq_length 512 unless noted')) # ------------------------------------------------------------------------------------------------ 11 256 vs 512 tokens def c11(): mids = sorted(PUBS, key=lambda m: -sc(m, 'test')) width, top, rowh = 1440, 44, 60 name_w = 350 panels = [('test', 'Test split', 380, 850), ('dev', 'Dev split', 950, 1420)] lo, hi = 0.40, 0.66 p = [] height = top + rowh * len(mids) + 40 for split, title, a, b in panels: X = lambda v, a=a, b=b: a + (b - a - 120) * (v - lo) / (hi - lo) p.append(tx(a, 18, title, 21, 'start', 600)) for t in (0.40, 0.45, 0.50, 0.55, 0.60, 0.65): p.append(ln(X(t), top - 6, X(t), top + rowh * len(mids), GRID)) p.append(tx(X(t), top + rowh * len(mids) + 24, '%.2f' % t, 15, 'middle', fill=MUTED)) p.append(tx(b, 18, 'change', 15, 'end', 600, MUTED)) for i, m in enumerate(mids): y = top + rowh * i + rowh / 2 s256, s512 = sc(m, split, tab=M256), sc(m, split) up = s512 >= s256 p.append(ln(X(s256), y, X(s512), y, '#AEB7C3', 3)) p.append(mark('circle', X(s256), y, 8, '#FFFFFF', '#8E9AAB', 2.5)) p.append(mark('circle', X(s512), y, 8.5, '#334155', '#FFFFFF', 2)) lo_v, hi_v = (s256, s512) if up else (s512, s256) p.append(tx(X(lo_v) - 14, y + 5, f4(lo_v), 15, 'end', 700 if lo_v == s512 else 400, INK if lo_v == s512 else MUTED, True)) p.append(tx(X(hi_v) + 14, y + 5, f4(hi_v), 15, 'start', 700 if hi_v == s512 else 400, INK if hi_v == s512 else MUTED, True)) p.append(tx(b, y + 6, dlt(s512 - s256), 18, 'end', 700, POS if up else NEG)) for i, m in enumerate(mids): y = top + rowh * i + rowh / 2 org, sid = m.split('/') p.append(tx(name_w - 10, y - 1, sid, 17, 'end', 600)) p.append(tx(name_w - 10, y + 18, org, 14, 'end', fill=MUTED)) if i: p.append(ln(0, top + rowh * i, width, top + rowh * i, '#F0F2F5')) n_up = {s: sum(1 for m in PUBS if sc(m, s) > sc(m, s, tab=M256)) for s in ('test', 'dev')} body = (legend_marks([('256 tokens, float32 (round 1)', '#FFFFFF', 'circle', '#8E9AAB'), ('512 tokens, float16 (re-run for round 2)', '#334155', 'circle', '#FFFFFF')], mt=16) + svgw(width, height, p, 8) + '

Length and precision changed together, so their effects cannot be separated. ' 'Higher at 512: %d of 7 models on test, %d of 7 on dev. The other charts compare public models at 512.

' % (n_up['test'], n_up['dev'])) return page('Public models at 256 and at 512 tokens', 'Untuned public models, overall score: round 1 (max_seq_length 256, float32) and the re-run for round 2 (512, float16).', body, foot2('Dev and test splits', ' · untuned public models only')) # ------------------------------------------------------------------------------------------------ 12 headline by source def c12(): models = [OURS, R2B, R1, TFIDF, BGE] panels = [] for split in ('test', 'dev'): n = {s: M[(OURS, split)]['tasks']['HeadlineRetrieval']['by_source'][s]['n_queries'] for s in ('newspaper', 'web')} groups = [('Newspaper headlines', '%d queries' % n['newspaper']), ('Web page titles', '%d queries' % n['web'])] series = [(nm(m), col(m), [sc(m, split, 'HeadlineRetrieval', src=s) for s in ('newspaper', 'web')], m == OURS) for m in models] svg = grouped(groups, series, vmax=1.0, width=690, height=455, x0=50, bar_w=46, gap=8, lab_size=15) panels.append('

%sHeadlineRetrieval nDCG@10 · %d queries in all

' % ('Test' if split == 'test' else 'Dev', sum(n.values()), svg)) body = (legend2([(nm(m) + ('' if m in (OURS, TFIDF) else ' (512)' if m != R1 else ' (256)'), col(m)) for m in models]) + '

' % ''.join(panels)) return page('Headline retrieval by query source', 'nDCG@10 for newspaper headlines and web page titles, test and dev. max_seq_length 512 (round-1 model 256). ' 'The web subsets are tiny: one query can move them by up to %.3f (test) and %.3f (dev).' % tuple(1 / M[(OURS, s_)]['tasks']['HeadlineRetrieval']['by_source']['web']['n_queries'] for s_ in ('test', 'dev')), body, foot2('Test split (left) and dev split (right)', ' · key models')) # ------------------------------------------------------------------------------------------------ 13 continuation by source def c13(): models = [OURS, R2B, R1, TFIDF, BGE, E5L] srcs = ['book', 'newspaper', 'web'] bs = M[(OURS, 'test')]['tasks']['PassageContinuation'] groups = [('%s passages' % s.capitalize() if s != 'web' else 'Web pages', '%d queries' % bs['by_source'][s]['n_queries']) for s in srcs] groups.append(('All', '%d queries' % bs['n_queries'])) series = [(nm(m), col(m), [sc(m, 'test', 'PassageContinuation', src=s) for s in srcs] + [sc(m, 'test', 'PassageContinuation')], m == OURS) for m in models] body = (legend2([(SHORT[m] if m in (TFIDF, BGE, E5L) else nm(m), col(m)) for m in models], 17, 22) + grouped(groups, series, vmax=1.0, width=1440, height=500, bar_w=42, gap=9, lab_size=15)) lead = [s for s in srcs if all(sc(OURS, 'test', 'PassageContinuation', src=s) > sc(m, 'test', 'PassageContinuation', src=s) for m in models[1:])] lost = [s for s in srcs if s not in lead] beat = {s: [SHORT[m] for m in models[1:] if sc(m, 'test', 'PassageContinuation', src=s) > sc(OURS, 'test', 'PassageContinuation', src=s)] for s in lost} return page('Passage continuation by source', 'PassageContinuation nDCG@10 on the test split by source, max_seq_length 512 (round-1 model 256). This model leads on %s passages; %s.' % (' and '.join(lead), '; '.join('on the %d %s queries %s score%s higher' % (bs['by_source'][s]['n_queries'], s, ' and '.join(beat[s]), '' if len(beat[s]) > 1 else 's') for s in lost)), body, foot('test', ' · the first half of a passage retrieves its second half')) # ------------------------------------------------------------------------------------------------ 14 retrieval depth def c14(): models = [OURS, TFIDF, BGE] mets = [('recall_at_1', 'Recall@1'), ('mrr_at_10', 'MRR@10'), ('ndcg_at_10', 'nDCG@10'), ('recall_at_10', 'Recall@10')] panels = [] for task, title in (('HeadlineRetrieval', 'Headline'), ('BookTitleRetrieval', 'Book title'), ('PassageContinuation', 'Continuation')): r = M[(OURS, 'test')]['tasks'][task] sub = '%d queries' % r['n_queries'] + (' (57 relevant)' if task == 'BookTitleRetrieval' else '') p, y = [], 8 for key, lab in mets: p.append(tx(0, y + 46, lab, 18, 'start', 600)) rows = [('', '', sc(m, 'test', task, key), col(m), m == OURS, None) for m in models] parts, y = hbar_rows(rows, 0, 108, 390, 1.0, y, 30, 22, f3, val_size=15) p += parts y += 20 for t in (0.0, 0.5, 1.0): p.insert(0, ln(108 + 282 * t, 0, 108 + 282 * t, y - 14, GRID)) p.append(tx(108 + 282 * t, y + 6, '%.1f' % t, 14, 'middle', fill=MUTED)) panels.append('

%s%s

' % (title, sub, svgw(450, y + 10, p, 12))) body = (legend([(nm(m) + (' (512)' if m == BGE else ''), col(m)) for m in models]) + '

' % ''.join(panels)) return page('Retrieval at different depths', 'Test split, max_seq_length 512. Recall@1 asks for the right document first; Recall@10 for it anywhere in the top ten. ' 'Book titles can have several relevant passages, so Recall@1 is below hit@1 there.', body, foot('test', ' · neural models at max_seq_length 512')) # ------------------------------------------------------------------------------------------------ 15 secondary metrics def c15(): models = [OURS, R2B, R1, R1S, TFIDF, BGE, E5L] width, top, rowh, lane = 1440, 52, 58, 12 panels = [('VerseProseClassification', 'Verse / prose classification', [('f1_macro', 'f1_macro_std', 'macro-F1', 'circle'), ('accuracy', 'accuracy_std', 'accuracy', 'diamond')], 0.92, 1.00, (0.92, 0.94, 0.96, 0.98, 1.00), 330, 630, 650), ('BookClustering', 'Book clustering', [('v_measure', 'v_measure_std', 'V-measure', 'circle'), ('ari', None, 'ARI', 'diamond')], 0.2, 1.0, (0.2, 0.4, 0.6, 0.8, 1.0), 900, 1200, 1220)] bottom = top + rowh * len(models) p = [] for task, title, mets, lo, hi, ticks, a, b, vx in panels: X = lambda v, a=a, b=b, lo=lo, hi=hi: a + (b - a) * (v - lo) / (hi - lo) p.append(tx(a, 18, title, 21, 'start', 600)) p.append(tx(vx, top - 12, mets[0][2], 14, 'start', 600, MUTED) + tx(vx + 150, top - 12, mets[1][2], 14, 'start', 600, MUTED)) for t in ticks: p.append(ln(X(t), top - 4, X(t), bottom, GRID)) p.append(tx(X(t), bottom + 22, '%.2f' % t, 14, 'middle', fill=MUTED)) for i, m in enumerate(models): yc = top + rowh * i + rowh / 2 r = M[(m, 'test')]['tasks'][task] me = m == OURS for li, (k, ks, lab, shape) in enumerate(mets): y = yc + (li - .5) * 2 * lane v, s = r[k], (r[ks] if ks else None) if s: p.append(ln(X(max(lo, v - s)), y, X(min(hi, v + s)), y, mix(col(m), INK, .25) if m != BGE else '#7C8796', 2)) for e in (max(lo, v - s), min(hi, v + s)): p.append(ln(X(e), y - 5, X(e), y + 5, mix(col(m), INK, .25) if m != BGE else '#7C8796', 2)) p.append(mark(shape, X(v), y, 7 if me else 6, col(m), '#7C8796' if m == BGE else '#FFFFFF', 1.5)) txt = f3(v) + (' ± %s' % f3(s) if s is not None else '') p.append(tx(vx + (0 if li == 0 else 150), yc + 6, txt, 15, 'start', 700 if me else 400, POS if me else INK)) for i, m in enumerate(models): yc = top + rowh * i + rowh / 2 p.append(tx(300, yc + 6, nm(m) if m not in (BGE, E5L) else {BGE: 'BAAI/bge-m3', E5L: 'multilingual-e5-large'}[m], 17, 'end', 700 if m == OURS else 400, POS if m == OURS else INK)) if i: p.append(ln(0, top + rowh * i, width, top + rowh * i, '#F0F2F5')) body = (legend_marks([('main metric (macro-F1, V-measure)', '#8E9AAB', 'circle', '#FFFFFF'), ('second metric (accuracy, ARI)', '#8E9AAB', 'diamond', '#FFFFFF'), ('whiskers: ± one standard deviation over 10 runs', '#5B6576', 'line', '')], mt=16) + svgw(width, bottom + 30, p, 6) + '

The verse class of the test set comes almost entirely from one book. TF-IDF gives the same ' 'classification score in all 10 test runs (std 0). ARI has no spread recorded.

') return page('Classification and clustering in detail', 'Test split, max_seq_length 512 (round-1 models 256). 10 logistic-regression runs on 157 passages; 10 k-means runs on 176 passages from 5 books. ' 'Note the different axis ranges.', body, foot('test', ' · neural models at max_seq_length 512 unless noted')) # ------------------------------------------------------------------------------------------------ 16 hard-negative ablation def c16(): width, top, rowh = 1440, 56, 66 a, b, lim = 640, 1240, 0.015 X = lambda d: (a + b) / 2 + (b - a) / 2 * d / lim bottom = top + rowh * len(COLS) p = [rect(X(-0.01), top - 6, X(0.01) - X(-0.01), bottom - top + 6, '#F2F5F9', 0)] p.append(tx(300, top - 16, 'with', 15, 'end', 600, MUTED)) p.append(tx(300, top + 0, 'negatives + blocks', 13, 'end', fill=MUTED)) p.append(tx(440, top - 16, 'without', 15, 'end', 600, MUTED)) p.append(tx(440, top + 0, '(plain)', 13, 'end', fill=MUTED)) p.append(tx((a + b) / 2, top - 16, 'with minus without', 15, 'middle', 600, MUTED)) for t in (-0.015, -0.01, -0.005, 0, 0.005, 0.01, 0.015): p.append(ln(X(t), top - 6, X(t), bottom, INK if t == 0 else GRID, 1.5 if t == 0 else 1)) p.append(tx(X(t), bottom + 22, ('%+.3f' % t if t else '0').replace('-', '−'), 14, 'middle', fill=MUTED)) for i, c in enumerate(COLS): y = top + rowh * i + rowh / 2 lab, metric = TASK_LABEL[c] w_, wo = sc(R2B, 'dev', c), sc(R2P, 'dev', c) d = w_ - wo p.append(tx(0, y - 1, lab, 20, 'start', 600)) p.append(tx(0, y + 19, metric, 14, 'start', fill=MUTED)) p.append(tx(300, y + 7, f4(w_), 19, 'end', 600)) p.append(tx(440, y + 7, f4(wo), 19, 'end')) x0, x1 = sorted((X(0), X(d))) p.append(rect(x0, y - 15, x1 - x0, 30, '#6F95DE' if d >= 0 else '#D98A75', 3)) p.append(tx(X(d) + (10 if d >= 0 else -10), y + 7, dlt(d), 19, 'start' if d >= 0 else 'end', 700, POS if d >= 0 else NEG)) if i: p.append(ln(0, top + rowh * i, width, top + rowh * i, '#F0F2F5')) body = (svgw(width, bottom + 34, p, 22) + '

Shaded: ±0.01, about one standard error of the dev overall score (estimate from the training plan); ' 'single tasks are noisier. One seed per run, so none of these differences is established. The recipe gate (fixed in advance) ' 'kept mined negatives and same-book blocks for the larger runs.

') return page('Ablation: mined hard negatives and same-book batches', 'Dev split, max_seq_length 512. Round-2 e5-base trained twice on the same pairs: with mined hard negatives and same-book blocks of 32, ' 'and without both (in-batch negatives only).', body, foot('dev', ' · Round-2 e5-base vs Round-2 e5-base, no hard negatives')) # ------------------------------------------------------------------------------------------------ 17 training loss def c17(): width, height = 1440, 520 x0, x1, y0, y1, ymax = 60, 1100, 10, 470, 6.5 X = lambda s: x0 + (x1 - x0) * s / 321 Y = lambda v: y1 - (y1 - y0) * v / ymax warm = TRAIN[OURS]['training_args']['warmup_steps'] p = [rect(X(0), y0, X(warm) - X(0), y1 - y0, '#F2F5F9', 0), tx(X(warm / 2), y0 + 18, 'warm-up', 14, 'middle', fill=MUTED)] for t in range(0, 7): p.append(ln(x0, Y(t), x1, Y(t), GRID) + tx(x0 - 10, Y(t) + 5, str(t), 15, 'end', fill=MUTED)) for s in (0, 50, 100, 150, 200, 250, 300, 321): p.append(tx(X(s), y1 + 24, str(s), 15, 'middle', fill=MUTED)) ep = TRAIN[OURS]['steps'] // TRAIN[OURS]['training_args']['num_train_epochs'] p.append(ln(X(ep - .5), y0, X(ep - .5), y1, INK, 1.5, ' stroke-dasharray="6 5"')) p.append(tx(X(ep - .5) + 8, y0 + 18, 'epoch 2 starts (step %d)' % ep, 15, 'start', 600)) p.append(tx((x0 + x1) / 2, y1 + 48, 'optimizer step (batch of 256 rows from one dataset)', 15, 'middle', fill=MUTED)) order = [R2P, R2S, R2G, R2B, OURS] ends = {} for m in order: vals = [v for _, _, v in sorted(LOSS[m])] p.append('' % (' '.join('%.1f,%.1f' % (X(i), Y(v)) for i, v in enumerate(vals)), col(m))) for m in order: vals = [v for _, _, v in sorted(LOSS[m])] ma = mavg(vals, 15) ends[m] = ma[-1] p.append('' % (' '.join('%.1f,%.1f' % (X(i), Y(v)) for i, v in enumerate(ma)), col(m), 4 if m == OURS else 2.5)) ys = spread([Y(ends[m]) for m in order], 48, y0 + 10, y1) for m, ly in zip(order, ys): last = [v for _, _, v in sorted(LOSS[m])][-15:] p.append(ln(X(321) + 4, Y(ends[m]), 1122, ly - 7, col(m), 2)) p.append(rect(1122, ly - 13, 18, 12, col(m), 2)) # colour key of the run p.append(tx(1148, ly - 2, nm(m), 16, 'start', 700 if m == OURS else 600, POS if m == OURS else INK)) p.append(tx(1148, ly + 17, 'last 15 steps %.3f · all %.3f' % (sum(last) / len(last), sum(v for _, _, v in LOSS[m]) / len(LOSS[m])), 13, 'start', fill=MUTED)) body = (legend_marks([('15-step moving average', '#5B6576', 'line', ''), ('loss at every step (faint)', '#C9D0DA', 'line', '')], mt=14) + svgw(width, height, p, 6)) return page('Training loss of the five round-2 runs', 'CachedMultipleNegativesRankingLoss per optimizer step, max_seq_length 512, 2 epochs of 161 steps. Each step is one dataset, so the raw ' 'loss jumps with the dataset; runs without mined negatives have fewer candidates per anchor and a lower loss.', body, foot2('Round-2 training on the train split', ' · one NVIDIA A100-SXM4-80GB · seed 20261001'), eyebrow='Training') # ------------------------------------------------------------------------------------------------ 18 loss by dataset type def c18(): width, height = 920, 520 x0, x1, y0, y1, ymax = 50, 760, 10, 470, 6.5 X = lambda s: x0 + (x1 - x0) * s / 321 Y = lambda v: y1 - (y1 - y0) * v / ymax p = [] for t in range(0, 7): p.append(ln(x0, Y(t), x1, Y(t), GRID) + tx(x0 - 10, Y(t) + 5, str(t), 15, 'end', fill=MUTED)) for s in (0, 100, 200, 321): p.append(tx(X(s), y1 + 24, str(s), 15, 'middle', fill=MUTED)) p.append(ln(X(160.5), y0, X(160.5), y1, INK, 1.5, ' stroke-dasharray="6 5"') + tx(X(160.5) + 8, y0 + 16, 'epoch 2', 14, 'start', 600)) p.append(tx((x0 + x1) / 2, y1 + 48, 'optimizer step', 15, 'middle', fill=MUTED)) ends = {} for ty in TYPES: pts = [(s, v) for s, d, v in sorted(LOSS[OURS]) if d == ty] ma = mavg([v for _, v in pts], 5) ends[ty] = (pts[-1][0], ma[-1], len(pts), sum(v for _, v in pts[-5:]) / 5) p.append(''.join('' % (X(s), Y(v), TYPECOL[ty]) for s, v in pts)) p.append('' % (' '.join('%.1f,%.1f' % (X(s), Y(v)) for (s, _), v in zip(pts, ma)), TYPECOL[ty])) ys = spread([Y(ends[ty][1]) for ty in TYPES], 46, y0 + 10, y1) for ty, ly in zip(TYPES, ys): s, v, n, last5 = ends[ty] p.append(ln(X(s) + 4, Y(v), 776, ly - 6, '#AEB7C3', 1)) p.append(tx(782, ly - 1, ty, 17, 'start', 700)) p.append(tx(782, ly + 17, 'last 5: %.2f · %d steps' % (last5, n), 13, 'start', fill=MUTED)) chart = svgw(width, height, p, 6) cand = [OURS, R2B, R2S, R2G, R2P] head = ''.join('' '%s' % (TYPECOL[t], t) for t in TYPES) rows = [] for m in cand: ml = TRAIN[m]['mean_loss_per_dataset'] me = m == OURS rows.append('%s%s' % (' class="me"' if me else '', SHORT[m] if m != R2P else 'Round-2 e5-base,
no hard negatives', ''.join('%.2f' % ml[t] for t in TYPES))) table = ('

Mean loss per datasetall steps

' '%s%s

Run

' '

Title and verse rows also carry mined negatives from the same book, and their loss stays the highest. ' 'Without mined negatives (last row) the mean title loss is %.2f, against %.2f with them (Round-2 e5-base).

' % (head, ''.join(rows), TRAIN[R2P]['mean_loss_per_dataset']['title'], TRAIN[R2B]['mean_loss_per_dataset']['title'])) body = ('

%s%s

' % (legend_marks([('5-point moving average per dataset', '#5B6576', 'line', ''), ('loss at each step of that dataset', '#AEB7C3', 'circle', '#FFFFFF')], mt=4), chart, table)) return page('This model\'s loss by training dataset', 'Hindko Embeddings (round-2 e5-large), max_seq_length 512: loss at each step, split by the dataset the batch came from. ' 'Right: the mean loss of each dataset for all five round-2 runs.', body, foot2('Round-2 training on the train split', ' · batches of 256 rows from one dataset each'), eyebrow='Training') # ------------------------------------------------------------------------------------------------ 19 training-data mix def c19(): ts = TDATA['rows_by_type_source'] tv = TDATA['rows_by_ds_type_variety'] rows = [('title', 'title_any', 'table-of-contents title → passage'), ('title', 'verse_first', 'first line of a poem → the rest'), ('verse_ict', 'verse_line', 'one verse line → the poem without it'), ('prose_q', 'headline', 'headline or page title → article'), ('prose_q', 'prose_sent', 'one sentence → its surrounding text'), ('continuation', 'continuation', 'first half → second half'), ('neighbour', 'neighbour', 'a chunk → the next chunk')] p = [tx(0, 18, 'By pair type', 21, 'start', 600), tx(150, 18, 'coloured by dataset', 15, 'start', fill=MUTED)] y, x0, x1, vmax = 40, 330, 640, 15000 for ds, ty, desc in rows: n = TDATA['rows_by_ds_type']['%s/%s' % (ds, ty)] src = [(s, ts.get('%s/%s' % (ty, s), 0)) for s in ('book', 'newspaper', 'web')] var = [(v, tv.get('%s/%s/%s' % (ds, ty, v), 0)) for v in ('hindko', 'urdu', 'mixed')] cy = y + 22 p.append(tx(x0 - 14, cy - 2, ty + ' (%s)' % (MUTED, ds), 18, 'end', 600)) p.append(tx(x0 - 14, cy + 17, desc, 14, 'end', fill=MUTED)) w = (x1 - x0) * n / vmax p.append(rect(x0, cy - 13, w, 24, TYPECOL[ds], 3)) p.append(tx(x0 + w + 8, cy + 6, thou(n), 17, 'start', 700)) p.append(tx(x0, cy + 30, ' · '.join('%s %s' % (s, thou(c)) for s, c in src if c) + ' | ' + ' · '.join('%s %s' % (v, thou(c)) for v, c in var if c), 13, 'start', fill=MUTED)) y += 66 total = TDATA['n_rows'] p.append(tx(x0 - 14, y + 18, 'Total', 18, 'end', 700) + tx(x0, y + 18, '%s rows · %s distinct texts' % (thou(total), thou(TDATA['n_texts'])), 18, 'start', 700)) left = svgw(800, y + 30, p, 4) def summary(title, items): q = [tx(0, 18, title, 21, 'start', 600)] yy = 40 mx = max(c for _, c, _ in items) for lab, c, colr in items: q.append(tx(150, yy + 22, lab, 17, 'end')) w = 200 * c / mx q.append(rect(162, yy + 6, w, 22, colr, 3)) q.append(tx(162 + w + 8, yy + 23, '%s %.1f%%' % (thou(c), MUTED, 100 * c / total), 16, 'start', 600)) yy += 34 return svgw(560, yy + 4, q, 4) by_src = {s: sum(c for k, c in ts.items() if k.endswith('/' + s)) for s in ('book', 'newspaper', 'web')} by_var = {v: sum(c for k, c in tv.items() if k.endswith('/' + v)) for v in ('hindko', 'urdu', 'mixed')} by_ds = TDATA['rows_by_ds'] assert sum(by_src.values()) == sum(by_var.values()) == sum(by_ds.values()) == total right = (summary('By dataset', [(d, by_ds[d], TYPECOL[d]) for d in TYPES]) + summary('By source', [(s, by_src[s], '#7D8899') for s in ('book', 'newspaper', 'web')]) + summary('By variety of the record', [(v, by_var[v], '#7D8899') for v in ('hindko', 'urdu', 'mixed')])) body = '

' % (left, right) return page('What the model was trained on', 'The %s round-2 training rows (anchor and positive, from the train split only), by pair type, source and variety. ' 'Under each bar: rows by source | by variety.' % thou(total), body, foot2('Training pairs, train split', ' · leak-checked against every dev and test text'), eyebrow='Training') # ------------------------------------------------------------------------------------------------ 20 benchmark sizes def c20(): B = lambda task, split: BENCH['%s.%s' % (task, split)] hq = {s: B('HeadlineRetrieval', s)['query_sources'] for s in ('dev', 'test')} rows = [('Headline', 'queries', 'dev: news %d + web %d · test: news %d + web %d' % (hq['dev']['newspaper'], hq['dev']['web'], hq['test']['newspaper'], hq['test']['web']), B('HeadlineRetrieval', 'dev')['sizes']['queries'], B('HeadlineRetrieval', 'test')['sizes']['queries']), ('', 'documents', 'retrieval corpus, shared with book titles', B('HeadlineRetrieval', 'dev')['sizes']['corpus'], B('HeadlineRetrieval', 'test')['sizes']['corpus']), ('Book title', 'queries', 'dev: %d books · test: %d books' % (B('BookTitleRetrieval', 'dev')['n_books'], B('BookTitleRetrieval', 'test')['n_books']), B('BookTitleRetrieval', 'dev')['sizes']['queries'], B('BookTitleRetrieval', 'test')['sizes']['queries']), ('', 'relevant passages', '', B('BookTitleRetrieval', 'dev')['relevant_docs_total'], B('BookTitleRetrieval', 'test')['relevant_docs_total']), ('Continuation', 'queries = documents', '', B('PassageContinuation', 'dev')['sizes']['queries'], B('PassageContinuation', 'test')['sizes']['queries']), ('Verse / prose', 'eval passages', 'verse + prose: %d + %d (dev) · %d + %d (test)' % (B('VerseProseClassification', 'dev')['eval']['verse']['n'], B('VerseProseClassification', 'dev')['eval']['prose']['n'], B('VerseProseClassification', 'test')['eval']['verse']['n'], B('VerseProseClassification', 'test')['eval']['prose']['n']), B('VerseProseClassification', 'dev')['sizes']['eval'], B('VerseProseClassification', 'test')['sizes']['eval']), ('', 'training pool', 'from the train split, same for both', B('VerseProseClassification', 'dev')['sizes']['train'], B('VerseProseClassification', 'test')['sizes']['train']), ('Book clustering', 'passages', '', B('BookClustering', 'dev')['sizes']['items'], B('BookClustering', 'test')['sizes']['items']), ('', 'books (clusters)', '', B('BookClustering', 'dev')['n_books'], B('BookClustering', 'test')['n_books'])] width, top, rowh = 1440, 30, 56 x0, x1, vmax = 470, 1240, 3000 X = lambda v: x0 + (x1 - x0) * math.log10(v) / math.log10(vmax) bottom = top + rowh * len(rows) p = [tx(x1 + 30, top + rowh * len(rows) + 22, 'count, log scale', 14, 'start', 600, MUTED)] for t in (1, 3, 10, 30, 100, 300, 1000, 3000): p.append(ln(X(t), top - 6, X(t), bottom, GRID) + tx(X(t), bottom + 22, thou(t), 14, 'middle', fill=MUTED)) for i, (task, what, note, dv, tv) in enumerate(rows): y = top + rowh * i + rowh / 2 if task: p.append(tx(0, y + 1, task, 19, 'start', 600)) if i: p.append(ln(0, top + rowh * i, width, top + rowh * i, GRID)) p.append(tx(x0 - 30, y + (0 if note else 6), what, 17, 'end')) if note: p.append(tx(x0 - 30, y + 18, note, 13, 'end', fill=MUTED)) p.append(ln(X(min(dv, tv)), y, X(max(dv, tv)), y, '#AEB7C3', 3)) p.append(mark('circle', X(dv), y, 8.5, '#334155', '#FFFFFF', 2)) p.append(mark('diamond', X(tv), y, 8, '#E3A33B', '#FFFFFF', 1.5)) if dv == tv: p.append(tx(X(dv) + 16, y + 6, '%s (dev and test)' % thou(dv), 16, 'start', 600)) else: hn, hv, lname, lv = ('dev', dv, 'test', tv) if dv > tv else ('test', tv, 'dev', dv) p.append(tx(X(hv) + 16, y + 6, '%s %s' % (hn, thou(hv)), 16, 'start', 600)) p.append(tx(X(lv) - 16, y + 6, '%s %s' % (lname, thou(lv)), 16, 'end', 600)) body = (legend_marks([('dev split (model selection)', '#334155', 'circle', '#FFFFFF'), ('test split (scored once)', '#E3A33B', 'diamond', '#FFFFFF')], mt=12) + svgw(width, bottom + 30, p, 4)) return page('How big each benchmark task is', 'Dev and test splits (sizes do not depend on max_seq_length). The test book side is small: 28 book-title queries, 5 books.', body, foot2('Dev and test splits', ' · built automatically from corpus metadata; group-disjoint from the training data')) # ------------------------------------------------------------------------------------------------ 21 training cost def c21(): runs = sorted(CANDS, key=lambda m: -sc(m, 'dev')) params = {OURS: PARAMS[E5L], R2B: PARAMS[E5B], R2P: PARAMS[E5B], R2S: PARAMS[E5S], R2G: PARAMS[BGE]} width, top, rowh = 1440, 56, 78 panels = [('Training time', 'minutes', lambda m: TRAIN[m]['seconds'] / 60, 30, lambda v: '%.1f min' % v), ('Seconds per step', '322 steps', lambda m: TRAIN[m]['s_per_step'], 6, lambda v: '%.2f s' % v), ('Peak GPU memory', 'allocated, GB', lambda m: TRAIN[m]['mem_alloc_gb'], 30, lambda v: '%.1f GB' % v)] p = [] bottom = top + rowh * len(runs) for pi, (title, unit, f, vmax, fmt) in enumerate(panels): a = 340 + pi * 280 p.append(tx(a, 18, title, 20, 'start', 600) + tx(a, 40, unit, 14, 'start', fill=MUTED)) for i, m in enumerate(runs): y = top + rowh * i + rowh / 2 v = f(m) w = 180 * v / vmax p.append(rect(a, y - 13, w, 26, col(m), 3)) p.append(tx(a + w + 8, y + 6, fmt(v), 16, 'start', 700 if m == OURS else 400, POS if m == OURS else INK)) a, b, lo, hi = 1200, 1420, 0.695, 0.715 X = lambda v: a + (b - a) * (v - lo) / (hi - lo) p.append(tx(a, 18, 'Dev overall', 20, 'start', 600) + tx(a, 40, 'axis 0.695 to 0.715', 14, 'start', fill=MUTED)) for t in (0.695, 0.705, 0.715): p.append(ln(X(t), top - 4, X(t), bottom, GRID) + tx(X(t), bottom + 20, '%.3f' % t, 13, 'middle', fill=MUTED)) for i, m in enumerate(runs): y = top + rowh * i + rowh / 2 v = sc(m, 'dev') p.append(mark('circle', X(v), y - 6, 9 if m == OURS else 7.5, col(m), '#FFFFFF', 2)) p.append(tx(X(v), y + 22, f4(v), 15, 'middle', 700 if m == OURS else 400, POS if m == OURS else INK)) for i, m in enumerate(runs): y = top + rowh * i + rowh / 2 me = m == OURS p.append(tx(310, y - 2, nm(m), 18, 'end', 700 if me else 600, POS if me else INK)) p.append(tx(310, y + 18, '%dM params · mini-batch %d · lr %s' % (params[m], TRAIN[m]['mini'], ('%g' % TRAIN[m]['lr']).replace('e-05', 'e-5')), 14, 'end', fill=MUTED)) if i: p.append(ln(0, top + rowh * i, width, top + rowh * i, '#F0F2F5')) gpu = TRJ['runs'][OURS]['gpu'] body = (svgw(width, bottom + 30, p, 22) + '

Same data, 2 epochs, batch 256 and max_seq_length 512 for every run. The gradient cache splits each batch of 256 ' 'into mini-batches, so peak memory depends on the mini-batch size as well as on the model. Peak memory reserved for this model: %.1f GB.

' % TRAIN[OURS]['mem_reserved_gb']) return page('What each round-2 run cost', 'Training time, speed and peak GPU memory of the five round-2 runs on one %s, next to their dev score ' '(dev split, max_seq_length 512).' % gpu, body, foot2('Round-2 training runs', ' · dev scores from the dev split'), eyebrow='Training') # ------------------------------------------------------------------------------------------------ 22 improvement path def c22(): a = [sc(E5L, 'test', c) for c in COLS] r1 = [sc(R1, 'test', c) for c in COLS] b = [sc(OURS, 'test', c) for c in COLS] series = [('multilingual-e5-large, untuned (512 tokens)', COL[E5L], a, False), ('Round-1 e5-base fine-tune (256 tokens)', COL[R1], r1, False), ('Hindko Embeddings, this model (e5-large fine-tune, 512 tokens)', COL[OURS], b, True)] def below(gi, gx, gw, y1): d1 = b[gi] - a[gi] d2 = b[gi] - r1[gi] return ''.join('%s %s' % (gx + gw / 2, y1 + yy, POS if d >= 0 else NEG, dlt(d, 3), lab) for d, lab, yy in ((d1, 'vs untuned', 76), (d2, 'vs round 1', 98))) body = (legend([(n, c) for n, c, _, _ in series]) + grouped([TASK_LABEL[c] for c in COLS], series, height=515, below=below, bar_w=56, gap=8, lab_size=15, bottom=108)) return page('From the base model to this model', 'Test split. Untuned multilingual-e5-large at 512 tokens, the round-1 model (e5-base, 256 tokens) and this model (e5-large, 512 tokens). ' 'The round-1 model has a different base, so the middle step is a reference, not an ancestor.', body, foot('test', ' · changes below each group are this model minus the other two')) # ------------------------------------------------------------------------------------------------ 23 truncation def c23(): tasks = [('HeadlineRetrieval', 'Headline', 'queries + shared corpus'), ('BookTitleRetrieval', 'Book title', 'queries only'), ('PassageContinuation', 'Continuation', 'queries + documents'), ('VerseProseClassification', 'Verse / prose', 'train pool + eval'), ('BookClustering', 'Clustering', 'passages not yet encoded'), ('all', 'All texts', 'every text of the run')] width, top, rowh = 1440, 44, 70 panels = [('test', 'Test split', 330, 840), ('dev', 'Dev split', 910, 1420)] p = [] shares = {} for split, title, a, b in panels: p.append(tx(a, 18, title, 21, 'start', 600)) vmax = 0.6 X = lambda v, a=a: a + 300 * v / vmax for t in (0, 0.2, 0.4, 0.6): p.append(ln(X(t), top - 4, X(t), top + rowh * len(tasks), GRID) + tx(X(t), top + rowh * len(tasks) + 20, '%d%%' % (100 * t), 13, 'middle', fill=MUTED)) for i, (task, _, _) in enumerate(tasks): y = top + rowh * i + rowh / 2 for j, (m, msl) in enumerate(((R1, 256), (OURS, 512))): tk = M[(m, split)]['tasks'] assert M[(m, split)]['msl'] == msl if task == 'all': enc = sum(tk[t]['texts_encoded'] for t in TASKS) tr = sum(tk[t]['texts_truncated'] for t in TASKS) else: enc, tr = tk[task]['texts_encoded'], tk[task]['texts_truncated'] v = tr / enc if enc else 0 shares[(split, task, msl)] = (tr, enc) yy = y - 12 + j * 25 p.append(rect(X(0), yy - 10, X(v) - X(0), 20, col(m), 3)) p.append(tx(X(v) + 8, yy + 6, '%.1f%% %s of %s' % (100 * v, MUTED, thou(tr), thou(enc)), 15, 'start', 700 if msl == 512 else 600, POS if msl == 512 else INK)) for i, (task, lab, sub) in enumerate(tasks): y = top + rowh * i + rowh / 2 p.append(tx(300, y - 1, lab, 19, 'end', 700 if task == 'all' else 600)) p.append(tx(300, y + 18, sub, 14, 'end', fill=MUTED)) if i: p.append(ln(0, top + rowh * i, width, top + rowh * i, '#F0F2F5')) body = (legend([('256 tokens: Round-1 e5-base fine-tune', COL[R1]), ('512 tokens: Hindko Embeddings (this model)', COL[OURS])]) + svgw(width, top + rowh * len(tasks) + 30, p, 10) + '

Same XLM-RoBERTa tokenizer and prefixes in both runs, so only max_seq_length differs.%s ' 'Each text is encoded once per run and counted under the first task that needs it.

' % (' At 512 no continuation text is cut.' if all(shares[(s, 'PassageContinuation', 512)][0] == 0 for s in ('dev', 'test')) else '')) share = lambda s, msl: 100 * shares[(s, 'all', msl)][0] / shares[(s, 'all', msl)][1] return page('Why the length went from 256 to 512 tokens', 'Share of texts the model cuts off. All texts: %.1f%% at 256 vs %.1f%% at 512 tokens on dev, %.1f%% vs %.1f%% on test.' % (share('dev', 256), share('dev', 512), share('test', 256), share('test', 512)), body, foot2('Dev and test splits', ' · texts_truncated / texts_encoded from the evaluation runs')) # ------------------------------------------------------------------------------------------------ 24 rank summary def c24(): mids = [OURS, R2B, R1, R1S, TFIDF] + PUBS rank = {} for c in COLS: vals = {m: round(sc(m, 'test', c), 4) for m in mids} for m in mids: rank[(m, c)] = 1 + sum(1 for o in mids if vals[o] > vals[m]) width, top, rowh = 1440, 56, 44 xs = [420 + i * 150 for i in range(len(COLS))] Y = lambda r: top + (r - 1) * rowh p = [] for x, c in zip(xs, COLS): lab, metric = TASK_LABEL[c] p.append(tx(x, 14, lab, 18, 'middle', 600) + tx(x, 34, metric, 13, 'middle', fill=MUTED)) for r in range(1, len(mids) + 1): p.append(ln(xs[0] - 30, Y(r), xs[-1] + 30, Y(r), '#F0F2F5')) order = [m for m in mids if m not in (OURS,)][::-1] + [OURS] for m in order: me = m == OURS c_ = col(m) if m not in PUBS or m == BGE else '#D5DAE1' w = 5 if me else (3 if m in (TFIDF, BGE, R2B, R1, R1S) else 2) p.append('' % (' '.join('%.1f,%.1f' % (x, Y(rank[(m, c)])) for x, c in zip(xs, COLS)), c_ if m != BGE else '#AEB7C3', w, '' if me or m not in PUBS else ' stroke-opacity="0.9"')) for m in order: me = m == OURS fill = col(m) if m in COL else PUBCOL ink = '#FFFFFF' if m in (OURS, E5L) else INK for x, c in zip(xs, COLS): r = rank[(m, c)] p.append(mark('circle', x, Y(r), 15 if me else 13, fill, '#FFFFFF' if fill not in (PUBCOL, COL[BGE]) else '#8E9AAB', 2 if fill not in (PUBCOL, COL[BGE]) else 1.5)) p.append(tx(x, Y(r) + 5, str(r), 14 if me else 13, 'middle', 700, ink)) for m in mids: me = m == OURS r0, r5 = rank[(m, 'overall')], rank[(m, COLS[-1])] lab = nm(m) if m in NAME else m.split('/')[1] p.append(tx(xs[0] - 36, Y(r0) + 6, lab, 17 if me else 16, 'end', 700 if me else 400, POS if me else INK)) p.append(tx(xs[-1] + 36, Y(r5) + 6, SHORT[m], 16 if me else 15, 'start', 700 if me else 400, POS if me else MUTED)) body = svgw(width, Y(len(mids)) + 24, p, 18) firsts = sum(1 for c in COLS if rank[(OURS, c)] == 1) return page('Rank of every model on every task', 'Test split, max_seq_length 512 (round-1 models 256); 1 = best of the %d models. This model is first in %d of the 6 columns, ' 'overall included; its lowest rank, %d, is on book titles.' % (len(mids), firsts, rank[(OURS, 'BookTitleRetrieval')]), body, foot('test', ' · left labels in overall order, right labels in clustering order')) CHARTS = [('00_scorecard', c00), ('01_test_overall_all_models', c01), ('02_test_per_task', c02), ('03_dev_candidates', c03), ('04_round1_vs_round2', c04), ('05_size_vs_score', c05), ('06_booktitle_detail', c06), ('07_task_heatmap_test', c07), ('08_task_heatmap_dev', c08), ('09_task_profile', c09), ('10_dev_vs_test', c10), ('11_context_256_vs_512', c11), ('12_headline_by_source', c12), ('13_continuation_by_source', c13), ('14_retrieval_depth', c14), ('15_secondary_metrics', c15), ('16_hard_negative_ablation', c16), ('17_training_loss', c17), ('18_loss_by_task_type', c18), ('19_training_data_mix', c19), ('20_benchmark_sizes', c20), ('21_training_cost', c21), ('22_improvement_path', c22), ('23_truncation', c23), ('24_rank_summary', c24)] FONT_CSS = {} def embed_fonts(html): for m in re.finditer(r']+href="(https://fonts.googleapis.com/css2?[^"]+)"[^>]*>', html): url = m.group(1).replace('&', '&') if url not in FONT_CSS: css = urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=30).read().decode() for u in sorted(set(re.findall(r'url\((https://fonts.gstatic.com/[^)]+)\)', css))): data = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=60).read() css = css.replace(u, 'data:font/woff2;base64,' + base64.b64encode(data).decode()) FONT_CSS[url] = css html = html.replace(m.group(0), '') return html def main(): args = [a for a in sys.argv[1:] if not a.startswith('--')] keep = '--keep-html' in sys.argv only = args[0].split(',') if args else None os.makedirs(OUT, exist_ok=True) os.makedirs(WORK, exist_ok=True) env = dict(os.environ, TEMP=WORK, TMP=WORK) try: for name, fn in CHARTS: if only and name[:2] not in only: continue html = os.path.join(WORK, name + '.html') open(html, 'w', encoding='utf-8', newline='\n').write(embed_fonts(fn())) png = os.path.join(OUT, name + '.png') subprocess.run([CHROME, '--headless=new', '--disable-gpu', '--hide-scrollbars', '--force-device-scale-factor=2', '--user-data-dir=' + os.path.join(WORK, 'chrome'), '--no-first-run', '--disable-crash-reporter', '--window-size=%d,%d' % (W, H), '--virtual-time-budget=20000', '--screenshot=' + png, 'file:///' + html.replace('\\', '/')], check=True, capture_output=True, timeout=300, env=env) print(name) finally: if not keep: shutil.rmtree(WORK, ignore_errors=True) if __name__ == '__main__': main()