paper / index.html
xcomet-tr's picture
Remove tokenizer example from method section
279ced1 verified
Raw History Blame Contribute Delete
34.2 kB
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>TurCOMET · EMNLP 2026</title>
<meta name="description" content="TurCOMET: A Morphologically-Aware Evaluation System for Document-Level Turkish-English Machine Translation. Full paper, poster, results and resources (EMNLP 2026).">
<meta property="og:title" content="TurCOMET · EMNLP 2026">
<meta property="og:description" content="A morphologically-aware evaluation system for document-level Turkish↔English MT. Full paper, poster and resources.">
<meta property="og:image" content="https://huggingface.co/spaces/TurCOMET/paper/resolve/main/img/banner.png">
<meta name="twitter:card" content="summary_large_image">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
<script>
// Hugging Face passes the viewer's theme to Space iframes as ?__theme=light|dark
(function () {
try {
var t = new URLSearchParams(location.search).get('__theme');
if (t === 'light' || t === 'dark') document.documentElement.setAttribute('data-theme', t);
} catch (e) {}
})();
</script>
<style>
:root {
--bg: #F6F7F9;
--surface: #FFFFFF;
--surface-2: #F1F4F8;
--text: #0F172A;
--muted: #475569;
--faint: #64748B;
--border: #E2E8F0;
--accent: #D21F3C;
--accent-fill: #F0374B;
--accent-soft: rgba(240, 55, 75, .09);
--link: #1D4ED8;
--viewer-bg: #E5E9F0;
--shadow: 0 1px 2px rgba(15, 23, 42, .06), 0 8px 24px rgba(15, 23, 42, .06);
--hero: #0B1220;
--mono: "JetBrains Mono", ui-monospace, Menlo, Consolas, monospace;
--sans: "Inter", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
}
@media (prefers-color-scheme: dark) {
:root:not([data-theme="light"]) {
--bg: #0B1220; --surface: #111A2E; --surface-2: #16213A; --text: #E2E8F0; --muted: #A3B1C6;
--faint: #7C8BA1; --border: #22304A; --accent: #FB7185; --accent-fill: #F0374B;
--accent-soft: rgba(240, 55, 75, .14); --link: #7DD3FC; --viewer-bg: #0A101C;
--shadow: 0 1px 2px rgba(0, 0, 0, .3), 0 8px 24px rgba(0, 0, 0, .25);
}
}
:root[data-theme="dark"] {
--bg: #0B1220; --surface: #111A2E; --surface-2: #16213A; --text: #E2E8F0; --muted: #A3B1C6;
--faint: #7C8BA1; --border: #22304A; --accent: #FB7185; --accent-fill: #F0374B;
--accent-soft: rgba(240, 55, 75, .14); --link: #7DD3FC; --viewer-bg: #0A101C;
--shadow: 0 1px 2px rgba(0, 0, 0, .3), 0 8px 24px rgba(0, 0, 0, .25);
}
* { box-sizing: border-box; }
html { scroll-behavior: smooth; }
body { margin: 0; background: var(--bg); color: var(--text); font-family: var(--sans); font-size: 16px; line-height: 1.6; -webkit-font-smoothing: antialiased; }
a { color: var(--link); text-decoration: none; }
a:hover { text-decoration: underline; }
img { max-width: 100%; height: auto; display: block; }
.wrap { max-width: 1080px; margin: 0 auto; padding: 0 24px; }
@media (max-width: 640px) { .wrap { padding: 0 16px; } }
/* ---------- hero ---------- */
.hero { background: radial-gradient(ellipse 70% 90% at 85% 10%, #1B3352 0%, rgba(11,18,32,0) 70%), var(--hero); color: #F8FAFC; position: relative; }
.hero::before { content: ""; position: absolute; inset: 0 0 auto 0; height: 4px; background: linear-gradient(90deg, #F0374B 0%, #B4467F 55%, #38BDF8 100%); }
.hero .wrap { padding-top: 56px; padding-bottom: 48px; }
.venue { display: inline-flex; align-items: center; gap: 10px; padding: 6px 14px 6px 12px; border-radius: 999px; border: 1px solid rgba(240,55,75,.6); background: rgba(240,55,75,.12); color: #FDA4AF; font-weight: 600; font-size: 14px; letter-spacing: .02em; }
.venue i { width: 8px; height: 8px; border-radius: 50%; background: #F0374B; }
.hero h1 { margin: 22px 0 6px; font-size: clamp(44px, 8vw, 76px); line-height: 1; font-weight: 800; letter-spacing: -0.03em; }
.hero .subtitle { margin: 14px 0 0; font-size: clamp(18px, 2.4vw, 24px); line-height: 1.35; color: #CBD5E1; font-weight: 500; max-width: 820px; }
.authors { margin: 28px 0 4px; font-size: 18px; font-weight: 600; display: flex; flex-wrap: wrap; gap: 4px 22px; }
.affil { color: #94A3B8; font-size: 15px; overflow-wrap: anywhere; }
.affil code { font-family: var(--mono); font-size: 13px; color: #7C8BA1; }
.btns { display: flex; flex-wrap: wrap; gap: 10px; margin-top: 30px; }
.btn { display: inline-flex; align-items: center; gap: 8px; padding: 10px 16px; border-radius: 10px; font-weight: 600; font-size: 15px; color: #E2E8F0; background: rgba(255,255,255,.06); border: 1px solid rgba(255,255,255,.14); transition: background .15s, border-color .15s; }
.btn:hover { text-decoration: none; background: rgba(255,255,255,.12); border-color: rgba(255,255,255,.28); }
.btn.primary { background: #F0374B; border-color: #F0374B; color: #fff; }
.btn.primary:hover { background: #E11D48; border-color: #E11D48; }
.btn svg { width: 17px; height: 17px; flex: none; }
/* ---------- stats ---------- */
.stats { display: grid; grid-template-columns: repeat(5, 1fr); gap: 12px; margin-top: -28px; position: relative; }
.stat { background: var(--surface); border: 1px solid var(--border); border-radius: 14px; padding: 18px 16px 16px; box-shadow: var(--shadow); }
.stat b { display: block; font-size: 30px; line-height: 1.1; font-weight: 800; letter-spacing: -0.02em; font-variant-numeric: tabular-nums; }
.stat b.hl { color: var(--accent); }
.stat span { display: block; margin-top: 6px; font-weight: 600; font-size: 14px; line-height: 1.3; }
.stat small { display: block; margin-top: 4px; color: var(--faint); font-size: 12.5px; line-height: 1.35; }
@media (max-width: 920px) { .stats { grid-template-columns: repeat(3, 1fr); } }
@media (max-width: 560px) { .stats { grid-template-columns: repeat(2, 1fr); } .stat b { font-size: 26px; } .stat:last-child { grid-column: span 2; } }
/* ---------- sections ---------- */
section { padding: 56px 0 8px; }
h2 { font-size: 26px; line-height: 1.2; letter-spacing: -0.015em; margin: 0 0 6px; font-weight: 700; }
h2 .num { color: var(--accent); margin-right: 10px; font-variant-numeric: tabular-nums; }
.lede { color: var(--muted); margin: 0 0 22px; max-width: 760px; }
.card { background: var(--surface); border: 1px solid var(--border); border-radius: 14px; box-shadow: var(--shadow); }
.abstract { padding: 26px 28px; font-size: 16.5px; line-height: 1.7; }
.abstract p { margin: 0; }
@media (max-width: 640px) { .abstract { padding: 20px; } }
/* ---------- viewer ---------- */
.tabs { display: flex; gap: 6px; padding: 6px; background: var(--surface-2); border: 1px solid var(--border); border-radius: 12px; width: max-content; max-width: 100%; }
.tab { appearance: none; border: 0; background: transparent; color: var(--muted); font: 600 15px var(--sans); padding: 9px 18px; border-radius: 8px; cursor: pointer; display: inline-flex; gap: 8px; align-items: center; }
.tab[aria-selected="true"] { background: var(--surface); color: var(--text); box-shadow: 0 1px 3px rgba(15,23,42,.12); }
.tab .count { font-weight: 500; color: var(--faint); font-size: 13px; }
.viewer { margin-top: 14px; overflow: hidden; }
.toolbar { display: flex; align-items: center; justify-content: space-between; gap: 12px; flex-wrap: wrap; padding: 12px 16px; border-bottom: 1px solid var(--border); }
.toolbar .title { font-weight: 600; font-size: 15px; }
.toolbar .title span { color: var(--faint); font-weight: 500; margin-left: 8px; font-variant-numeric: tabular-nums; }
.toolbar .actions { display: flex; gap: 8px; flex-wrap: wrap; }
.tbtn { display: inline-flex; align-items: center; gap: 6px; font-size: 14px; font-weight: 600; padding: 7px 12px; border-radius: 8px; border: 1px solid var(--border); color: var(--text); background: var(--surface); }
.tbtn:hover { text-decoration: none; background: var(--surface-2); }
.tbtn svg { width: 15px; height: 15px; }
.pages { background: var(--viewer-bg); height: min(88vh, 1180px); overflow-y: auto; padding: 24px 16px; overscroll-behavior: contain; }
.page { max-width: 860px; margin: 0 auto 18px; background: #fff; box-shadow: 0 2px 10px rgba(0,0,0,.18); position: relative; }
.page img { width: 100%; height: auto; aspect-ratio: 1654 / 2339; }
.page a { display: block; }
.page .pn { position: absolute; right: 10px; bottom: 8px; font: 500 12px var(--mono); color: #64748B; background: rgba(255,255,255,.85); padding: 2px 6px; border-radius: 4px; }
.poster-pane { background: var(--viewer-bg); padding: 24px 16px; }
.poster-pane a { display: block; max-width: 860px; margin: 0 auto; box-shadow: 0 2px 10px rgba(0,0,0,.18); }
.poster-pane img { width: 100%; height: auto; aspect-ratio: 2600 / 3680; background: #fff; }
.hint { text-align: center; color: var(--faint); font-size: 13px; margin: 14px 0 0; }
[hidden] { display: none !important; }
/* ---------- method ---------- */
.figure { padding: 20px; }
.figure .img { background: #fff; border-radius: 10px; padding: 8px; }
.figure figcaption { color: var(--muted); font-size: 14px; margin-top: 14px; line-height: 1.55; }
.grid3 { display: grid; grid-template-columns: repeat(3, 1fr); gap: 14px; margin-top: 14px; }
.point { padding: 18px 18px 16px; }
.point h3 { margin: 0 0 6px; font-size: 16px; }
.point p { margin: 0; color: var(--muted); font-size: 14.5px; line-height: 1.55; }
.chips { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 10px; }
.chip { font: 500 12.5px var(--mono); padding: 3px 8px; border-radius: 6px; background: var(--surface-2); border: 1px solid var(--border); color: var(--text); }
@media (max-width: 820px) { .grid3 { grid-template-columns: 1fr; } }
/* ---------- tables ---------- */
.twocol { display: grid; grid-template-columns: 1fr; gap: 14px; }
.twocol > *, .grid3 > *, .res > *, .stats > * { min-width: 0; }
.tcard { padding: 18px 18px 14px; }
.tcard h3 { margin: 0 0 2px; font-size: 16px; }
.tcard .cap { color: var(--faint); font-size: 13px; margin: 0 0 12px; line-height: 1.45; }
.corr-wrap { display: grid; grid-template-columns: minmax(0, 1.4fr) minmax(0, 1fr); gap: 24px; align-items: center; }
.corr-wrap .note { margin: 0; }
@media (max-width: 820px) { .corr-wrap { grid-template-columns: 1fr; gap: 14px; } }
.scroll { overflow-x: auto; -webkit-overflow-scrolling: touch; }
table { border-collapse: collapse; width: 100%; font-size: 14px; font-variant-numeric: tabular-nums; }
th, td { padding: 7px 10px; text-align: right; white-space: nowrap; }
th:first-child, td:first-child, table.corr td:nth-child(2) { text-align: left; }
td.src { text-align: left; color: var(--muted); }
thead th { font-size: 12px; text-transform: uppercase; letter-spacing: .05em; color: var(--faint); font-weight: 600; border-bottom: 1px solid var(--border); }
tbody td { border-bottom: 1px solid var(--border); }
tbody tr:last-child td { border-bottom: 0; }
td.lvl { color: var(--faint); font-size: 12.5px; text-transform: uppercase; letter-spacing: .05em; }
tr.ours td { font-weight: 700; }
tr.ours td:not(.lvl) { color: var(--accent); }
tr.sep td { border-top: 2px solid var(--border); }
td.neg { color: var(--faint); }
.note { margin: 14px 0 0; padding: 12px 14px; border-left: 3px solid var(--accent-fill); background: var(--accent-soft); border-radius: 0 8px 8px 0; color: var(--muted); font-size: 14px; }
.note b { color: var(--text); }
/* ---------- resources ---------- */
.res { display: grid; grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); gap: 14px; }
.r { padding: 18px; display: flex; flex-direction: column; gap: 6px; color: var(--text); transition: border-color .15s, transform .15s; }
.r:hover { text-decoration: none; border-color: var(--accent-fill); transform: translateY(-1px); }
.r .kind { font-size: 12px; font-weight: 600; text-transform: uppercase; letter-spacing: .06em; color: var(--faint); display: flex; align-items: center; gap: 8px; }
.r .kind i { width: 8px; height: 8px; border-radius: 2px; }
.r .name { font: 500 14.5px var(--mono); word-break: break-word; }
.r p { margin: 0; color: var(--muted); font-size: 14px; line-height: 1.5; }
/* ---------- cite ---------- */
.cite { position: relative; }
.cite pre { margin: 0; padding: 20px 22px; overflow-x: auto; font: 13.5px/1.6 var(--mono); color: var(--text); }
.copy { position: absolute; top: 12px; right: 12px; font: 600 13px var(--sans); padding: 6px 12px; border-radius: 8px; border: 1px solid var(--border); background: var(--surface-2); color: var(--text); cursor: pointer; }
.copy:hover { border-color: var(--faint); }
footer { margin-top: 64px; border-top: 1px solid var(--border); padding: 28px 0 40px; }
footer .wrap { display: flex; align-items: center; justify-content: space-between; gap: 20px; flex-wrap: wrap; }
.logos { display: flex; align-items: center; gap: 14px; background: #fff; padding: 8px 14px; border-radius: 12px; border: 1px solid var(--border); }
.logos img { height: 44px; width: auto; }
footer p { margin: 0; color: var(--faint); font-size: 14px; }
</style>
</head>
<body>
<svg width="0" height="0" style="position:absolute" aria-hidden="true">
<symbol id="i-doc" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"/><path d="M14 2v6h6M8 13h8M8 17h5"/></symbol>
<symbol id="i-img" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="3" width="18" height="18" rx="2"/><circle cx="8.5" cy="8.5" r="1.5"/><path d="m21 15-5-5L5 21"/></symbol>
<symbol id="i-box" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 16V8a2 2 0 0 0-1-1.73l-7-4a2 2 0 0 0-2 0l-7 4A2 2 0 0 0 3 8v8a2 2 0 0 0 1 1.73l7 4a2 2 0 0 0 2 0l7-4A2 2 0 0 0 21 16z"/><path d="M3.27 6.96 12 12.01l8.73-5.05M12 22.08V12"/></symbol>
<symbol id="i-db" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><ellipse cx="12" cy="5" rx="9" ry="3"/><path d="M21 12c0 1.66-4 3-9 3s-9-1.34-9-3"/><path d="M3 5v14c0 1.66 4 3 9 3s9-1.34 9-3V5"/></symbol>
<symbol id="i-quote" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 21c3 0 7-1 7-8V5c0-1.25-.76-2-2-2H4c-1.25 0-2 .75-2 2v6c0 1.25.75 2 2 2h3c0 4-2 6-4 6M15 21c3 0 7-1 7-8V5c0-1.25-.76-2-2-2h-4c-1.25 0-2 .75-2 2v6c0 1.25.75 2 2 2h3c0 4-2 6-4 6"/></symbol>
<symbol id="i-ext" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6M15 3h6v6M10 14 21 3"/></symbol>
<symbol id="i-dl" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4M7 10l5 5 5-5M12 15V3"/></symbol>
</svg>
<header class="hero">
<div class="wrap">
<span class="venue"><i></i>EMNLP 2026 · Budapest, Hungary</span>
<h1>TurCOMET</h1>
<p class="subtitle">A Morphologically-Aware Evaluation System for Document-Level Turkish-English Machine Translation</p>
<div class="authors"><span>Atahan Uz</span><span>Hüseyin Emir Akdağ</span><span>Susan Üsküdarlı</span></div>
<div class="affil">Boğaziçi University · <code>{atahan.uz, huseyin.akdag}@std.bogazici.edu.tr · suzan.uskudarli@bogazici.edu.tr</code></div>
<div class="btns">
<a class="btn primary" href="TurCOMET_EMNLP2026_paper.pdf" target="_blank" rel="noopener"><svg><use href="#i-doc"/></svg>Paper (PDF)</a>
<a class="btn" href="TurCOMET_EMNLP2026_poster.pdf" target="_blank" rel="noopener"><svg><use href="#i-img"/></svg>Poster (PDF)</a>
<a class="btn" href="https://huggingface.co/TurCOMET/XCOMET-XL-TR" target="_blank" rel="noopener"><svg><use href="#i-box"/></svg>Model</a>
<a class="btn" href="https://huggingface.co/datasets/TurCOMET/TurkDoc-MT" target="_blank" rel="noopener"><svg><use href="#i-db"/></svg>TurkDoc-MT</a>
<a class="btn" href="https://huggingface.co/datasets/TurCOMET/TurkDoc-MT-MQM-Annotations" target="_blank" rel="noopener"><svg><use href="#i-db"/></svg>MQM annotations</a>
<a class="btn" href="#cite"><svg><use href="#i-quote"/></svg>BibTeX</a>
</div>
</div>
</header>
<main class="wrap">
<div class="stats" role="list">
<div class="stat" role="listitem"><b class="hl">+0.113</b><span>system-level Kendall τ</span><small>vs. xCOMET-XL: 0.636 → 0.749</small></div>
<div class="stat" role="listitem"><b class="hl">+0.024</b><span>segment-level Kendall τ</span><small>0.267 → 0.291, p = 0.022</small></div>
<div class="stat" role="listitem"><b>330K</b><span>trainable parameters</span><small>&lt;0.01% of frozen xCOMET-XL</small></div>
<div class="stat" role="listitem"><b>1,322</b><span>documents, 7 domains</span><small>TurkDoc-MT benchmark, EN↔TR</small></div>
<div class="stat" role="listitem"><b>2,200</b><span>expert MQM annotations</span><small>by 4 professional translators</small></div>
</div>
<section id="abstract">
<h2>Abstract</h2>
<div class="card abstract">
<p>We introduce TurCOMET, the first Turkish-specific augmentation of COMET for evaluating translations between Turkish and English. By extending xCOMET-XL with Turkish morphological features and training it on additional Turkish-English data via an additive bottleneck layer, TurCOMET resolves neural evaluation failures caused by sub-word over-segmentation. To validate our metric, we introduce TurkDoc-MT, a meticulously curated Turkish-English document-level aligned dataset of 1,322 instances spanning seven domains. Meta-evaluation demonstrates that TurCOMET significantly outperforms the off-the-shelf baseline, yielding absolute Kendall’s τ correlation gains of +0.113 (system-level) and +0.024 (segment-level), with the largest gains in the TR→EN direction. All models, datasets, code, and annotations are publicly released for the benefit of Turkish natural language processing and benchmarking any translation system.</p>
</div>
</section>
<section id="read">
<h2>Paper &amp; poster</h2>
<p class="lede">Read the full camera-ready paper or the EMNLP 2026 poster right here. Click any page to open the PDF at that page.</p>
<div class="tabs" role="tablist" aria-label="Document">
<button class="tab" role="tab" id="tab-paper" aria-controls="pane-paper" aria-selected="true">Paper <span class="count">21 pages</span></button>
<button class="tab" role="tab" id="tab-poster" aria-controls="pane-poster" aria-selected="false">Poster <span class="count">A0</span></button>
</div>
<div class="card viewer" id="pane-paper" role="tabpanel" aria-labelledby="tab-paper">
<div class="toolbar">
<div class="title">TurCOMET — full paper<span id="pageind">Page 1 / 21</span></div>
<div class="actions">
<a class="tbtn" href="TurCOMET_EMNLP2026_paper.pdf" target="_blank" rel="noopener"><svg><use href="#i-ext"/></svg>Open PDF</a>
<a class="tbtn" href="TurCOMET_EMNLP2026_paper.pdf" download target="_blank" rel="noopener"><svg><use href="#i-dl"/></svg>Download</a>
</div>
</div>
<div class="pages" id="pages" tabindex="0" aria-label="Paper pages"></div>
<noscript><p class="hint"><a href="TurCOMET_EMNLP2026_paper.pdf">Open the paper (PDF)</a></p></noscript>
</div>
<div class="card viewer" id="pane-poster" role="tabpanel" aria-labelledby="tab-poster" hidden>
<div class="toolbar">
<div class="title">EMNLP 2026 poster<span>A0 portrait</span></div>
<div class="actions">
<a class="tbtn" href="TurCOMET_EMNLP2026_poster.pdf" target="_blank" rel="noopener"><svg><use href="#i-ext"/></svg>Open PDF</a>
<a class="tbtn" href="TurCOMET_EMNLP2026_poster.pdf" download target="_blank" rel="noopener"><svg><use href="#i-dl"/></svg>Download</a>
</div>
</div>
<div class="poster-pane">
<a href="TurCOMET_EMNLP2026_poster.pdf" target="_blank" rel="noopener"><img src="img/poster.webp" alt="TurCOMET EMNLP 2026 poster" loading="lazy" width="2600" height="3680"></a>
<p class="hint">Click the poster to open the vector PDF and zoom in on any detail.</p>
</div>
</div>
</section>
<section id="method">
<h2>How TurCOMET works</h2>
<p class="lede">Turkish is agglutinative: a single root can carry a long chain of derivational and inflectional suffixes, which the XLM-R sub-word tokenizer fragments aggressively. This over-segmentation depresses surface overlap and dilutes the pooled sentence representation, so TurCOMET feeds the metric explicit Turkish-aware signals.</p>
<figure class="card figure" style="margin:0">
<div class="img"><img src="img/architecture.webp" alt="TurCOMET architecture: frozen xCOMET-XL backbone with a trainable bottleneck that fuses five Turkish lexical features into the pooled representation" loading="lazy" width="1939" height="1053"></div>
<figcaption><b>Figure 1.</b> The frozen xCOMET-XL backbone (3.5B parameters) is kept intact. The only addition is a violet bottleneck that fuses a five-dimensional Turkish feature vector into the pooled embedding: <i>h′ = h + W₂ tanh(W₁[h; f] + b₁) + b₂</i>, width 64.</figcaption>
</figure>
<div class="grid3">
<div class="card point">
<h3>Five Turkish-aware features</h3>
<p>Character overlap that survives suffix chains, lemma-level edit rate, a reference-free semantic anchor, an omission detector and a direction flag.</p>
<div class="chips"><span class="chip">chrF++</span><span class="chip">lemma-TER</span><span class="chip">LaBSE cos</span><span class="chip">length-z</span><span class="chip">direction</span></div>
</div>
<div class="card point">
<h3>Never worse at step zero</h3>
<p>W₂ and b₂ are zero-initialized, so training starts from scores identical to xCOMET-XL. Only ~330K parameters (&lt;0.01% of the network) are trained.</p>
</div>
<div class="card point">
<h3>One model, both directions</h3>
<p>Fine-tuned on 35,357 WMT EN↔TR direct-assessment ratings (2017–18) with AdamW and MSE loss on a single A100. Held-out segment τ: 0.368 → 0.394.</p>
</div>
</div>
</section>
<section id="results">
<h2>Results</h2>
<p class="lede">An expert MQM audit of 550 translations (4 professional translators, 11 LLMs × 2 directions) shows that TurCOMET agrees best with human judgment.</p>
<div class="twocol">
<div class="card tcard">
<h3>Correlation with expert MQM judgments</h3>
<p class="cap">Segment level n = 550, system level n = 22. Higher is better.</p>
<div class="corr-wrap">
<div class="scroll">
<table class="corr">
<thead><tr><th></th><th style="text-align:left">Metric</th><th>Pearson</th><th>Spearman</th><th>Kendall</th></tr></thead>
<tbody>
<tr><td class="lvl">Segment</td><td>BLEU</td><td>0.385</td><td>0.335</td><td>0.256</td></tr>
<tr><td class="lvl"></td><td>xCOMET-XL</td><td>0.419</td><td>0.350</td><td>0.267</td></tr>
<tr class="ours"><td class="lvl"></td><td>TurCOMET</td><td>0.436</td><td>0.383</td><td>0.291</td></tr>
<tr class="sep"><td class="lvl">System</td><td>BLEU</td><td>0.828</td><td>0.852</td><td>0.628</td></tr>
<tr><td class="lvl"></td><td>xCOMET-XL</td><td>0.950</td><td>0.802</td><td>0.636</td></tr>
<tr class="ours"><td class="lvl"></td><td>TurCOMET</td><td>0.976</td><td>0.913</td><td>0.749</td></tr>
</tbody>
</table>
</div>
<p class="note"><b>Where the gain comes from.</b> Segment-level Δτ = +0.0247 overall (95% CI [+0.004, +0.049], p = 0.022), carried almost entirely by TR→EN (Δτ = +0.049, p &lt; 0.001). In EN→TR the two neural metrics are statistically indistinguishable.</p>
</div>
</div>
<div class="card tcard">
<h3>LLM leaderboard on TurkDoc-MT</h3>
<p class="cap">Averaged over both directions, sorted by the human MQM judges (negated z-score, higher = better).</p>
<div class="scroll">
<table>
<thead><tr><th>Model</th><th>Humans</th><th>BLEU</th><th>xCOMET-XL</th><th>TurCOMET</th></tr></thead>
<tbody>
<tr><td><b>gemma-4-31B-it</b></td><td><b>+0.32</b></td><td><b>31.3</b></td><td><b>0.814</b></td><td><b>0.714</b></td></tr>
<tr><td>translategemma-27b-it</td><td>+0.26</td><td>21.4</td><td>0.805</td><td>0.709</td></tr>
<tr><td>translategemma-12b-it</td><td>+0.22</td><td>19.0</td><td>0.785</td><td>0.695</td></tr>
<tr><td>gpt-oss-120b</td><td>+0.21</td><td>22.5</td><td>0.782</td><td>0.688</td></tr>
<tr><td>gemma-3-12b-it</td><td>+0.21</td><td>25.5</td><td>0.766</td><td>0.675</td></tr>
<tr><td>gemma-3-27b-it</td><td>+0.17</td><td>28.7</td><td>0.782</td><td>0.686</td></tr>
<tr><td>gpt-oss-20b</td><td class="neg">−0.19</td><td>22.4</td><td>0.749</td><td>0.665</td></tr>
<tr><td>Qwen3.5-9B</td><td class="neg">−0.20</td><td>22.6</td><td>0.734</td><td>0.657</td></tr>
<tr><td>translategemma-4b-it</td><td class="neg">−0.30</td><td>10.1</td><td>0.732</td><td>0.651</td></tr>
<tr><td>Llama-3.1-8B-Instruct</td><td class="neg">−0.42</td><td>16.2</td><td>0.668</td><td>0.607</td></tr>
<tr><td>gemma-3-4b-it</td><td class="neg">−0.43</td><td>14.6</td><td>0.665</td><td>0.606</td></tr>
</tbody>
</table>
</div>
</div>
</div>
</section>
<section id="benchmark">
<h2>TurkDoc-MT benchmark</h2>
<p class="lede">1,322 English–Turkish document pairs kept at their native document boundaries, preserving discourse-level context, across seven domains from curated Wikipedia prose to multi-thousand-word TED talks.</p>
<div class="card tcard">
<div class="scroll">
<table>
<thead><tr><th>Domain</th><th style="text-align:left">Source</th><th>Docs</th><th>Mean words</th><th>90th pct.</th><th>99th pct.</th></tr></thead>
<tbody>
<tr><td>Wikipedia articles</td><td class="src">FLORES-200</td><td>200</td><td>72.8</td><td>106.1</td><td>124.2</td></tr>
<tr><td>Technical &amp; scientific</td><td class="src">TWiST</td><td>200</td><td>119.9</td><td>205.4</td><td>363.3</td></tr>
<tr><td>Educational dialogue</td><td class="src">XQuAD</td><td>200</td><td>134.1</td><td>190.4</td><td>330.5</td></tr>
<tr><td>Journalistic prose</td><td class="src">NTREX-128</td><td>122</td><td>342.9</td><td>543.6</td><td>689.1</td></tr>
<tr><td>Legal rulings</td><td class="src">AYM</td><td>200</td><td>107.2</td><td>193.1</td><td>426.3</td></tr>
<tr><td>Web crawl</td><td class="src">DocHPLT</td><td>200</td><td>225.1</td><td>348.1</td><td>1,260.0</td></tr>
<tr><td>Spoken language</td><td class="src">TED2020</td><td>200</td><td>2,646.3</td><td>3,285.4</td><td>4,495.6</td></tr>
<tr class="sep"><td><b>Total</b></td><td></td><td><b>1,322</b></td><td></td><td></td><td></td></tr>
</tbody>
</table>
</div>
</div>
</section>
<section id="resources">
<h2>Released resources</h2>
<p class="lede">Everything needed to use TurCOMET, reproduce the training, and re-run the meta-evaluation.</p>
<div class="res">
<a class="card r" href="https://huggingface.co/TurCOMET/XCOMET-XL-TR" target="_blank" rel="noopener">
<span class="kind"><i style="background:#F0374B"></i>Model</span>
<span class="name">TurCOMET/XCOMET-XL-TR</span>
<p>The TurCOMET checkpoint (bf16, ~7 GB). Scores (source, translation, reference) triplets in both directions.</p>
</a>
<a class="card r" href="https://huggingface.co/datasets/TurCOMET/TurkDoc-MT" target="_blank" rel="noopener">
<span class="kind"><i style="background:#38BDF8"></i>Dataset</span>
<span class="name">TurCOMET/TurkDoc-MT</span>
<p>The benchmark: 1,322 document-level EN↔TR pairs across seven domains.</p>
</a>
<a class="card r" href="https://huggingface.co/datasets/TurCOMET/TurkDoc-MT-MQM-Annotations" target="_blank" rel="noopener">
<span class="kind"><i style="background:#38BDF8"></i>Dataset</span>
<span class="name">TurCOMET/TurkDoc-MT-MQM-Annotations</span>
<p>2,200 expert MQM annotations (error spans, categories, severities, z-scores) from four professional translators.</p>
</a>
<a class="card r" href="https://huggingface.co/datasets/TurCOMET/TurkDoc-MT_with_model-translations" target="_blank" rel="noopener">
<span class="kind"><i style="background:#38BDF8"></i>Dataset</span>
<span class="name">TurCOMET/TurkDoc-MT_with_model-translations</span>
<p>TurkDoc-MT with the EN→TR and TR→EN translations produced by every evaluated LLM.</p>
</a>
<a class="card r" href="https://huggingface.co/datasets/TurCOMET/wmt-da-en-tr_tr-en" target="_blank" rel="noopener">
<span class="kind"><i style="background:#38BDF8"></i>Dataset</span>
<span class="name">TurCOMET/wmt-da-en-tr_tr-en</span>
<p>Training data: 35,357 WMT EN↔TR DA ratings with precomputed features and the exact 90/5/5 splits.</p>
</a>
</div>
</section>
<section id="cite">
<h2>Citation</h2>
<p class="lede">If you use TurCOMET or TurkDoc-MT, please cite:</p>
<div class="card cite">
<button class="copy" type="button" id="copy">Copy</button>
<pre id="bibtex">@inproceedings{uz-etal-2026-turcomet,
title = {{TurCOMET}: A Morphologically-Aware Evaluation System for
Document-Level {T}urkish-{E}nglish Machine Translation},
author = {Uz, Atahan and Akda{\u{g}}, H{\"u}seyin Emir and
{\"U}sk{\"u}darl{\i}, Susan},
booktitle = {Proceedings of the 2026 Conference on Empirical Methods
in Natural Language Processing},
year = {2026},
address = {Budapest, Hungary},
publisher = {Association for Computational Linguistics}
}</pre>
</div>
</section>
</main>
<footer>
<div class="wrap">
<div class="logos"><img src="img/bogazici.png" alt="Boğaziçi University" width="246" height="240"><img src="img/tabilab.png" alt="TABILAB" width="151" height="240"></div>
<p>Boğaziçi University · EMNLP 2026, October 24–29, Budapest, Hungary</p>
</div>
</footer>
<script>
(function () {
var N = 21, pdf = 'TurCOMET_EMNLP2026_paper.pdf';
var box = document.getElementById('pages'), ind = document.getElementById('pageind');
for (var i = 1; i <= N; i++) {
var n = (i < 10 ? '0' : '') + i;
var d = document.createElement('div');
d.className = 'page'; d.dataset.page = i;
d.innerHTML = '<a href="' + pdf + '#page=' + i + '" target="_blank" rel="noopener" aria-label="Open page ' + i + ' in the PDF">' +
'<img src="pages/page-' + n + '.webp" alt="Paper page ' + i + '" width="1654" height="2339"' + (i > 2 ? ' loading="lazy"' : '') + '></a>' +
'<span class="pn">' + i + ' / ' + N + '</span>';
box.appendChild(d);
}
if ('IntersectionObserver' in window) {
var io = new IntersectionObserver(function (es) {
es.forEach(function (e) { if (e.isIntersecting) ind.textContent = 'Page ' + e.target.dataset.page + ' / ' + N; });
}, { root: box, rootMargin: '-45% 0px -45% 0px' });
box.querySelectorAll('.page').forEach(function (p) { io.observe(p); });
}
var tabs = { paper: document.getElementById('tab-paper'), poster: document.getElementById('tab-poster') };
var panes = { paper: document.getElementById('pane-paper'), poster: document.getElementById('pane-poster') };
function show(k) {
Object.keys(tabs).forEach(function (t) {
var on = t === k;
tabs[t].setAttribute('aria-selected', on ? 'true' : 'false');
tabs[t].tabIndex = on ? 0 : -1;
panes[t].hidden = !on;
});
}
tabs.paper.addEventListener('click', function () { show('paper'); });
tabs.poster.addEventListener('click', function () { show('poster'); });
document.querySelector('.tabs').addEventListener('keydown', function (e) {
if (e.key === 'ArrowRight' || e.key === 'ArrowLeft') {
var k = tabs.paper.getAttribute('aria-selected') === 'true' ? 'poster' : 'paper';
show(k); tabs[k].focus();
}
});
if (/^#(poster|paper)$/.test(location.hash)) {
if (location.hash === '#poster') show('poster');
setTimeout(function () { document.getElementById('read').scrollIntoView(); }, 0);
}
document.getElementById('copy').addEventListener('click', function () {
var b = this, txt = document.getElementById('bibtex').textContent;
function done(ok) { b.textContent = ok ? 'Copied' : 'Select & copy'; setTimeout(function () { b.textContent = 'Copy'; }, 1600); }
if (navigator.clipboard && navigator.clipboard.writeText) {
navigator.clipboard.writeText(txt).then(function () { done(true); }, function () { done(fallback(txt)); });
} else { done(fallback(txt)); }
function fallback(t) {
try { var ta = document.createElement('textarea'); ta.value = t; document.body.appendChild(ta); ta.select(); var ok = document.execCommand('copy'); ta.remove(); return ok; } catch (e) { return false; }
}
});
})();
</script>
</body>
</html>