poster/poster.html

<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>FinePhrase poster (COLM 2026, 72in x 36in)</title>
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;800;900&display=swap" rel="stylesheet" />
<style>
  /* Poster is 72in (W) x 36in (H), landscape. All sizes are physical units so the PDF prints 1:1. */
  @page { size: 72in 36in; margin: 0; }
  :root {
    --gold: #EBA937;
    --gold-dark: #B87A12;
    --gold-soft: #FFF4DE;
    --ink: #1a1a1a;
    --muted: #5c5c5c;
    --gray: #C9C9C9;
    --gray-dark: #7a7a7a;
    --line: #E4E4E4;
    --bg: #ffffff;
  }
  * { box-sizing: border-box; }
  html, body { margin: 0; padding: 0; background: var(--bg); color: var(--ink); }
  body { font-family: "Inter", "Helvetica Neue", Arial, sans-serif; font-size: 35pt; line-height: 1.32; -webkit-font-smoothing: antialiased; }
  /* Explicit row heights (4.6 + 0.9 + 28.2 + padding 2.3 = 36in): Chromium's PDF pipeline
     stretches `1fr` rows differently from the screen layout, so every region gets a fixed height. */
  .poster { width: 72in; height: 36in; padding: 1.2in 1.5in 1.1in; display: grid; grid-template-rows: 4.6in 28.2in; row-gap: 0.9in; overflow: hidden; break-inside: avoid; }
  p { margin: 0 0 0.35in; }
  strong { font-weight: 800; }
  .accent { color: var(--gold-dark); }

  /* Header */
  header { display: grid; grid-template-columns: 4.4in 1fr auto; column-gap: 1.1in; align-items: center; height: 4.6in; }
  header .logo { width: 4.4in; }
  header h1 { margin: 0 0 0.25in; font-size: 92pt; line-height: 1.05; font-weight: 900; letter-spacing: -0.02em; }
  header h1 span { color: var(--gold-dark); }
  header .authors { font-size: 32pt; line-height: 1.35; margin: 0; }
  header .authors sup { font-size: 60%; }
  header .affils { font-size: 27pt; color: var(--muted); margin: 0.1in 0 0; }
  header .affils b { color: var(--ink); }
  .qrs { display: flex; gap: 0.6in; }
  .qr { width: 4in; text-align: center; }
  .qr img { width: 3.3in; height: 3.3in; display: block; margin: 0 auto; }
  .qr .t { font-size: 27pt; font-weight: 800; line-height: 1.2; margin-top: 0.12in; }
  .qr .u { font-size: 20pt; color: var(--muted); font-family: "SF Mono", Menlo, Consolas, monospace; white-space: nowrap; }

  /* Columns */
  main { display: grid; grid-template-columns: 17in 32in 17in; column-gap: 1.5in; height: 28.2in; }
  .col { display: flex; flex-direction: column; justify-content: space-between; row-gap: 0.55in; height: 28.2in; overflow: hidden; }
  h2 { margin: 0 0 0.3in; font-size: 46pt; line-height: 1.1; font-weight: 800; letter-spacing: -0.01em; padding-bottom: 0.15in; border-bottom: 0.06in solid var(--gold); }
  h3 { margin: 0 0 0.12in; font-size: 36pt; line-height: 1.15; font-weight: 800; }
  .caption { font-size: 27pt; color: var(--muted); line-height: 1.3; margin: 0.15in 0 0; }
  ul { margin: 0; padding-left: 0.55in; }
  li { margin-bottom: 0.14in; }

  /* Stats */
  .stats { display: grid; grid-template-columns: 1fr 1fr; gap: 0.3in 0.5in; }
  .stat .n { font-size: 92pt; font-weight: 900; line-height: 1; color: var(--gold-dark); letter-spacing: -0.02em; }
  .stat .l { font-size: 30pt; color: var(--muted); margin-top: 0.08in; line-height: 1.25; }

  .overview { width: 100%; display: block; border-radius: 0.15in; }

  /* Recipe box */
  .recipe { background: var(--gold-soft); border-left: 0.16in solid var(--gold); border-radius: 0.12in; padding: 0.45in 0.55in; }
  .recipe h3 { color: var(--gold-dark); }
  .recipe ul { padding-left: 0.5in; }
  .recipe code { font-family: "SF Mono", Menlo, Consolas, monospace; font-size: 88%; background: rgba(0,0,0,0.05); padding: 0 0.08in; border-radius: 0.06in; }

  /* Central message */
  .message { flex: 1; display: flex; flex-direction: column; align-items: center; justify-content: center; text-align: center; background: var(--ink); color: #fff; border-radius: 0.3in; padding: 0.8in 1.4in; }
  .message .big { font-size: 165pt; font-weight: 900; line-height: 1.02; letter-spacing: -0.03em; margin: 0; }
  .message .big em { color: var(--gold); font-style: normal; }
  .message .sub { font-size: 42pt; line-height: 1.3; margin: 0.55in 0 0; color: #e8e8e8; max-width: 27in; }
  .message .sub b { color: var(--gold); font-weight: 800; }

  /* Bar charts (plain divs so fonts stay consistent everywhere) */
  /* Right padding leaves room for the value label after the longest bar. */
  .chart { display: flex; flex-direction: column; gap: 0.11in; position: relative; padding-right: 1.3in; }
  .row { display: grid; grid-template-columns: var(--label-w, 6.2in) 1fr; align-items: center; column-gap: 0.3in; }
  .row .lbl { text-align: right; font-size: var(--lbl-size, 28pt); line-height: 1.15; white-space: nowrap; }
  .row.hl .lbl { font-weight: 800; }
  .row .track { position: relative; height: var(--bar-h, 0.5in); }
  .row .bar { position: absolute; left: 0; top: 0; height: 100%; border-radius: 0 0.08in 0.08in 0; background: var(--gray); }
  .row.hl .bar { background: var(--gold); }
  .row.ref .bar { background: var(--gray-dark); }
  .row .val { position: absolute; top: 50%; transform: translateY(-50%); font-size: var(--val-size, 27pt); font-weight: 800; padding: 0 0.12in 0 0.18in; white-space: nowrap; background: var(--bg); z-index: 1; }
  .row.hl .val { color: var(--gold-dark); }
  .refline { position: absolute; top: -0.05in; bottom: -0.05in; border-left: 0.06in dashed var(--gray-dark); pointer-events: none; }
  .refline span { position: absolute; top: -0.42in; left: 0.12in; font-size: 22pt; color: var(--muted); white-space: nowrap; }
  .chart.main { --label-w: 8.6in; --bar-h: 0.78in; --lbl-size: 36pt; --val-size: 36pt; gap: 0.16in; }
  .chart.main .row .val { font-size: 36pt; }
  .legend { display: flex; gap: 0.8in; font-size: 26pt; color: var(--muted); margin-top: 0.25in; }
  .legend i { display: inline-block; width: 0.42in; height: 0.3in; border-radius: 0.05in; vertical-align: -0.03in; margin-right: 0.15in; }

  /* Cost strip */
  .cost { display: grid; grid-template-columns: 1.15fr 1fr; column-gap: 1in; align-items: center; border-top: 0.06in solid var(--line); padding-top: 0.6in; }
  .bignums { display: grid; grid-template-columns: repeat(2, 1fr); gap: 0.45in 0.6in; }
  .bignums .n { font-size: 108pt; font-weight: 900; line-height: 1; letter-spacing: -0.03em; color: var(--gold-dark); }
  .bignums .l { font-size: 29pt; color: var(--muted); margin-top: 0.1in; line-height: 1.25; }
  table { border-collapse: collapse; width: 100%; font-size: 30pt; }
  th, td { padding: 0.14in 0.2in; text-align: right; border-bottom: 0.03in solid var(--line); white-space: nowrap; }
  th { font-weight: 800; font-size: 24pt; color: var(--muted); text-transform: uppercase; letter-spacing: 0.04em; }
  th:first-child, td:first-child { text-align: left; }
  tr.hl td { font-weight: 800; color: var(--gold-dark); background: var(--gold-soft); }

  /* Findings (right column) */
  .finding { display: flex; flex-direction: column; }
  .finding h3 { display: flex; align-items: baseline; gap: 0.25in; }
  .finding h3 .num { display: inline-flex; align-items: center; justify-content: center; width: 0.75in; height: 0.75in; border-radius: 50%; background: var(--gold); color: var(--ink); font-size: 30pt; font-weight: 900; flex: none; align-self: center; }
  .finding .chart { --label-w: 5.8in; --bar-h: 0.5in; --lbl-size: 27pt; --val-size: 26pt; gap: 0.1in; margin-top: 0.2in; }
  .finding .caption { margin-top: 0.22in; }
</style>
</head>
<body>
<div class="poster">

  <header>
    <img class="logo" src="assets/hf-logo.svg" alt="Hugging Face" />
    <div>
      <h1>How Can We Synthesize High-Quality Pretraining Data?<br/><span>A systematic study of prompt design, generator model, and source data</span></h1>
      <p class="authors">Joel Niklaus<sup>1</sup> · Atsuki Yamaguchi<sup>2</sup> · Michal Štefánik<sup>3</sup> · Guilherme Penedo<sup>1</sup> · Hynek Kydlíček<sup>1</sup> · Elie Bakouch<sup>1</sup> · Lewis Tunstall<sup>1</sup> · Edward Beeching<sup>1</sup> · Thibaud Frere<sup>1</sup> · Colin Raffel<sup>1</sup> · Leandro von Werra<sup>1</sup> · Thomas Wolf<sup>1</sup></p>
      <p class="affils"><sup>1</sup>Hugging Face &nbsp;·&nbsp; <sup>2</sup>University of Sheffield &nbsp;·&nbsp; <sup>3</sup>National Institute of Informatics, Japan &nbsp;&nbsp;|&nbsp;&nbsp; <b>COLM 2026</b>, San Francisco &nbsp;·&nbsp; Poster F-4-126, Franciscan, Wednesday afternoon</p>
    </div>
    <div class="qrs">
      <div class="qr"><img src="assets/qr-paper.svg" alt="QR: paper" /><div class="t">Paper</div><div class="u">arXiv 2604.13977</div></div>
      <div class="qr"><img src="assets/qr-dataset.svg" alt="QR: dataset" /><div class="t">Dataset</div><div class="u">HuggingFaceFW/finephrase</div></div>
      <div class="qr"><img src="assets/qr-code.svg" alt="QR: code" /><div class="t">Prompts + code</div><div class="u">huggingface/finephrase</div></div>
    </div>
  </header>

  <main>
    <!-- LEFT: question, setup, recipe -->
    <section class="col">
      <div>
        <h2>The question</h2>
        <p>Crawlable web text is running out, so labs now rephrase the web with LLMs at trillion-token scale. But every prior method (WRAP, Nemotron-CC, BeyondWeb, REWIRE) reports gains against its own baseline. Nobody had compared <strong>which prompt</strong>, <strong>which generator</strong>, and <strong>which source data</strong> matter, under one controlled setup.</p>
      </div>

      <div>
        <h2>What we did</h2>
        <img class="overview" src="assets/overview.jpg" alt="Source docs, prompt, generator and mix-in data feed a trained model that is then evaluated" />
        <p class="caption">Rephrase ~10.5B source tokens with a prompt <em>P</em> and generator <em>G</em>, mix 50/50 with original web data, pretrain a 1.2B model from scratch on 21B tokens, evaluate on 12 benchmarks (3-shot cloze, macro average).</p>
        <div class="stats">
          <div class="stat"><div class="n">90</div><div class="l">rephrasing configurations</div></div>
          <div class="stat"><div class="n">333</div><div class="l">train-and-evaluate runs</div></div>
          <div class="stat"><div class="n">&gt;1T</div><div class="l">synthetic tokens generated</div></div>
          <div class="stat"><div class="n">12.7</div><div class="l">GPU-years of generation</div></div>
        </div>
      </div>

      <div class="recipe">
        <h3>The FinePhrase recipe</h3>
        <ul>
          <li><strong>Generator:</strong> <code>SmolLM2-1.7B-Instruct</code></li>
          <li><strong>Prompts:</strong> Table, Math, FAQ, Tutorial</li>
          <li><strong>Source:</strong> FineWeb (educational subset)</li>
          <li><strong>Training:</strong> mix with strong original web data (DCLM or FineWeb-HQ)</li>
        </ul>
      </div>
    </section>

    <!-- CENTER: central message (top, visible over heads), main result, cost -->
    <section class="col">
      <div class="message">
        <p class="big">The <em>prompt</em> matters more than the <em>model</em>.</p>
        <p class="sub">Rephrasing the web into <b>tables, math problems, FAQs and tutorials</b> with a <b>1.7B</b> model beats rewrites from 70B models and curated web data, at up to <b>30× lower</b> generation cost.</p>
      </div>

      <div>
        <h2>FinePhrase beats every existing synthetic dataset</h2>
        <div class="chart main" id="chart-main"></div>
        <div class="legend"><span><i style="background:var(--gold)"></i>FinePhrase (SmolLM2 1.7B)</span><span><i style="background:var(--gray-dark)"></i>Curated web baseline</span><span><i style="background:var(--gray)"></i>Prior synthetic datasets</span></div>
        <p class="caption">Macro average over 12 benchmarks (×100) of a 1.2B model trained on 21B tokens. FinePhrase-Table: +3.4 over DCLM, +3.6 over Nemotron-HQ-Synth. FinePhrase wins on knowledge and reading comprehension (ARC, SQuAD, DROP); DCLM keeps a small edge on PIQA and HellaSwag, which is why we always mix.</p>
      </div>

      <div class="cost">
        <div class="bignums">
          <div><div class="n">486B</div><div class="l">tokens, 1.35B samples, openly released</div></div>
          <div><div class="n">30×</div><div class="l">more tokens per GPU-hour than REWIRE</div></div>
          <div><div class="n">14.7K</div><div class="l">H100 GPU-hours total (612 GPU-days)</div></div>
          <div><div class="n">9.2K</div><div class="l">tokens / s / GPU with speculative decoding</div></div>
        </div>
        <table>
          <thead><tr><th>Dataset</th><th>Generator</th><th>Tokens</th><th>GPU-hrs</th><th>Tok / GPU-hr</th></tr></thead>
          <tbody>
            <tr><td>Cosmopedia</td><td>Mixtral 8x7B</td><td>25B</td><td>&gt;10K</td><td>&lt;2.5M</td></tr>
            <tr><td>SYNTH</td><td>fine-tuned</td><td>80B</td><td>4K</td><td>20M</td></tr>
            <tr><td>REWIRE</td><td>Llama 3.3 70B</td><td>400B</td><td>~352K</td><td>~1.1M</td></tr>
            <tr class="hl"><td>FinePhrase</td><td>SmolLM2 1.7B</td><td>486B</td><td>~14.7K</td><td>~33.1M</td></tr>
          </tbody>
        </table>
      </div>
    </section>

    <!-- RIGHT: four findings -->
    <section class="col">
      <div class="finding">
        <h3><span class="num">1</span>Structured formats beat paraphrasing</h3>
        <div class="chart" id="chart-prompts"></div>
        <p class="caption">Same generator (Gemma 3 1B), same source. Pedagogical restructuring wins. Free-form rewrites (article, narrative, commentary) do not even beat DCLM.</p>
      </div>

      <div class="finding">
        <h3><span class="num">2</span>Generators above 1B do not help</h3>
        <div class="chart" id="chart-size"></div>
        <p class="caption">Gemma 3 family, Math prompt. 270M is too small; from 1B up, quality is flat while cost explodes (27B guided rewrite: 11K GPU-hours vs 205 for FinePhrase-Table).</p>
      </div>

      <div class="finding">
        <h3><span class="num">3</span>SmolLM2 is the best small generator</h3>
        <div class="chart" id="chart-family"></div>
        <p class="caption">Tutorial prompt, ~1B instruct models. Messier, more diverse outputs beat polished ones: Qwen3 follows the format 100% of the time but collapses onto a few templates; SmolLM2 only 68%, yet trains better.</p>
      </div>

      <div class="finding">
        <h3><span class="num">4</span>Always mix with original web data</h3>
        <div class="chart" id="chart-mix"></div>
        <p class="caption">Synthetic-only training loses commonsense and NLU. Which data you mix in matters more than the source quality: DCLM or FineWeb-HQ 14.3 vs Cosmopedia 13.0.</p>
      </div>
    </section>
  </main>

</div>

<script>
  // All scores are macro averages (x100) from paper/data/*.dat, rounded to one decimal.
  const DCLM = 13.8;

  function chart(id, rows, { max = 18, ref = null } = {}) {
    const el = document.getElementById(id);
    el.innerHTML = rows.map(r => {
      const cls = r.hl ? 'hl' : r.ref ? 'ref' : '';
      const w = (r.v / max * 100).toFixed(2);
      return `<div class="row ${cls}"><div class="lbl">${r.label}</div><div class="track"><div class="bar" style="width:${w}%"></div><div class="val" style="left:${w}%">${r.v.toFixed(1)}</div></div></div>`;
    }).join('');
    if (ref !== null) {
      // Dashed reference line for DCLM, positioned over the track column.
      const track = el.querySelector('.track');
      const left = track.offsetLeft + track.offsetWidth * ref / max;
      el.insertAdjacentHTML('beforeend', `<div class="refline" style="left:${left}px"><span>DCLM ${ref.toFixed(1)}</span></div>`);
    }
  }

  chart('chart-main', [
    { label: 'FinePhrase-Table', v: 17.2, hl: true },
    { label: 'FinePhrase-Math', v: 17.0, hl: true },
    { label: 'FinePhrase-FAQ', v: 16.2, hl: true },
    { label: 'FinePhrase-Tutorial', v: 15.9, hl: true },
    { label: 'DCLM', v: 13.8, ref: true },
    { label: 'Nemotron-HQ-Synth', v: 13.5 },
    { label: 'REWIRE', v: 13.5 },
    { label: 'Cosmopedia', v: 10.3 },
    { label: 'SYNTH', v: 10.0 },
  ]);

  chart('chart-prompts', [
    { label: 'Math', v: 15.3, hl: true },
    { label: 'Table', v: 14.8, hl: true },
    { label: 'Diverse QA (Nemotron)', v: 14.6 },
    { label: 'FAQ', v: 14.5, hl: true },
    { label: 'Tutorial', v: 14.3, hl: true },
    { label: 'Guided rewrite (REWIRE)', v: 13.7 },
    { label: 'Article', v: 13.6 },
    { label: 'Narrative', v: 13.6 },
    { label: 'Commentary', v: 12.3 },
  ], { ref: DCLM });

  chart('chart-size', [
    { label: 'Gemma 3 270M', v: 13.8 },
    { label: 'Gemma 3 1B', v: 15.3, hl: true },
    { label: 'Gemma 3 4B', v: 15.1 },
    { label: 'Gemma 3 12B', v: 14.7 },
    { label: 'Gemma 3 27B', v: 14.8 },
  ], { ref: DCLM });

  chart('chart-family', [
    { label: 'SmolLM2 1.7B', v: 15.9, hl: true },
    { label: 'Falcon3 1B', v: 15.0 },
    { label: 'Qwen3 1.7B', v: 14.5 },
    { label: 'Gemma 3 1B', v: 14.3 },
    { label: 'Granite 3.1 1B', v: 14.1 },
    { label: 'Llama 3.2 1B', v: 14.0 },
  ], { ref: DCLM });

  chart('chart-mix', [
    { label: 'Math · 50% synthetic', v: 15.3, hl: true },
    { label: 'Math · synthetic only', v: 15.2 },
    { label: 'Table · 50% synthetic', v: 14.8, hl: true },
    { label: 'Table · synthetic only', v: 13.7 },
    { label: 'FAQ · 50% synthetic', v: 14.5, hl: true },
    { label: 'FAQ · synthetic only', v: 13.1 },
    { label: 'Tutorial · 50% synthetic', v: 14.3, hl: true },
    { label: 'Tutorial · synthetic only', v: 12.3 },
  ], { ref: DCLM });
</script>
</body>
</html>