Blame for server/static/index.html

<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Common Crawl WARC Repackager</title>
<link rel="icon" href="/static/assets/favicon.png" />
<link rel="stylesheet" href="/static/app.css" />
</head>
<body>
<header class="cc-header-wrap">
  <div class="cc-header">
    <a class="cc-brand" href="/" id="homeLink" aria-label="WARC Repackager — home">
      <span class="cc-logo"><img src="/static/assets/logo.svg" alt="Common Crawl" height="30"></span>
      <span class="cc-htxt">
        <span class="cc-h1">WARC Repackager</span>
        <span class="cc-hp">Extract a custom WARC archive from Common Crawl</span>
      </span>
    </a>
  </div>
</header>

<main class="cc-twrap">
  <!-- Stepper -->
  <nav class="stepper" id="stepper"></nav>

  <!-- Tutorial banner (demo mode) -->
  <div id="demoBanner" hidden>
    <span>🎓 Tutorial mode — follow the <em>“Click here”</em> prompts to run this precomputed example.</span>
    <span class="grow"></span>
    <button id="demoExit">Exit tutorial</button>
  </div>

  <!-- STEP 0: Start -->
  <section class="panel" data-step="0">
    <div class="card">
      <h2>Start</h2>
      <p class="sub">Repackage the Common Crawl records you care about into a new WARC archive in your own Hugging Face bucket. Sign in to run your own — or try a precomputed example right now, no account needed.</p>
      <div class="start-split">
        <div class="card signin-card" style="margin:0;">
          <div class="txt"><strong>Run your own repackaging.</strong><br>Sign in with Hugging Face to submit jobs to your account.</div>
          <div>
            <button class="hf-btn" id="signinBtn">🤗 Sign in with Hugging Face</button>
            <span class="hf-user" id="hfUser"><span class="dot"></span> Signed in as <strong>&nbsp;you</strong></span>
          </div>
        </div>
      </div>
      <div style="margin-top:18px;display:flex;justify-content:flex-end;">
        <button class="cc-btn cc-btn-accent cc-btn-lg" data-goto="1" id="startBtn">Build a custom archive →</button>
      </div>
    </div>

    <div class="card">
      <h2>Examples <span style="font-weight:500;font-size:13px;color:var(--cc-text-muted)">— try without signing in</span></h2>
      <p class="sub">Each example runs on <strong>precomputed</strong> data: the form fills in, the job logs stream like a real run, and you get a working download link. Two use the structured fields, two use the advanced SQL filter.</p>
      <div class="ex-grid" id="exGrid"></div>
    </div>
  </section>

  <!-- STEP 1: Filter -->
  <section class="panel" data-step="1">
    <div class="card">
      <h2>Define your filter</h2>
      <p class="sub">Pick which records to keep. Use the guided fields for the common cases, or switch to raw SQL for full control over the columnar index.</p>
      <div class="seg" id="modeSeg">
        <button data-mode="guided" class="on">Guided</button>
        <button data-mode="advanced">Advanced (SQL)</button>
      </div>

      <div id="guidedFields">
        <p class="group-note">Set at least one of the three below — registered domains, exact hostnames, or content languages. Each is optional on its own.</p>
        <div class="row2">
          <div class="field">
            <label>Registered domains <span style="font-weight:500;color:var(--cc-text-muted)">(optional)</span></label>
            <input class="cc-input" id="fDomains" placeholder="example.com, example.org">
            <div class="hint">Matches the domain and all its subdomains.</div>
          </div>
          <div class="field">
            <label>Exact hostnames <span style="font-weight:500;color:var(--cc-text-muted)">(optional)</span></label>
            <input class="cc-input" id="fHosts" placeholder="www.example.com">
            <div class="hint">Matches exact hosts only. Combine with domains.</div>
          </div>
        </div>
        <div class="field">
          <label>Content languages <span style="font-weight:500;color:var(--cc-text-muted)">(optional)</span></label>
          <div class="quick-label">Quick picks</div>
          <div class="chips" id="langChips"></div>
          <div class="lang-ms" id="langMs">
            <button type="button" class="lang-ms-btn" id="langMsBtn">
              <span>All languages…</span><span class="ms-count" id="langMsCount" hidden></span><span class="caret">▾</span>
            </button>
            <div class="lang-dd" id="langDd">
              <input class="lang-dd-i" id="langDdSearch" placeholder="Search all 119 languages…" autocomplete="off">
              <div class="lang-dd-list" id="langDdList"></div>
              <div class="lang-dd-empty" id="langDdEmpty">No language matches.</div>
            </div>
          </div>
          <div class="lang-pills" id="langPills"></div>
          <div class="hint">Keeps records whose <em>primary</em> detected language is one of these (CLD2 / ISO-639-3). Use the quick picks, or search the full set of 119.</div>
        </div>
      </div>

      <div id="advFields" hidden>
        <div class="field">
          <label>SQL <code>WHERE</code> predicate</label>
          <textarea class="cc-input" id="fSql" rows="3" placeholder="content_mime_detected = 'application/pdf' AND fetch_status = 200"></textarea>
          <div class="hint">A single boolean expression over the index columns. Replaces the guided fields. Validated (DuckDB EXPLAIN) before the scan runs.</div>
          <div class="chips" style="margin-top:8px;" id="colChips"></div>
        </div>
      </div>

      <details class="disclose">
        <summary>More options</summary>
        <div class="field field-num">
          <label>Max records <span style="font-weight:500;color:var(--cc-text-muted)">(optional)</span></label>
          <input class="cc-input" id="fMax" type="number" min="0" value="0">
          <div class="hint">Cap the output to the first N matching records. 0 = all.</div>
        </div>
      </details>
    </div>
    <div class="step-error" id="filterErr" hidden></div>
    <div class="nav">
      <button class="cc-btn" data-goto="0">← Back</button>
      <div class="spacer"></div>
      <button class="cc-btn cc-btn-accent" data-goto="2" id="toCrawls">Next: Crawls →</button>
    </div>
  </section>

  <!-- STEP 2: Crawls -->
  <section class="panel" data-step="2">
    <div class="card">
      <h2>Choose crawls</h2>
      <p class="sub">Select which Common Crawl archives to scan. Fewer crawls keep the index scan cheap.</p>
      <div class="crawl-list" id="crawlList"></div>
      <div class="warn" id="crawlWarn">⚠️ More than 10 crawls selected — the index scan may be large and slow.</div>
      <div style="margin-top:12px;"><button class="cc-btn" id="refreshCrawls">🔄 Refresh crawl list</button> <span id="crawlCount" style="font-size:13px;color:var(--cc-text-secondary);margin-left:8px;"></span></div>
    </div>
    <div class="step-error" id="crawlsErr" hidden></div>
    <div class="nav">
      <button class="cc-btn" data-goto="1">← Back</button>
      <div class="spacer"></div>
      <button class="cc-btn cc-btn-accent" data-goto="3">Next: Output →</button>
    </div>
  </section>

  <!-- STEP 3: Output -->
  <section class="panel" data-step="3">
    <div class="card">
      <h2>Output destination</h2>
      <p class="sub">Where the repackaged WARC is written. Must be an existing Hugging Face bucket you can write to.</p>
      <div class="gate show" id="outGate">You're browsing as a guest. Sign in on the Start step to pick one of your buckets — a placeholder is shown for now.</div>
      <div class="row2">
        <div class="field">
          <label>Target bucket</label>
          <select class="cc-select" id="oBucket"><option>you/my-exports</option><option>your-org/shared-bucket</option></select>
          <div class="hint">Buckets you can write to (yours + write-access orgs).</div>
        </div>
        <div class="field">
          <label>Path within bucket</label>
          <input class="cc-input" id="oPath" placeholder="exports/run1" value="exports/run1">
          <div class="hint">Sub-folder for the output (required); created if missing.</div>
        </div>
      </div>
      <div class="field" style="max-width:340px;">
        <label>Output WARC name</label>
        <input class="cc-input" id="oName" value="repackaged">
        <div class="hint">Base name for the output file(s), e.g. <span class="mono">repackaged</span> → <span class="mono">repackaged-001.warc.gz</span>.</div>
      </div>
      <details class="disclose">
        <summary>Advanced: hardware</summary>
        <div class="field" style="max-width:340px;">
          <label>Hardware flavor</label>
          <select class="cc-select" id="oFlavor">
            <option value="cpu-basic">cpu-basic — $0.01/hr</option>
            <option value="cpu-upgrade" selected>cpu-upgrade — $0.03/hr (8 vCPU)</option>
            <option value="cpu-xl">cpu-xl — $1.00/hr</option>
            <option value="cpu-performance">cpu-performance — $1.90/hr</option>
          </select>
        </div>
      </details>
    </div>
    <div class="step-error" id="outputErr" hidden></div>
    <div class="nav">
      <button class="cc-btn" data-goto="2">← Back</button>
      <div class="spacer"></div>
      <button class="cc-btn cc-btn-accent" data-goto="4">Next: Estimate →</button>
    </div>
  </section>

  <!-- STEP 4: Estimate -->
  <section class="panel" data-step="4">
    <div class="card">
      <h2>Review &amp; estimate</h2>
      <p class="sub">A cheap index-scan job counts the matching records and estimates the runtime and cost before you commit.</p>
      <div class="summary" id="reviewSummary"></div>
      <div class="est-actions">
        <button class="cc-btn cc-btn-accent cc-btn-lg" id="estimateBtn">📊 Estimate costs via HF job</button>
        <button class="cc-btn" id="showLocalBtn">💻 Run on my own hardware</button>
      </div>

      <div class="ag-local" id="agLocal" hidden>
        <div class="ag-local-head">Run locally — copy &amp; paste (reproduces your exact filter; reads the public Common Crawl bucket, no account needed):</div>
        <pre class="code-block" id="localCmds"></pre>
        <button class="cc-btn cc-btn-sm" id="copyCmdsBtn">📋 Copy commands</button>
      </div>

      <div class="auth-gate" id="authGate" hidden>
        <div class="ag-head">🔒 Sign in to run this on Hugging Face Jobs</div>
        <p class="ag-text">Estimating and repackaging run as <a href="https://huggingface.co/docs/hub/jobs" target="_blank" rel="noopener noreferrer">HF Jobs</a> on your own account (needs a Pro/Team plan). Sign in to continue — or use <strong>“Run on my own hardware”</strong> above to run it yourself (no account needed).</p>
        <div class="ag-actions">
          <a class="hf-btn" href="/login">🤗 Sign in with Hugging Face</a>
        </div>
      </div>

      <div class="console-wrap" id="estConsoleWrap">
        <div class="console-head"><span>Estimate job · logs</span><span class="job-status"><span class="spinner" id="estSpin" hidden></span><span class="job-stage" id="estStage"></span></span></div>
        <div class="console" id="estConsole"></div>
      </div>
      <div class="cost-card" id="costCard" style="display:none;">
        <div class="cost-head">💸 Cost estimate</div>
        <div class="tiles" id="costTiles"></div>
        <div class="cost-note">Estimate assumes conservative benchmark throughput; actual cost is billed per second of job runtime.</div>
      </div>
    </div>
    <div class="nav">
      <button class="cc-btn" data-goto="3">← Back</button>
      <div class="spacer"></div>
      <button class="cc-btn cc-btn-accent" data-goto="5" id="toRun" disabled>Next: Run →</button>
    </div>
  </section>

  <!-- STEP 5: Run -->
  <section class="panel" data-step="5">
    <div class="card">
      <h2>Run repackage</h2>
      <p class="sub">This launches the fetch job, streams progress, and writes the WARC to your bucket. You can cancel any time.</p>
      <div style="display:flex;gap:10px;align-items:center;">
        <button class="cc-btn cc-btn-accent cc-btn-lg" id="launchBtn">🚀 Launch repackage</button>
        <button class="cc-btn cc-btn-danger" id="cancelBtn" disabled>🛑 Cancel</button>
      </div>
      <div class="progress" id="progress"><div class="bar" id="progressBar"></div></div>
      <div class="console-wrap" id="runConsoleWrap">
        <div class="console-head"><span>Repackage job · logs</span><span class="job-status"><span class="spinner" id="runSpin" hidden></span><span class="job-stage" id="runStage"></span></span></div>
        <div class="console" id="runConsole"></div>
      </div>
      <div class="result" id="result">
        <h3>✅ Repackaging complete</h3>
        <div class="demo-note" id="resultNote"></div>
        <div class="dl-row" id="dlRow"></div>
      </div>
    </div>
    <div class="nav">
      <button class="cc-btn" data-goto="4">← Back</button>
      <div class="spacer"></div>
      <button class="cc-btn" data-goto="0" id="restartBtn">Start over</button>
    </div>
  </section>
</main>

<footer class="cc-footer">
  <div class="cc-footer-inner">
    <div class="cc-footer-info">
      <span><a href="https://commoncrawl.org" target="_blank" rel="noopener noreferrer">Common Crawl</a> is a California 501(c)(3) registered non-profit organization. Hosting of Common Crawl data is provided by <a href="https://huggingface.co/storage" target="_blank">Hugging Face Storage Buckets</a>.</span>
      <div class="cc-footer-links">
        <a href="https://commoncrawl.org/terms-of-use" target="_blank" rel="noopener noreferrer">Terms of Use</a>
        <a href="https://commoncrawl.org/privacy-policy" target="_blank" rel="noopener noreferrer">Privacy</a>
      </div>
    </div>
  </div>
</footer>


<div id="coachTip">👉 Click here</div>
<script src="/static/app.js"></script>
</body>
</html>