Spaces:

malteos
/

cc-repackage

Running

malteos
Collapse reveal toggles when navigating between wizard steps
e637202 unverified about 1 month ago
29 kB
"use strict";
/* Common Crawl WARC Repackager — wizard frontend.
   Talks to the FastAPI backend: /api/config for setup, POST /api/estimate|launch to
   start real jobs, and Server-Sent Events (/api/jobs/{id}/events, /api/examples/{key}/events)
   to stream job logs live. No job logic here — the backend wraps src/. */

const STEPS = ["Start", "Filter", "Crawls", "Output", "Estimate", "Run"];
const QUICK_CODES = ["eng","deu","fra","spa","ita","por","nld","rus","jpn","zho","ara","hin","kor","pol","tur","swe"];

let CFG = null;                 // /api/config
let LANG_NAME = {};             // code -> name (all languages)
let EXAMPLES = [];              // from config
const $ = s => document.querySelector(s);
const $$ = s => Array.from(document.querySelectorAll(s));

const state = {
  step: 0, maxStep: 0, mode: "guided", signedIn: false, isPodman: false,
  domains: "", hosts: "", langs: new Set(), sql: "", max: 0,
  crawls: new Set(), bucket: "", path: "exports/run1", name: "repackaged", flavor: "cpu-upgrade",
  est: null, demo: null, es: null,
};

/* ---------- formatting ---------- */
function fmtBytes(n){ if(n<1024) return n+" B"; const u=["KiB","MiB","GiB","TiB"]; let v=n/1024,i=0; while(v>=1024&&i<u.length-1){v/=1024;i++;} return v.toFixed(1)+" "+u[i]; }
function fmtCost(u){ if(u<=0) return "$0.00"; if(u<0.01) return "<$0.01"; return "$"+u.toFixed(2); }
function fmtDur(s){ s=Math.round(s); if(s<60) return s+"s"; if(s<3600) return Math.floor(s/60)+"m "+(s%60)+"s"; return Math.floor(s/3600)+"h "+Math.floor((s%3600)/60)+"m"; }
function escapeHtml(s){ return (s||"").replace(/[&<>"]/g,c=>({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;"}[c])); }

/* ---------- tutorial coach marks ---------- */
function clearCoach(){
  const t=$("#coachTip"); t.classList.remove("show");
  if(state.coachEl){ state.coachEl.classList.remove("coach-pulse"); state.coachEl=null; }
}
function setCoach(el, text){
  clearCoach();
  if(!el){ return; }
  const tip=$("#coachTip"); tip.textContent = text || "👉 Click here"; tip.classList.add("show");
  const r = el.getBoundingClientRect();
  tip.style.top = (r.top + window.scrollY - tip.offsetHeight - 8) + "px";
  tip.style.left = (r.left + window.scrollX) + "px";
  el.classList.add("coach-pulse"); state.coachEl = el;
}
function updateCoach(){
  if(!state.demo){ clearCoach(); return; }
  const nextAccent = n => document.querySelector(`.cc-btn-accent[data-goto="${n}"]`);
  const s = state.step;
  if(s===1) setCoach(nextAccent(2), "👉 Click here");
  else if(s===2) setCoach(nextAccent(3), "👉 Click here");
  else if(s===3) setCoach(nextAccent(4), "👉 Click here");
  else if(s===4) setCoach(state.demoEstDone ? $("#toRun") : $("#estimateBtn"), "👉 Click here");
  else if(s===5) setCoach(state.demoResultDone ? null : $("#launchBtn"), "👉 Click here");
  else clearCoach();
}
function exitDemo(){
  state.demo=null; state.demoEstDone=false; state.demoResultDone=false;
  closeES(); clearCoach(); const b=$("#demoBanner"); if(b) b.hidden=true;
}

/* ---------- stepper ---------- */
function renderStepper(){
  const el = $("#stepper"); el.innerHTML = "";
  STEPS.forEach((label,i)=>{
    const d = document.createElement("div");
    d.className = i===state.step ? "step active" : (i<=state.maxStep ? "step done" : "step");
    d.innerHTML = `<div class="step-dot">${(i<state.step)?"✓":(i+1)}</div><div class="step-label">${label}</div>`;
    if(i<=state.maxStep) d.onclick = ()=>goto(i);
    el.appendChild(d);
  });
}
// Collapse transient reveal toggles so each step starts in a clean state.
function resetToggles(){
  const a=$("#agLocal"); if(a) a.hidden=true;                 // "Run on my own hardware" commands
  const g=$("#authGate"); if(g) g.hidden=true;               // signed-out gate (re-shown on Estimate)
  const dd=$("#langDd"); if(dd) dd.classList.remove("open");  // "All languages" dropdown
  $$("details[open]").forEach(d=>{ d.open=false; });          // "More options" / "Advanced: hardware"
}
function goto(i){
  state.step = i; state.maxStep = Math.max(state.maxStep, i);
  $$(".panel").forEach(p=>p.classList.toggle("active", +p.dataset.step===i));
  resetToggles();
  renderStepper();
  if(i===4) renderReview();
  window.scrollTo({top:0, behavior:"smooth"});
  updateCoach();
}

/* ---------- examples ---------- */
function renderExamples(){
  const g = $("#exGrid"); g.innerHTML = "";
  EXAMPLES.forEach(ex=>{
    const c = document.createElement("div"); c.className = "ex-card";
    const badge = ex.kind==="sql" ? `<span class="cc-badge badge-sql">SQL filter</span>` : `<span class="cc-badge badge-form">Guided</span>`;
    c.innerHTML = `
      <div class="ex-top"><div class="ex-title">${escapeHtml(ex.title)}</div>${badge}</div>
      <div class="ex-desc">${escapeHtml(ex.description)}</div>
      <div class="ex-meta"><span class="cc-badge badge-mono">${ex.n_records.toLocaleString()} records</span><span class="cc-badge badge-mono">${fmtBytes(ex.total_bytes)}</span></div>
      <div style="margin-top:6px;"><button class="cc-btn cc-btn-accent" style="width:100%;justify-content:center;">▶ Run demo</button></div>`;
    c.querySelector("button").onclick = ()=>runDemo(ex);
    g.appendChild(c);
  });
}

/* ---------- language filter ---------- */
function toggleLang(code){ state.langs.has(code)?state.langs.delete(code):state.langs.add(code); renderLangs(); showErr("#filterErr",""); }
function renderLangs(){ renderLangChips(); renderLangSelect(); renderLangPills(); }
function renderLangChips(){
  const g = $("#langChips"); g.innerHTML = "";
  QUICK_CODES.forEach(code=>{
    const name = LANG_NAME[code] || code;
    const b = document.createElement("button"); b.className="chip"+(state.langs.has(code)?" on":"");
    b.innerHTML = `${name} <span class="code">${code}</span>`;
    b.onclick = ()=>toggleLang(code);
    g.appendChild(b);
  });
}
function renderLangSelect(){
  const list = $("#langDdList"), q = ($("#langDdSearch").value||"").trim().toLowerCase();
  const rows = CFG.languages.filter(([n,c]) => !q || n.toLowerCase().includes(q) || c.includes(q));
  list.innerHTML = "";
  rows.forEach(([name,code])=>{
    const o = document.createElement("label"); o.className="lang-dd-o";
    o.innerHTML = `<input type="checkbox" ${state.langs.has(code)?"checked":""}><span>${name}</span><span class="code">${code}</span>`;
    o.querySelector("input").onchange = ()=>toggleLang(code);
    list.appendChild(o);
  });
  $("#langDdEmpty").style.display = rows.length ? "none" : "block";
  const n = state.langs.size, cnt = $("#langMsCount"); cnt.hidden = n===0; cnt.textContent = n+" selected";
}
function renderLangPills(){
  const p = $("#langPills"); p.innerHTML = "";
  [...state.langs].sort((a,b)=>(LANG_NAME[a]||a).localeCompare(LANG_NAME[b]||b)).forEach(code=>{
    const pill = document.createElement("span"); pill.className="lang-pill";
    pill.innerHTML = `${LANG_NAME[code]||code} <span class="code" style="font-family:var(--cc-font-mono);font-size:11px;opacity:.7;">${code}</span> <button class="x" title="Remove">×</button>`;
    pill.querySelector(".x").onclick = ()=>toggleLang(code);
    p.appendChild(pill);
  });
}

/* ---------- columns palette (SQL) ---------- */
const COLS = ["url","url_host_name","url_host_registered_domain","url_host_tld","content_languages","content_mime_detected","content_mime_type","fetch_status","url_path"];
function renderCols(){
  const g = $("#colChips"); g.innerHTML = "";
  COLS.forEach(col=>{
    const b = document.createElement("button"); b.className="chip"; b.innerHTML = `<span class="code">${col}</span>`;
    b.onclick = ()=>{ const t=$("#fSql"); t.value += (t.value && !/\s$/.test(t.value)?" ":"")+col+" "; t.focus(); state.sql=t.value; };
    g.appendChild(b);
  });
}
function setMode(m){
  state.mode = m;
  $$("#modeSeg button").forEach(b=>b.classList.toggle("on", b.dataset.mode===m));
  $("#guidedFields").hidden = m!=="guided";
  $("#advFields").hidden = m!=="advanced";
}

/* ---------- crawls ---------- */
function renderCrawls(){
  const el = $("#crawlList"); el.innerHTML = "";
  (CFG.crawls||[]).forEach(id=>{
    const on = state.crawls.has(id);
    const d = document.createElement("label"); d.className="crawl-item"+(on?" on":"");
    d.innerHTML = `<input type="checkbox" ${on?"checked":""}><span class="cid">${id}</span><span class="cmeta">data + index</span>`;
    d.querySelector("input").onchange = e=>{ e.target.checked?state.crawls.add(id):state.crawls.delete(id); renderCrawls(); showErr("#crawlsErr",""); };
    el.appendChild(d);
  });
  $("#crawlCount").textContent = state.crawls.size+" selected";
  $("#crawlWarn").classList.toggle("show", state.crawls.size>(CFG.cost_guard_max_crawls||10));
}

/* ---------- review ---------- */
function collectFilterSummary(){
  if(state.mode==="advanced" && state.sql.trim()) return [["SQL filter", `<span class="mono">${escapeHtml(state.sql.trim())}</span>`]];
  const rows = [];
  if(state.domains.trim()) rows.push(["Domains", escapeHtml(state.domains.trim())]);
  if(state.hosts.trim()) rows.push(["Hostnames", escapeHtml(state.hosts.trim())]);
  if(state.langs.size) rows.push(["Languages", `<span class="mono">${[...state.langs].join(", ")}</span>`]);
  if(!rows.length) rows.push(["Filter", `<span style="color:var(--cc-text-muted)">none set</span>`]);
  return rows;
}
function renderReview(){
  const fl = CFG.flavors[state.flavor] || {usd:0};
  const rows = [
    ...collectFilterSummary(),
    ["Max records", state.max>0 ? state.max.toLocaleString() : "all"],
    ["Crawls", `<span class="mono">${[...state.crawls].join(", ")||"none"}</span>`],
    ["Output", `<span class="mono">${escapeHtml(state.bucket || "<your bucket>")}/${escapeHtml(state.path)}/${escapeHtml(state.name)}-*.warc.gz</span>`],
    ["Hardware", `<span class="mono">${state.flavor}</span> ($${fl.usd.toFixed(2)}/hr)`],
  ];
  $("#reviewSummary").innerHTML = rows.map(([k,v])=>`<div class="summary-row"><div class="k">${k}</div><div class="v">${v}</div></div>`).join("");
}

/* ---------- cost tiles ---------- */
function renderCostTiles(est){
  $("#costCard").style.display = "block";
  $("#costTiles").innerHTML = `
    <div class="tile"><div class="tl">Records</div><div class="tv">${est.n_records.toLocaleString()}</div></div>
    <div class="tile"><div class="tl">Data size</div><div class="tv">${fmtBytes(est.total_bytes)}</div></div>
    <div class="tile"><div class="tl">Est. runtime</div><div class="tv">${fmtDur(est.duration_seconds)}</div></div>
    <div class="tile"><div class="tl">Est. cost</div><div class="tv accent">${fmtCost(est.cost_usd)}</div></div>`;
}

/* ---------- payload ---------- */
function syncForm(){
  if(state.demo) return;   // demo fields are fixed by the example; don't overwrite from the (empty) guest inputs
  hideErrors();            // editing dismisses shown validation errors (re-checked on next click)
  state.domains=$("#fDomains").value; state.hosts=$("#fHosts").value; state.sql=$("#fSql").value;
  state.max=Math.max(0,parseInt($("#fMax").value||"0",10));
  state.path=$("#oPath").value; state.name=$("#oName").value;   // no default: validation requires a non-empty name
  state.bucket=$("#oBucket").value||state.bucket; state.flavor=$("#oFlavor").value||state.flavor;
}
function payload(extra){
  return Object.assign({
    domains: state.mode==="advanced"?"":state.domains, hostnames: state.mode==="advanced"?"":state.hosts,
    languages: state.mode==="advanced"?[]:[...state.langs], sql_where: state.mode==="advanced"?state.sql.trim():"",
    max_records: state.max, crawls: [...state.crawls],
    target_bucket: state.bucket, target_path: state.path, name: state.name, flavor: state.flavor,
  }, extra||{});
}

/* ---------- SSE plumbing + job status ---------- */
const CONSOLE = {
  est: {body:"#estConsole", spin:"#estSpin", stage:"#estStage"},
  run: {body:"#runConsole", spin:"#runSpin", stage:"#runStage"},
};
function jobStatus(kind, text, active){
  const c = CONSOLE[kind]; $(c.spin).hidden = !active; $(c.stage).textContent = text || "";
}
function humanStage(s){
  return ({SCHEDULING:"waiting for a worker", RUNNING:"running", COMPLETED:"completed",
           ERROR:"failed", CANCELED:"canceled", DELETED:"deleted", UNKNOWN:"starting up"}[s])
         || (s||"").toLowerCase();
}
function setBtn(sel, loading, label){
  const b = $(sel); b.disabled = loading;
  b.innerHTML = loading ? `<span class="spinner"></span> ${label}` : label;
}
function closeES(){ if(state.es){ state.es.close(); state.es=null; } }
function streamJob(jobId, kind, onDone){
  closeES();
  const body = $(CONSOLE[kind].body);
  jobStatus(kind, "connecting…", true);
  const es = new EventSource(`/api/jobs/${jobId}/events`); state.es = es;
  es.onmessage = ev => {
    const d = JSON.parse(ev.data);
    if(d.type==="tick"){
      jobStatus(kind, humanStage(d.stage), true);
      if(d.log && d.log.trim()){ body.textContent = d.log; body.scrollTop = body.scrollHeight; }
    } else if(d.type==="done"){ closeES(); jobStatus(kind, humanStage(d.stage), false); onDone(d); }
  };
  es.onerror = ()=>{ closeES(); jobStatus(kind, "connection lost", false); };
}

/* ---------- run-it-yourself commands (signed-out gate) ---------- */
const CDXT_SPEC = "cdx_toolkit[hf] @ git+https://github.com/commoncrawl/cdx_toolkit.git@feat/warc-range-sources";
function shq(v){ return "'" + String(v).replace(/'/g, "'\\''") + "'"; }
function buildLocalCommands(){
  syncForm();
  const env = ["CC_INDEX_MODE=cdn", `CC_CRAWLS=${shq([...state.crawls].join(","))}`];
  if(state.mode==="advanced" && state.sql.trim()){
    env.push(`CC_SQL_WHERE=${shq(state.sql.trim())}`);
  } else {
    if(state.domains.trim()) env.push(`CC_DOMAINS=${shq(state.domains.trim())}`);
    if(state.hosts.trim()) env.push(`CC_HOSTNAMES=${shq(state.hosts.trim())}`);
    if(state.langs.size) env.push(`CC_LANGUAGES=${shq([...state.langs].join(","))}`);
  }
  if(state.max>0) env.push(`CC_MAX_RECORDS=${state.max}`);
  env.push("CC_RANGES_OUT=ranges.csv");
  const name = state.name || "repackaged";
  return [
    "# 1) Get the tool (public repo) + dependencies",
    "git clone https://huggingface.co/spaces/malteos/cc-repackage && cd cc-repackage",
    `pip install duckdb huggingface_hub uvloop ${shq(CDXT_SPEC)}`,
    "",
    "# 2) Scan the Common Crawl index for your filter -> ranges.csv",
    "#    (reads the public commoncrawl bucket over its CDN — no HF account needed)",
    `${env.join(" ")} python -m src.index_query`,
    "",
    "# 3) Fetch + repackage the matching WARC records into ./" + name + "-*.warc.gz",
    "CDXT_UVLOOP=1 cdxt repackage --target-source csv --csv-path ranges.csv \\",
    "  --warc-download-prefix hf://buckets/commoncrawl/commoncrawl --hf-reader cdn \\",
    `  --prefix ./${name} --processes "$(nproc)" --parallel_readers 48`,
  ].join("\n");
}
function showAuthGate(){
  $("#estConsoleWrap").classList.remove("show"); $("#costCard").style.display="none"; $("#toRun").disabled=true;
  $("#localCmds").textContent = buildLocalCommands();
  $("#agLocal").hidden = false;              // auto-expand the run-locally commands
  $("#authGate").hidden = false;
  $("#authGate").scrollIntoView({behavior:"smooth", block:"nearest"});
}

/* ---------- estimate ---------- */
$("#estimateBtn").onclick = async ()=>{
  if(state.demo){ return runDemoPhase("estimate"); }
  syncForm();
  const bad = firstInvalidStep();
  if(bad){ goto(bad); return; }   // jump to the first invalid step (its error is already shown)
  if(!state.isPodman && !state.signedIn){ return showAuthGate(); }
  const body = $("#estConsole");
  $("#authGate").hidden = true;
  $("#estConsoleWrap").classList.add("show"); $("#costCard").style.display="none"; $("#toRun").disabled = true;
  body.textContent = "⏳ Submitting estimate job to Hugging Face…";
  jobStatus("est", "submitting…", true); setBtn("#estimateBtn", true, "Estimating…");
  let r;
  try { r = await (await fetch("/api/estimate",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(payload())})).json(); }
  catch(e){ body.textContent = "❌ Request failed: "+e; jobStatus("est","failed",false); setBtn("#estimateBtn",false,"📊 Estimate costs via HF job"); return; }
  if(r.error){ body.textContent = "❌ "+r.error; jobStatus("est","failed",false); setBtn("#estimateBtn",false,"📊 Estimate costs via HF job"); return; }
  body.textContent = `✅ Job launched: ${r.job_id}\n⏳ Waiting for a worker to pick it up — this can take a few seconds…`;
  streamJob(r.job_id, "est", d=>{
    setBtn("#estimateBtn", false, "📊 Estimate costs via HF job");
    if(d.estimate){ state.est=d.estimate; renderCostTiles(d.estimate); $("#toRun").disabled = d.estimate.n_records<=0; }
    else { body.textContent += "\n❌ "+(d.error||"Estimate failed."); }
  });
};

/* ---------- launch ---------- */
$("#launchBtn").onclick = async ()=>{
  if(state.demo){ return runDemoPhase("fetch"); }
  syncForm();
  const body = $("#runConsole");
  $("#runConsoleWrap").classList.add("show");
  $("#result").classList.remove("show"); $("#progress").classList.add("show"); $("#progressBar").style.width="0%";
  body.textContent = "⏳ Submitting repackage job to Hugging Face…";
  jobStatus("run","submitting…",true); setBtn("#launchBtn", true, "Repackaging…"); $("#cancelBtn").disabled=false;
  let pct=0; const timer=setInterval(()=>{ pct=Math.min(95,pct+8); $("#progressBar").style.width=pct+"%"; },800);
  const nrec = state.est ? state.est.n_records : 0;
  let r;
  try { r = await (await fetch("/api/launch",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(payload({n_records:nrec}))})).json(); }
  catch(e){ clearInterval(timer); body.textContent="❌ Request failed: "+e; jobStatus("run","failed",false); setBtn("#launchBtn",false,"🚀 Launch repackage"); $("#cancelBtn").disabled=true; return; }
  if(r.error){ clearInterval(timer); body.textContent="❌ "+r.error; jobStatus("run","failed",false); setBtn("#launchBtn",false,"🚀 Launch repackage"); $("#cancelBtn").disabled=true; return; }
  state.activeJob = r.job_id;
  body.textContent = `✅ Job launched: ${r.job_id}\n⏳ Waiting for a worker to pick it up — this can take a few seconds…`;
  streamJob(r.job_id, "run", d=>{
    clearInterval(timer); $("#progressBar").style.width="100%";
    setBtn("#launchBtn", false, "🚀 Launch repackage"); $("#cancelBtn").disabled=true;
    if(d.result) showResult(Object.assign({name:state.name}, d.result));
  });
};
$("#cancelBtn").onclick = async ()=>{
  if(state.activeJob){ try{ await fetch(`/api/jobs/${state.activeJob}/cancel`,{method:"POST"}); }catch(e){} }
  closeES(); $("#cancelBtn").disabled=true; setBtn("#launchBtn", false, "🚀 Launch repackage");
  jobStatus("run","canceled",false);
  const c=$("#runConsole"); c.textContent += "\n🛑 Cancel requested.";
};

function showResult(res){
  const r = $("#result"); r.classList.add("show");
  if(res.demo){
    $("#resultNote").textContent = "🧪 Precomputed example — no HF Job ran and no sign-in was needed. The download below is a real file.";
    $("#dlRow").innerHTML =
      `<a class="dl-btn" href="${res.download_url}" target="_blank" rel="noopener">⬇ Download ${res.file} <span class="sz">${fmtBytes(res.total_bytes)}</span></a>
       <a class="link-plain" href="${res.folder_url}" target="_blank" rel="noopener">📂 Browse folder</a>`;
  } else if(res.completed){
    $("#resultNote").textContent = "Written to your bucket.";
    $("#dlRow").innerHTML =
      `<a class="dl-btn" href="${res.bucket_url}" target="_blank" rel="noopener">📂 Open in your bucket</a>
       <span class="link-plain" style="cursor:default">${escapeHtml(res.output_uri)}</span>`;
  } else {
    $("#result").classList.remove("show");
    $("#runConsole").textContent += "\n❌ Job did not complete successfully — see logs.";
  }
}

/* ---------- precomputed demo as a guided tutorial ---------- */
function runDemo(ex){
  state.demo = ex; state.demoEstDone=false; state.demoResultDone=false; state.est=null;
  state.mode = ex.kind==="sql" ? "advanced" : "guided";
  const f = ex.form;
  state.domains=f.domains; state.hosts=f.hostnames; state.langs=new Set(f.languages); state.sql=f.sql_where;
  state.max=f.max_records; state.crawls=new Set(f.crawls); state.name=f.name;
  state.bucket=ex.bucket; state.path=ex.path;   // real hosted location of this example's output
  setMode(state.mode);
  $("#fDomains").value=f.domains; $("#fHosts").value=f.hostnames; $("#fSql").value=f.sql_where; $("#fMax").value=f.max_records;
  renderLangs(); renderCrawls();
  // reset the estimate / run panels
  $("#estConsoleWrap").classList.remove("show"); $("#estConsole").textContent="";
  $("#costCard").style.display="none"; $("#toRun").disabled=true;
  $("#runConsoleWrap").classList.remove("show"); $("#runConsole").textContent="";
  $("#result").classList.remove("show"); $("#progress").classList.remove("show");
  // tutorial banner (shown only now, on demo click) + start at the Filter step (pre-filled)
  $("#demoBanner").hidden=false;
  state.maxStep = 5; goto(1);            // goto() -> updateCoach() places the first "Click here"
}

/* One tutorial step: stream the precomputed estimate OR fetch logs for the active demo. */
function runDemoPhase(phase){
  clearCoach(); closeES();
  const kind = phase==="estimate" ? "est" : "run";
  if(phase==="estimate"){
    $("#estConsoleWrap").classList.add("show"); $("#estConsole").textContent="⏳ Starting precomputed estimate…";
    $("#costCard").style.display="none"; $("#toRun").disabled=true; setBtn("#estimateBtn", true, "Estimating…");
  } else {
    $("#runConsoleWrap").classList.add("show"); $("#runConsole").textContent="⏳ Starting precomputed repackage…";
    $("#result").classList.remove("show"); $("#progress").classList.add("show"); $("#progressBar").style.width="12%";
    setBtn("#launchBtn", true, "Repackaging…"); $("#cancelBtn").disabled=false;
  }
  jobStatus(kind, "running", true);
  const es = new EventSource(`/api/examples/${state.demo.key}/events?phase=${phase}`); state.es = es;
  let first = true;
  es.onmessage = ev=>{
    const d = JSON.parse(ev.data);
    if(d.type==="log"){
      const el = phase==="estimate" ? $("#estConsole") : $("#runConsole");
      if(first){ el.textContent=""; first=false; }   // clear the "starting…" placeholder
      el.textContent += (el.textContent?"\n":"")+d.line; el.scrollTop=el.scrollHeight;
      if(phase==="fetch"){ const b=$("#progressBar"); b.style.width=Math.min(95,parseFloat(b.style.width||"12")+12)+"%"; }
    } else if(d.type==="estimate"){ state.est=d.estimate; renderCostTiles(d.estimate); $("#toRun").disabled=false; }
    else if(d.type==="done"){
      closeES(); jobStatus(kind, "completed", false);
      if(phase==="estimate"){ setBtn("#estimateBtn", false, "📊 Estimate costs via HF job"); state.demoEstDone=true; }
      else { $("#progressBar").style.width="100%"; setBtn("#launchBtn", false, "🚀 Launch repackage"); $("#cancelBtn").disabled=true; state.demoResultDone=true; if(d.result) showResult(d.result); }
      updateCoach();
    }
  };
  es.onerror = ()=>{ closeES(); jobStatus(kind,"connection lost",false); if(phase==="estimate") setBtn("#estimateBtn",false,"📊 Estimate costs via HF job"); else { setBtn("#launchBtn",false,"🚀 Launch repackage"); $("#cancelBtn").disabled=true; } };
}

/* ---------- form validation ---------- */
function showErr(sel, msg){ const e=$(sel); if(!e) return; e.textContent=msg||""; e.hidden=!msg; }
function hideErrors(){ ["#filterErr","#crawlsErr","#outputErr"].forEach(s=>showErr(s,"")); }
function validateFilter(){
  if(state.mode==="advanced") return state.sql.trim() ? "" : "Enter a SQL WHERE expression, or switch to Guided mode.";
  return (state.domains.trim() || state.hosts.trim() || state.langs.size)
    ? "" : "Set at least one of: registered domains, exact hostnames, or content languages.";
}
function validateCrawls(){ return state.crawls.size ? "" : "Select at least one crawl to scan."; }
function validateOutput(){
  if(!state.path.trim()) return "Enter a path within the bucket.";
  if(!state.name.trim()) return "Enter an output WARC name.";
  return "";
}
const STEP_CHECK = { 1: ["#filterErr", validateFilter], 2: ["#crawlsErr", validateCrawls], 3: ["#outputErr", validateOutput] };
// Validate one step; render + return its error message ("" if valid).
function checkStep(step){
  const entry = STEP_CHECK[step]; if(!entry) return "";
  const [sel, fn] = entry; const msg = fn(); showErr(sel, msg); return msg;
}
// First invalid step in 1..3, or 0 if all valid (also renders that step's error).
function firstInvalidStep(){
  for(const s of [1,2,3]){ if(checkStep(s)) return s; }
  return 0;
}

/* ---------- wiring ---------- */
function wire(){
  $$("#modeSeg button").forEach(b=>b.onclick=()=>{ setMode(b.dataset.mode); showErr("#filterErr",""); });
  $$("[data-goto]").forEach(b=>b.onclick=()=>{
    syncForm();
    const target = +b.dataset.goto;
    if(!state.demo && target > state.step && checkStep(state.step)) return;  // block forward nav on invalid step
    goto(target);
  });
  $("#langMsBtn").onclick=(e)=>{ e.stopPropagation(); const dd=$("#langDd"); if(dd.classList.toggle("open")) $("#langDdSearch").focus(); };
  $("#langDdSearch").oninput=()=>renderLangSelect();
  $("#langDd").onclick=(e)=>e.stopPropagation();
  document.addEventListener("click", ()=>$("#langDd").classList.remove("open"));
  $("#refreshCrawls").onclick=refreshCrawls;
  $("#restartBtn").onclick=()=>{ exitDemo(); };
  $("#demoExit").onclick=()=>{ exitDemo(); };
  $("#homeLink").onclick=(e)=>{ e.preventDefault(); exitDemo(); goto(0); };
  $("#showLocalBtn").onclick=()=>{ const a=$("#agLocal"); a.hidden=!a.hidden; if(!a.hidden) $("#localCmds").textContent=buildLocalCommands(); };
  $("#copyCmdsBtn").onclick=async ()=>{
    try{ await navigator.clipboard.writeText($("#localCmds").textContent); const b=$("#copyCmdsBtn"); const t=b.textContent; b.textContent="✅ Copied"; setTimeout(()=>b.textContent=t,1500); }catch(e){}
  };
  window.addEventListener("resize", ()=>{ if(state.demo) updateCoach(); });
  ["fDomains","fHosts","fSql","fMax","oPath","oName"].forEach(id=>{ const el=$("#"+id); if(el) el.addEventListener("input",syncForm); });
  $("#oBucket").addEventListener("change",syncForm); $("#oFlavor").addEventListener("change",syncForm);
  $("#signinBtn").onclick=()=>{ if(state.isPodman){ return; } location.href="/login"; };
}

async function refreshCrawls(){
  try{ const r = await (await fetch("/api/crawls")).json(); if(r.crawls){ CFG.crawls = r.crawls; renderCrawls(); } }catch(e){}
}
async function loadBuckets(){
  try{
    const r = await (await fetch("/api/buckets")).json();
    const sel = $("#oBucket"); sel.innerHTML="";
    (r.buckets||[]).forEach(b=>{ const o=document.createElement("option"); o.value=b; o.textContent=b; sel.appendChild(o); });
    if(!r.buckets || !r.buckets.length){ const o=document.createElement("option"); o.value=""; o.textContent="(no writable buckets — create one on huggingface.co)"; sel.appendChild(o); }
    state.bucket = sel.value;
  }catch(e){}
}
function applyAuthUI(){
  if(state.isPodman){
    $("#signinBtn").style.display="none"; $("#hfUser").classList.add("show"); $("#hfUser").innerHTML='<span class="dot"></span> Local mode (guest)';
    $("#outGate").classList.remove("show");
  } else if(state.signedIn){
    $("#signinBtn").style.display="none"; $("#hfUser").classList.add("show");
    $("#hfUser").innerHTML=`<span class="dot"></span> Signed in as <strong>&nbsp;${escapeHtml(CFG.user||"you")}</strong>`;
    $("#outGate").classList.remove("show");
  } else {
    $("#outGate").classList.add("show");
  }
}
function populateFlavors(){
  const sel = $("#oFlavor"); sel.innerHTML="";
  Object.entries(CFG.flavors).forEach(([name,f])=>{
    const o=document.createElement("option"); o.value=name;
    o.textContent = `${name} — $${f.usd.toFixed(2)}/hr` + (f.vcpus?` (${f.vcpus} vCPU)`:"");
    if(name===CFG.default_flavor) o.selected=true;
    sel.appendChild(o);
  });
  state.flavor = CFG.default_flavor;
}

function deepLink(){
  const h = location.hash; let m;
  if((m=h.match(/run=([\w-]+)/))){ const ex=EXAMPLES.find(e=>e.key===m[1]); if(ex){ runDemo(ex); return; } }
  if((m=h.match(/demo=([\w-]+)/))){ const ex=EXAMPLES.find(e=>e.key===m[1]); if(ex){ runDemo(ex); return; } }
  if((m=h.match(/step=(\d)/))){ state.maxStep=5; goto(Math.max(0,Math.min(5,+m[1]))); }
}

async function init(){
  CFG = await (await fetch("/api/config")).json();
  LANG_NAME = Object.fromEntries(CFG.languages.map(([n,c])=>[c,n]));
  EXAMPLES = CFG.examples;
  state.isPodman = CFG.is_podman; state.signedIn = CFG.signed_in;
  if(!state.crawls.size && CFG.crawls.length) state.crawls = new Set([CFG.crawls[0]]);
  wire();
  renderStepper(); renderExamples(); renderLangs(); renderCols(); renderCrawls(); populateFlavors(); setMode("guided");
  applyAuthUI();
  await loadBuckets();
  goto(0);
  deepLink();
}
init();