dehezhang2 commited on
Commit
26418a5
·
verified ·
1 Parent(s): 1a5b1f8

deploy rater (Docker/FastAPI): registration + completeness gate + /api/submit → dataset

Browse files
Files changed (2) hide show
  1. static/index.html +61 -7
  2. static/mobile.html +28 -2
static/index.html CHANGED
@@ -999,7 +999,9 @@ function scaleLabel(score) {
999
  /* =========================================================================
1000
  * State
1001
  * ========================================================================= */
1002
- let CASES = []; // raw manifest
 
 
1003
  let SHUFFLE = {}; // case_id -> [orig_idx, ...] (slot -> orig)
1004
  let RATINGS = {}; // case_id -> model_name -> { qkey: 1..5 }
1005
  let RATER = { id: "", name: "", email: "", note: "",
@@ -1067,6 +1069,26 @@ function genRaterId() {
1067
  /* =========================================================================
1068
  * Shuffling (anonymization)
1069
  * ========================================================================= */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1070
  function buildShuffle(cases) {
1071
  // Deterministic per (rater, case) and ALWAYS recomputed on load. The
1072
  // recomputation is idempotent (same seed → same order), so it costs nothing
@@ -1601,8 +1623,10 @@ function renderToolbar() {
1601
  const newName = e.target.value.trim();
1602
  if (!newName) { e.target.value = RATER.name; return; }
1603
  RATER.name = newName;
 
 
 
1604
  saveState();
1605
- // re-render so the admin-only toggle visibility refreshes on name change
1606
  renderShell();
1607
  },
1608
  });
@@ -1630,13 +1654,20 @@ function renderToolbar() {
1630
 
1631
  tb.appendChild(el("div", { class: "sep" }));
1632
 
1633
- // Admin-only reveal toggle. Non-admins always see anonymized view (and
1634
- // anonymize is force-set to true on first paint below).
 
1635
  if (isAdmin()) {
1636
  tb.appendChild(el("button", {
1637
  class: "btn ghost",
1638
  onclick: toggleAnon,
1639
  }, anonymize ? t("anonymized") : t("reveal_models")));
 
 
 
 
 
 
1640
  tb.appendChild(el("div", { class: "sep" }));
1641
  }
1642
 
@@ -2407,7 +2438,9 @@ function handleRegistrationSubmit() {
2407
  RATER.id = email.replace(/[^a-z0-9]+/gi, "_").replace(/^_+|_+$/g, "");
2408
  }
2409
  if (!RATER.started_at) RATER.started_at = new Date().toISOString();
2410
- // Re-shuffle since the seed depends on rater.id.
 
 
2411
  SHUFFLE = {};
2412
  buildShuffle(CASES);
2413
  saveState();
@@ -2666,8 +2699,8 @@ function migrateLegacyRatingsV1(legacy, allCases) {
2666
  renderEmptyApp(t("empty_app_load_error"));
2667
  return;
2668
  }
2669
- CASES = found.data || [];
2670
- if (!Array.isArray(CASES) || CASES.length === 0) {
2671
  renderEmptyApp(t("empty_app_empty_manifest"));
2672
  return;
2673
  }
@@ -2683,6 +2716,22 @@ function migrateLegacyRatingsV1(legacy, allCases) {
2683
  anonymize = prior.anonymize !== false;
2684
  RATINGS = prior.ratings || {};
2685
  SHUFFLE = prior.shuffle || {};
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2686
  } else {
2687
  // No v2 state. Try a one-time migration from v1.
2688
  const legacyRaw = localStorage.getItem(STORAGE_KEY_LEGACY);
@@ -2710,6 +2759,11 @@ function migrateLegacyRatingsV1(legacy, allCases) {
2710
  // Non-admins always see anonymized view — they can't toggle.
2711
  if (!isAdmin()) anonymize = true;
2712
 
 
 
 
 
 
2713
  // Sidebar is hidden by default; only auto-expand if the rater explicitly
2714
  // opened it in a prior session (sticky across reloads).
2715
  try {
 
999
  /* =========================================================================
1000
  * State
1001
  * ========================================================================= */
1002
+ let CASES = []; // cases visible to this rater (subset of ALL)
1003
+ let ALL_CASES = []; // full manifest as fetched from cases.json
1004
+ const CASES_PER_RATER = 10; // each rater sees 10 of the full set
1005
  let SHUFFLE = {}; // case_id -> [orig_idx, ...] (slot -> orig)
1006
  let RATINGS = {}; // case_id -> model_name -> { qkey: 1..5 }
1007
  let RATER = { id: "", name: "", email: "", note: "",
 
1069
  /* =========================================================================
1070
  * Shuffling (anonymization)
1071
  * ========================================================================= */
1072
+ // Per-rater 10-case subset of the 20-case manifest. Admins see everything.
1073
+ // Same-name+email always gets the same subset (deterministic Fisher–Yates
1074
+ // seeded by hash(rater_id + ':case-subset')), and the subset is then sorted
1075
+ // back into manifest order so the rater still flips through cases in the
1076
+ // natural cr_* → intake_img_* order.
1077
+ function selectCasesForRater(allCases) {
1078
+ if (isAdmin()) return allCases.slice();
1079
+ if (allCases.length <= CASES_PER_RATER) return allCases.slice();
1080
+ const idx = allCases.map((_, i) => i);
1081
+ let s = hashStr((RATER.id || "anon") + ":case-subset");
1082
+ for (let k = 0; k < 6; k++) s = (s * 1664525 + 1013904223) >>> 0;
1083
+ for (let i = idx.length - 1; i > 0; i--) {
1084
+ s = (s * 1664525 + 1013904223) >>> 0;
1085
+ const j = (s >>> 16) % (i + 1);
1086
+ [idx[i], idx[j]] = [idx[j], idx[i]];
1087
+ }
1088
+ const picked = idx.slice(0, CASES_PER_RATER).sort((a, b) => a - b);
1089
+ return picked.map(i => allCases[i]);
1090
+ }
1091
+
1092
  function buildShuffle(cases) {
1093
  // Deterministic per (rater, case) and ALWAYS recomputed on load. The
1094
  // recomputation is idempotent (same seed → same order), so it costs nothing
 
1623
  const newName = e.target.value.trim();
1624
  if (!newName) { e.target.value = RATER.name; return; }
1625
  RATER.name = newName;
1626
+ // Admin status may have changed → re-pick the visible case subset.
1627
+ CASES = selectCasesForRater(ALL_CASES);
1628
+ SHUFFLE = {}; buildShuffle(CASES);
1629
  saveState();
 
1630
  renderShell();
1631
  },
1632
  });
 
1654
 
1655
  tb.appendChild(el("div", { class: "sep" }));
1656
 
1657
+ // Admin-only reveal toggle + a link to the stats dashboard. Non-admins
1658
+ // always see anonymized view (and anonymize is force-set to true on first
1659
+ // paint below).
1660
  if (isAdmin()) {
1661
  tb.appendChild(el("button", {
1662
  class: "btn ghost",
1663
  onclick: toggleAnon,
1664
  }, anonymize ? t("anonymized") : t("reveal_models")));
1665
+ tb.appendChild(el("a", {
1666
+ class: "btn ghost",
1667
+ href: "./stats.html", target: "_blank", rel: "noopener",
1668
+ style: "text-decoration:none",
1669
+ title: "open the aggregated study stats dashboard",
1670
+ }, "📊 Stats"));
1671
  tb.appendChild(el("div", { class: "sep" }));
1672
  }
1673
 
 
2438
  RATER.id = email.replace(/[^a-z0-9]+/gi, "_").replace(/^_+|_+$/g, "");
2439
  }
2440
  if (!RATER.started_at) RATER.started_at = new Date().toISOString();
2441
+ // Re-pick the per-rater 10-case subset now that we know the email/admin
2442
+ // status, then re-shuffle since the seed depends on the new rater.id.
2443
+ CASES = selectCasesForRater(ALL_CASES);
2444
  SHUFFLE = {};
2445
  buildShuffle(CASES);
2446
  saveState();
 
2699
  renderEmptyApp(t("empty_app_load_error"));
2700
  return;
2701
  }
2702
+ ALL_CASES = found.data || [];
2703
+ if (!Array.isArray(ALL_CASES) || ALL_CASES.length === 0) {
2704
  renderEmptyApp(t("empty_app_empty_manifest"));
2705
  return;
2706
  }
 
2716
  anonymize = prior.anonymize !== false;
2717
  RATINGS = prior.ratings || {};
2718
  SHUFFLE = prior.shuffle || {};
2719
+ // One-shot client-side migration: prior to the huobao→univa rename,
2720
+ // RATINGS were v2-keyed by model name. If localStorage still has
2721
+ // RATINGS[case_id]["huobao"], copy them under "univa" so the rater's
2722
+ // previously-rated cells re-appear under the new column header.
2723
+ let huobaoRenames = 0;
2724
+ for (const cid of Object.keys(RATINGS)) {
2725
+ const byModel = RATINGS[cid];
2726
+ if (byModel && byModel.huobao && !byModel.univa) {
2727
+ byModel.univa = byModel.huobao;
2728
+ delete byModel.huobao;
2729
+ huobaoRenames++;
2730
+ }
2731
+ }
2732
+ if (huobaoRenames) {
2733
+ console.info(`[user-study] renamed huobao→univa ratings on ${huobaoRenames} case(s)`);
2734
+ }
2735
  } else {
2736
  // No v2 state. Try a one-time migration from v1.
2737
  const legacyRaw = localStorage.getItem(STORAGE_KEY_LEGACY);
 
2759
  // Non-admins always see anonymized view — they can't toggle.
2760
  if (!isAdmin()) anonymize = true;
2761
 
2762
+ // Admins see all cases; regular raters see a deterministic random subset
2763
+ // of size CASES_PER_RATER. Seeded by rater id so the same person always
2764
+ // gets the same set across sessions / browsers.
2765
+ CASES = selectCasesForRater(ALL_CASES);
2766
+
2767
  // Sidebar is hidden by default; only auto-expand if the rater explicitly
2768
  // opened it in a prior session (sticky across reloads).
2769
  try {
static/mobile.html CHANGED
@@ -588,7 +588,9 @@ const RUBRIC_I18N = {
588
  };
589
 
590
  /* ----- State (shared with desktop via localStorage) ----- */
591
- let CASES = [];
 
 
592
  let SHUFFLE = {};
593
  let RATINGS = {};
594
  let RATER = { id:"", name:"", email:"", note:"",
@@ -717,6 +719,21 @@ function saveState(){
717
  }
718
 
719
  /* ----- Shuffle ----- */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
720
  function buildShuffle(cases){
721
  for (const c of cases){
722
  if (SHUFFLE[c.id] && SHUFFLE[c.id].length===c.videos.length) continue;
@@ -1259,6 +1276,8 @@ function handleRegister(){
1259
  if (!RATER.started_at) RATER.started_at = new Date().toISOString();
1260
  // Non-admins always anonymized
1261
  if (!isAdmin()) anonymize = true;
 
 
1262
  SHUFFLE = {}; buildShuffle(CASES); saveState();
1263
  closeRegister();
1264
  renderShell();
@@ -1307,7 +1326,7 @@ function hideSubmittedOverlay(){ document.getElementById("submittedOverlay").cla
1307
  '<div style="padding:40px;color:var(--bad);text-align:center">Could not load cases.json</div>';
1308
  return;
1309
  }
1310
- CASES = manifest;
1311
 
1312
  // restore state
1313
  const prior = loadState();
@@ -1318,8 +1337,15 @@ function hideSubmittedOverlay(){ document.getElementById("submittedOverlay").cla
1318
  anonymize = prior.anonymize !== false;
1319
  RATINGS = prior.ratings || {};
1320
  SHUFFLE = prior.shuffle || {};
 
 
 
 
 
1321
  } else { RATER = blankRater(); }
1322
  if (!isAdmin()) anonymize = true;
 
 
1323
  buildShuffle(CASES); saveState();
1324
 
1325
  // wire DOM events
 
588
  };
589
 
590
  /* ----- State (shared with desktop via localStorage) ----- */
591
+ let CASES = []; // cases visible to this rater (subset of ALL)
592
+ let ALL_CASES = []; // full manifest from cases.json
593
+ const CASES_PER_RATER = 10;
594
  let SHUFFLE = {};
595
  let RATINGS = {};
596
  let RATER = { id:"", name:"", email:"", note:"",
 
719
  }
720
 
721
  /* ----- Shuffle ----- */
722
+ // Per-rater 10-case subset of the 20-case manifest. Admins see all 20.
723
+ function selectCasesForRater(allCases){
724
+ if (isAdmin()) return allCases.slice();
725
+ if (allCases.length <= CASES_PER_RATER) return allCases.slice();
726
+ const idx = allCases.map((_,i)=>i);
727
+ let s = hashStr((RATER.id||"anon")+":case-subset");
728
+ for (let k=0;k<6;k++) s = (s*1664525+1013904223)>>>0;
729
+ for (let i=idx.length-1;i>0;i--){
730
+ s = (s*1664525+1013904223)>>>0;
731
+ const j = (s>>>16)%(i+1);
732
+ [idx[i],idx[j]] = [idx[j],idx[i]];
733
+ }
734
+ return idx.slice(0,CASES_PER_RATER).sort((a,b)=>a-b).map(i => allCases[i]);
735
+ }
736
+
737
  function buildShuffle(cases){
738
  for (const c of cases){
739
  if (SHUFFLE[c.id] && SHUFFLE[c.id].length===c.videos.length) continue;
 
1276
  if (!RATER.started_at) RATER.started_at = new Date().toISOString();
1277
  // Non-admins always anonymized
1278
  if (!isAdmin()) anonymize = true;
1279
+ // Re-pick the per-rater subset now that we know admin status.
1280
+ CASES = selectCasesForRater(ALL_CASES);
1281
  SHUFFLE = {}; buildShuffle(CASES); saveState();
1282
  closeRegister();
1283
  renderShell();
 
1326
  '<div style="padding:40px;color:var(--bad);text-align:center">Could not load cases.json</div>';
1327
  return;
1328
  }
1329
+ ALL_CASES = manifest;
1330
 
1331
  // restore state
1332
  const prior = loadState();
 
1337
  anonymize = prior.anonymize !== false;
1338
  RATINGS = prior.ratings || {};
1339
  SHUFFLE = prior.shuffle || {};
1340
+ // One-shot client-side huobao→univa rename (see desktop for context).
1341
+ for (const cid of Object.keys(RATINGS)) {
1342
+ const m = RATINGS[cid];
1343
+ if (m && m.huobao && !m.univa) { m.univa = m.huobao; delete m.huobao; }
1344
+ }
1345
  } else { RATER = blankRater(); }
1346
  if (!isAdmin()) anonymize = true;
1347
+ // Admins see all cases; everyone else gets a deterministic 10-case subset.
1348
+ CASES = selectCasesForRater(ALL_CASES);
1349
  buildShuffle(CASES); saveState();
1350
 
1351
  // wire DOM events