Spaces:
Sleeping
Sleeping
deploy rater (Docker/FastAPI): registration + completeness gate + /api/submit → dataset
Browse files- static/index.html +61 -7
- static/mobile.html +28 -2
static/index.html
CHANGED
|
@@ -999,7 +999,9 @@ function scaleLabel(score) {
|
|
| 999 |
/* =========================================================================
|
| 1000 |
* State
|
| 1001 |
* ========================================================================= */
|
| 1002 |
-
let CASES = []; //
|
|
|
|
|
|
|
| 1003 |
let SHUFFLE = {}; // case_id -> [orig_idx, ...] (slot -> orig)
|
| 1004 |
let RATINGS = {}; // case_id -> model_name -> { qkey: 1..5 }
|
| 1005 |
let RATER = { id: "", name: "", email: "", note: "",
|
|
@@ -1067,6 +1069,26 @@ function genRaterId() {
|
|
| 1067 |
/* =========================================================================
|
| 1068 |
* Shuffling (anonymization)
|
| 1069 |
* ========================================================================= */
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1070 |
function buildShuffle(cases) {
|
| 1071 |
// Deterministic per (rater, case) and ALWAYS recomputed on load. The
|
| 1072 |
// recomputation is idempotent (same seed → same order), so it costs nothing
|
|
@@ -1601,8 +1623,10 @@ function renderToolbar() {
|
|
| 1601 |
const newName = e.target.value.trim();
|
| 1602 |
if (!newName) { e.target.value = RATER.name; return; }
|
| 1603 |
RATER.name = newName;
|
|
|
|
|
|
|
|
|
|
| 1604 |
saveState();
|
| 1605 |
-
// re-render so the admin-only toggle visibility refreshes on name change
|
| 1606 |
renderShell();
|
| 1607 |
},
|
| 1608 |
});
|
|
@@ -1630,13 +1654,20 @@ function renderToolbar() {
|
|
| 1630 |
|
| 1631 |
tb.appendChild(el("div", { class: "sep" }));
|
| 1632 |
|
| 1633 |
-
// Admin-only reveal toggle
|
| 1634 |
-
// anonymize is force-set to true on first
|
|
|
|
| 1635 |
if (isAdmin()) {
|
| 1636 |
tb.appendChild(el("button", {
|
| 1637 |
class: "btn ghost",
|
| 1638 |
onclick: toggleAnon,
|
| 1639 |
}, anonymize ? t("anonymized") : t("reveal_models")));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1640 |
tb.appendChild(el("div", { class: "sep" }));
|
| 1641 |
}
|
| 1642 |
|
|
@@ -2407,7 +2438,9 @@ function handleRegistrationSubmit() {
|
|
| 2407 |
RATER.id = email.replace(/[^a-z0-9]+/gi, "_").replace(/^_+|_+$/g, "");
|
| 2408 |
}
|
| 2409 |
if (!RATER.started_at) RATER.started_at = new Date().toISOString();
|
| 2410 |
-
// Re-
|
|
|
|
|
|
|
| 2411 |
SHUFFLE = {};
|
| 2412 |
buildShuffle(CASES);
|
| 2413 |
saveState();
|
|
@@ -2666,8 +2699,8 @@ function migrateLegacyRatingsV1(legacy, allCases) {
|
|
| 2666 |
renderEmptyApp(t("empty_app_load_error"));
|
| 2667 |
return;
|
| 2668 |
}
|
| 2669 |
-
|
| 2670 |
-
if (!Array.isArray(
|
| 2671 |
renderEmptyApp(t("empty_app_empty_manifest"));
|
| 2672 |
return;
|
| 2673 |
}
|
|
@@ -2683,6 +2716,22 @@ function migrateLegacyRatingsV1(legacy, allCases) {
|
|
| 2683 |
anonymize = prior.anonymize !== false;
|
| 2684 |
RATINGS = prior.ratings || {};
|
| 2685 |
SHUFFLE = prior.shuffle || {};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2686 |
} else {
|
| 2687 |
// No v2 state. Try a one-time migration from v1.
|
| 2688 |
const legacyRaw = localStorage.getItem(STORAGE_KEY_LEGACY);
|
|
@@ -2710,6 +2759,11 @@ function migrateLegacyRatingsV1(legacy, allCases) {
|
|
| 2710 |
// Non-admins always see anonymized view — they can't toggle.
|
| 2711 |
if (!isAdmin()) anonymize = true;
|
| 2712 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2713 |
// Sidebar is hidden by default; only auto-expand if the rater explicitly
|
| 2714 |
// opened it in a prior session (sticky across reloads).
|
| 2715 |
try {
|
|
|
|
| 999 |
/* =========================================================================
|
| 1000 |
* State
|
| 1001 |
* ========================================================================= */
|
| 1002 |
+
let CASES = []; // cases visible to this rater (subset of ALL)
|
| 1003 |
+
let ALL_CASES = []; // full manifest as fetched from cases.json
|
| 1004 |
+
const CASES_PER_RATER = 10; // each rater sees 10 of the full set
|
| 1005 |
let SHUFFLE = {}; // case_id -> [orig_idx, ...] (slot -> orig)
|
| 1006 |
let RATINGS = {}; // case_id -> model_name -> { qkey: 1..5 }
|
| 1007 |
let RATER = { id: "", name: "", email: "", note: "",
|
|
|
|
| 1069 |
/* =========================================================================
|
| 1070 |
* Shuffling (anonymization)
|
| 1071 |
* ========================================================================= */
|
| 1072 |
+
// Per-rater 10-case subset of the 20-case manifest. Admins see everything.
|
| 1073 |
+
// Same-name+email always gets the same subset (deterministic Fisher–Yates
|
| 1074 |
+
// seeded by hash(rater_id + ':case-subset')), and the subset is then sorted
|
| 1075 |
+
// back into manifest order so the rater still flips through cases in the
|
| 1076 |
+
// natural cr_* → intake_img_* order.
|
| 1077 |
+
function selectCasesForRater(allCases) {
|
| 1078 |
+
if (isAdmin()) return allCases.slice();
|
| 1079 |
+
if (allCases.length <= CASES_PER_RATER) return allCases.slice();
|
| 1080 |
+
const idx = allCases.map((_, i) => i);
|
| 1081 |
+
let s = hashStr((RATER.id || "anon") + ":case-subset");
|
| 1082 |
+
for (let k = 0; k < 6; k++) s = (s * 1664525 + 1013904223) >>> 0;
|
| 1083 |
+
for (let i = idx.length - 1; i > 0; i--) {
|
| 1084 |
+
s = (s * 1664525 + 1013904223) >>> 0;
|
| 1085 |
+
const j = (s >>> 16) % (i + 1);
|
| 1086 |
+
[idx[i], idx[j]] = [idx[j], idx[i]];
|
| 1087 |
+
}
|
| 1088 |
+
const picked = idx.slice(0, CASES_PER_RATER).sort((a, b) => a - b);
|
| 1089 |
+
return picked.map(i => allCases[i]);
|
| 1090 |
+
}
|
| 1091 |
+
|
| 1092 |
function buildShuffle(cases) {
|
| 1093 |
// Deterministic per (rater, case) and ALWAYS recomputed on load. The
|
| 1094 |
// recomputation is idempotent (same seed → same order), so it costs nothing
|
|
|
|
| 1623 |
const newName = e.target.value.trim();
|
| 1624 |
if (!newName) { e.target.value = RATER.name; return; }
|
| 1625 |
RATER.name = newName;
|
| 1626 |
+
// Admin status may have changed → re-pick the visible case subset.
|
| 1627 |
+
CASES = selectCasesForRater(ALL_CASES);
|
| 1628 |
+
SHUFFLE = {}; buildShuffle(CASES);
|
| 1629 |
saveState();
|
|
|
|
| 1630 |
renderShell();
|
| 1631 |
},
|
| 1632 |
});
|
|
|
|
| 1654 |
|
| 1655 |
tb.appendChild(el("div", { class: "sep" }));
|
| 1656 |
|
| 1657 |
+
// Admin-only reveal toggle + a link to the stats dashboard. Non-admins
|
| 1658 |
+
// always see anonymized view (and anonymize is force-set to true on first
|
| 1659 |
+
// paint below).
|
| 1660 |
if (isAdmin()) {
|
| 1661 |
tb.appendChild(el("button", {
|
| 1662 |
class: "btn ghost",
|
| 1663 |
onclick: toggleAnon,
|
| 1664 |
}, anonymize ? t("anonymized") : t("reveal_models")));
|
| 1665 |
+
tb.appendChild(el("a", {
|
| 1666 |
+
class: "btn ghost",
|
| 1667 |
+
href: "./stats.html", target: "_blank", rel: "noopener",
|
| 1668 |
+
style: "text-decoration:none",
|
| 1669 |
+
title: "open the aggregated study stats dashboard",
|
| 1670 |
+
}, "📊 Stats"));
|
| 1671 |
tb.appendChild(el("div", { class: "sep" }));
|
| 1672 |
}
|
| 1673 |
|
|
|
|
| 2438 |
RATER.id = email.replace(/[^a-z0-9]+/gi, "_").replace(/^_+|_+$/g, "");
|
| 2439 |
}
|
| 2440 |
if (!RATER.started_at) RATER.started_at = new Date().toISOString();
|
| 2441 |
+
// Re-pick the per-rater 10-case subset now that we know the email/admin
|
| 2442 |
+
// status, then re-shuffle since the seed depends on the new rater.id.
|
| 2443 |
+
CASES = selectCasesForRater(ALL_CASES);
|
| 2444 |
SHUFFLE = {};
|
| 2445 |
buildShuffle(CASES);
|
| 2446 |
saveState();
|
|
|
|
| 2699 |
renderEmptyApp(t("empty_app_load_error"));
|
| 2700 |
return;
|
| 2701 |
}
|
| 2702 |
+
ALL_CASES = found.data || [];
|
| 2703 |
+
if (!Array.isArray(ALL_CASES) || ALL_CASES.length === 0) {
|
| 2704 |
renderEmptyApp(t("empty_app_empty_manifest"));
|
| 2705 |
return;
|
| 2706 |
}
|
|
|
|
| 2716 |
anonymize = prior.anonymize !== false;
|
| 2717 |
RATINGS = prior.ratings || {};
|
| 2718 |
SHUFFLE = prior.shuffle || {};
|
| 2719 |
+
// One-shot client-side migration: prior to the huobao→univa rename,
|
| 2720 |
+
// RATINGS were v2-keyed by model name. If localStorage still has
|
| 2721 |
+
// RATINGS[case_id]["huobao"], copy them under "univa" so the rater's
|
| 2722 |
+
// previously-rated cells re-appear under the new column header.
|
| 2723 |
+
let huobaoRenames = 0;
|
| 2724 |
+
for (const cid of Object.keys(RATINGS)) {
|
| 2725 |
+
const byModel = RATINGS[cid];
|
| 2726 |
+
if (byModel && byModel.huobao && !byModel.univa) {
|
| 2727 |
+
byModel.univa = byModel.huobao;
|
| 2728 |
+
delete byModel.huobao;
|
| 2729 |
+
huobaoRenames++;
|
| 2730 |
+
}
|
| 2731 |
+
}
|
| 2732 |
+
if (huobaoRenames) {
|
| 2733 |
+
console.info(`[user-study] renamed huobao→univa ratings on ${huobaoRenames} case(s)`);
|
| 2734 |
+
}
|
| 2735 |
} else {
|
| 2736 |
// No v2 state. Try a one-time migration from v1.
|
| 2737 |
const legacyRaw = localStorage.getItem(STORAGE_KEY_LEGACY);
|
|
|
|
| 2759 |
// Non-admins always see anonymized view — they can't toggle.
|
| 2760 |
if (!isAdmin()) anonymize = true;
|
| 2761 |
|
| 2762 |
+
// Admins see all cases; regular raters see a deterministic random subset
|
| 2763 |
+
// of size CASES_PER_RATER. Seeded by rater id so the same person always
|
| 2764 |
+
// gets the same set across sessions / browsers.
|
| 2765 |
+
CASES = selectCasesForRater(ALL_CASES);
|
| 2766 |
+
|
| 2767 |
// Sidebar is hidden by default; only auto-expand if the rater explicitly
|
| 2768 |
// opened it in a prior session (sticky across reloads).
|
| 2769 |
try {
|
static/mobile.html
CHANGED
|
@@ -588,7 +588,9 @@ const RUBRIC_I18N = {
|
|
| 588 |
};
|
| 589 |
|
| 590 |
/* ----- State (shared with desktop via localStorage) ----- */
|
| 591 |
-
let CASES = [];
|
|
|
|
|
|
|
| 592 |
let SHUFFLE = {};
|
| 593 |
let RATINGS = {};
|
| 594 |
let RATER = { id:"", name:"", email:"", note:"",
|
|
@@ -717,6 +719,21 @@ function saveState(){
|
|
| 717 |
}
|
| 718 |
|
| 719 |
/* ----- Shuffle ----- */
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 720 |
function buildShuffle(cases){
|
| 721 |
for (const c of cases){
|
| 722 |
if (SHUFFLE[c.id] && SHUFFLE[c.id].length===c.videos.length) continue;
|
|
@@ -1259,6 +1276,8 @@ function handleRegister(){
|
|
| 1259 |
if (!RATER.started_at) RATER.started_at = new Date().toISOString();
|
| 1260 |
// Non-admins always anonymized
|
| 1261 |
if (!isAdmin()) anonymize = true;
|
|
|
|
|
|
|
| 1262 |
SHUFFLE = {}; buildShuffle(CASES); saveState();
|
| 1263 |
closeRegister();
|
| 1264 |
renderShell();
|
|
@@ -1307,7 +1326,7 @@ function hideSubmittedOverlay(){ document.getElementById("submittedOverlay").cla
|
|
| 1307 |
'<div style="padding:40px;color:var(--bad);text-align:center">Could not load cases.json</div>';
|
| 1308 |
return;
|
| 1309 |
}
|
| 1310 |
-
|
| 1311 |
|
| 1312 |
// restore state
|
| 1313 |
const prior = loadState();
|
|
@@ -1318,8 +1337,15 @@ function hideSubmittedOverlay(){ document.getElementById("submittedOverlay").cla
|
|
| 1318 |
anonymize = prior.anonymize !== false;
|
| 1319 |
RATINGS = prior.ratings || {};
|
| 1320 |
SHUFFLE = prior.shuffle || {};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1321 |
} else { RATER = blankRater(); }
|
| 1322 |
if (!isAdmin()) anonymize = true;
|
|
|
|
|
|
|
| 1323 |
buildShuffle(CASES); saveState();
|
| 1324 |
|
| 1325 |
// wire DOM events
|
|
|
|
| 588 |
};
|
| 589 |
|
| 590 |
/* ----- State (shared with desktop via localStorage) ----- */
|
| 591 |
+
let CASES = []; // cases visible to this rater (subset of ALL)
|
| 592 |
+
let ALL_CASES = []; // full manifest from cases.json
|
| 593 |
+
const CASES_PER_RATER = 10;
|
| 594 |
let SHUFFLE = {};
|
| 595 |
let RATINGS = {};
|
| 596 |
let RATER = { id:"", name:"", email:"", note:"",
|
|
|
|
| 719 |
}
|
| 720 |
|
| 721 |
/* ----- Shuffle ----- */
|
| 722 |
+
// Per-rater 10-case subset of the 20-case manifest. Admins see all 20.
|
| 723 |
+
function selectCasesForRater(allCases){
|
| 724 |
+
if (isAdmin()) return allCases.slice();
|
| 725 |
+
if (allCases.length <= CASES_PER_RATER) return allCases.slice();
|
| 726 |
+
const idx = allCases.map((_,i)=>i);
|
| 727 |
+
let s = hashStr((RATER.id||"anon")+":case-subset");
|
| 728 |
+
for (let k=0;k<6;k++) s = (s*1664525+1013904223)>>>0;
|
| 729 |
+
for (let i=idx.length-1;i>0;i--){
|
| 730 |
+
s = (s*1664525+1013904223)>>>0;
|
| 731 |
+
const j = (s>>>16)%(i+1);
|
| 732 |
+
[idx[i],idx[j]] = [idx[j],idx[i]];
|
| 733 |
+
}
|
| 734 |
+
return idx.slice(0,CASES_PER_RATER).sort((a,b)=>a-b).map(i => allCases[i]);
|
| 735 |
+
}
|
| 736 |
+
|
| 737 |
function buildShuffle(cases){
|
| 738 |
for (const c of cases){
|
| 739 |
if (SHUFFLE[c.id] && SHUFFLE[c.id].length===c.videos.length) continue;
|
|
|
|
| 1276 |
if (!RATER.started_at) RATER.started_at = new Date().toISOString();
|
| 1277 |
// Non-admins always anonymized
|
| 1278 |
if (!isAdmin()) anonymize = true;
|
| 1279 |
+
// Re-pick the per-rater subset now that we know admin status.
|
| 1280 |
+
CASES = selectCasesForRater(ALL_CASES);
|
| 1281 |
SHUFFLE = {}; buildShuffle(CASES); saveState();
|
| 1282 |
closeRegister();
|
| 1283 |
renderShell();
|
|
|
|
| 1326 |
'<div style="padding:40px;color:var(--bad);text-align:center">Could not load cases.json</div>';
|
| 1327 |
return;
|
| 1328 |
}
|
| 1329 |
+
ALL_CASES = manifest;
|
| 1330 |
|
| 1331 |
// restore state
|
| 1332 |
const prior = loadState();
|
|
|
|
| 1337 |
anonymize = prior.anonymize !== false;
|
| 1338 |
RATINGS = prior.ratings || {};
|
| 1339 |
SHUFFLE = prior.shuffle || {};
|
| 1340 |
+
// One-shot client-side huobao→univa rename (see desktop for context).
|
| 1341 |
+
for (const cid of Object.keys(RATINGS)) {
|
| 1342 |
+
const m = RATINGS[cid];
|
| 1343 |
+
if (m && m.huobao && !m.univa) { m.univa = m.huobao; delete m.huobao; }
|
| 1344 |
+
}
|
| 1345 |
} else { RATER = blankRater(); }
|
| 1346 |
if (!isAdmin()) anonymize = true;
|
| 1347 |
+
// Admins see all cases; everyone else gets a deterministic 10-case subset.
|
| 1348 |
+
CASES = selectCasesForRater(ALL_CASES);
|
| 1349 |
buildShuffle(CASES); saveState();
|
| 1350 |
|
| 1351 |
// wire DOM events
|