dehezhang2 commited on
Commit
809237c
·
verified ·
1 Parent(s): b3f8fa7

deploy rater (Docker/FastAPI): registration + completeness gate + /api/submit → dataset

Browse files
Files changed (1) hide show
  1. static/index.html +126 -35
static/index.html CHANGED
@@ -417,7 +417,12 @@ const DIMENSIONS = [
417
  },
418
  ];
419
 
420
- const STORAGE_KEY = "user_study_video_ratings_v1";
 
 
 
 
 
421
  const TOTAL_QUESTIONS = DIMENSIONS.reduce((n, d) => n + d.questions.length, 0);
422
 
423
  /* =========================================================================
@@ -926,7 +931,7 @@ function scaleLabel(score) {
926
  * ========================================================================= */
927
  let CASES = []; // raw manifest
928
  let SHUFFLE = {}; // case_id -> [orig_idx, ...] (slot -> orig)
929
- let RATINGS = {}; // case_id -> slot_idx -> { qkey: 1..5 }
930
  let RATER = { id: "", name: "", email: "", note: "",
931
  started_at: null, submitted_at: null };
932
  let currentIdx = 0;
@@ -1023,15 +1028,34 @@ function getCaseRatings(caseId) {
1023
  return RATINGS[caseId];
1024
  }
1025
 
1026
- function getSlotRatings(caseId, slotIdx) {
 
 
 
 
 
 
 
 
 
 
1027
  const m = getCaseRatings(caseId);
1028
- if (!m[slotIdx]) m[slotIdx] = {};
1029
- return m[slotIdx];
 
 
 
 
 
 
 
 
1030
  }
1031
 
1032
  function setRating(caseId, slotIdx, qkey, score) {
1033
- const r = getSlotRatings(caseId, slotIdx);
1034
- r[qkey] = score;
 
1035
  saveState();
1036
  updateProgress();
1037
  updateCaseStatus(caseId);
@@ -1065,14 +1089,17 @@ function buildFullSubmissionPayload() {
1065
  id: c.id,
1066
  category: c.category || null,
1067
  prompt: c.prompt,
1068
- videos: slots.map((origIdx, slotIdx) => ({
1069
- slot: String.fromCharCode(65 + slotIdx),
1070
- slot_index: slotIdx,
1071
- original_index: origIdx,
1072
- model: c.videos[origIdx].model,
1073
- path: c.videos[origIdx].path,
1074
- ratings: (RATINGS[c.id] || {})[slotIdx] || {},
1075
- })),
 
 
 
1076
  };
1077
  }),
1078
  };
@@ -1086,13 +1113,15 @@ function studyCompleteness() {
1086
  for (const c of CASES) {
1087
  const slots = SHUFFLE[c.id] || c.videos.map((_, i) => i);
1088
  for (let slotIdx = 0; slotIdx < slots.length; slotIdx++) {
1089
- const rs = (RATINGS[c.id] || {})[slotIdx] || {};
 
 
1090
  const slotLetter = String.fromCharCode(65 + slotIdx);
1091
  for (const qk of qkeys) {
1092
  expected++;
1093
  const v = rs[qk];
1094
  if (typeof v === "number" && v >= 1 && v <= 5) done++;
1095
- else missing.push({ case_id: c.id, slot: slotLetter, q: qk });
1096
  }
1097
  }
1098
  }
@@ -1176,14 +1205,17 @@ function buildProgressPayload() {
1176
  id: c.id,
1177
  category: c.category || null,
1178
  prompt: c.prompt,
1179
- videos: slots.map((origIdx, slotIdx) => ({
1180
- slot: String.fromCharCode(65 + slotIdx),
1181
- slot_index: slotIdx,
1182
- original_index: origIdx,
1183
- model: c.videos[origIdx].model,
1184
- path: c.videos[origIdx].path,
1185
- ratings: (RATINGS[c.id] || {})[slotIdx] || {},
1186
- })),
 
 
 
1187
  };
1188
  }),
1189
  };
@@ -1280,7 +1312,9 @@ function caseProgress(caseCase) {
1280
  const expected = slots.length * TOTAL_QUESTIONS;
1281
  let done = 0;
1282
  for (let s = 0; s < slots.length; s++) {
1283
- const r = (RATINGS[caseCase.id] || {})[s] || {};
 
 
1284
  done += Object.keys(r).length;
1285
  }
1286
  return { done, expected };
@@ -2078,14 +2112,17 @@ function exportJSON() {
2078
  return {
2079
  id: c.id,
2080
  prompt: c.prompt,
2081
- videos: slots.map((origIdx, slotIdx) => ({
2082
- slot: String.fromCharCode(65 + slotIdx),
2083
- slot_index: slotIdx,
2084
- original_index: origIdx,
2085
- model: c.videos[origIdx].model,
2086
- path: c.videos[origIdx].path,
2087
- ratings: (RATINGS[c.id] || {})[slotIdx] || {},
2088
- })),
 
 
 
2089
  };
2090
  }),
2091
  };
@@ -2182,6 +2219,40 @@ document.getElementById("regGoBtn")?.addEventListener("click", handleRegistratio
2182
  });
2183
  });
2184
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2185
  (async function init() {
2186
  // Restore LANG first so the empty-app + registration overlay show the right
2187
  // language even before the rest of the state is wired up.
@@ -2221,7 +2292,27 @@ document.getElementById("regGoBtn")?.addEventListener("click", handleRegistratio
2221
  RATINGS = prior.ratings || {};
2222
  SHUFFLE = prior.shuffle || {};
2223
  } else {
2224
- RATER = blankRater();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2225
  }
2226
 
2227
  // Non-admins always see anonymized view — they can't toggle.
 
417
  },
418
  ];
419
 
420
+ // v2 keys RATINGS by model name (per case) instead of by slot index, so
421
+ // adding/removing videos in a case doesn't misalign previously-rated cells.
422
+ // Old v1 storage is migrated on first load (using the v1 shuffle it saved
423
+ // alongside, to map slot_idx → original_index → model name).
424
+ const STORAGE_KEY = "user_study_video_ratings_v2";
425
+ const STORAGE_KEY_LEGACY = "user_study_video_ratings_v1";
426
  const TOTAL_QUESTIONS = DIMENSIONS.reduce((n, d) => n + d.questions.length, 0);
427
 
428
  /* =========================================================================
 
931
  * ========================================================================= */
932
  let CASES = []; // raw manifest
933
  let SHUFFLE = {}; // case_id -> [orig_idx, ...] (slot -> orig)
934
+ let RATINGS = {}; // case_id -> model_name -> { qkey: 1..5 }
935
  let RATER = { id: "", name: "", email: "", note: "",
936
  started_at: null, submitted_at: null };
937
  let currentIdx = 0;
 
1028
  return RATINGS[caseId];
1029
  }
1030
 
1031
+ // Translate (caseId, slotIdx) → the model name in that slot under the
1032
+ // current shuffle. Returns null if the case / slot isn't known yet.
1033
+ function modelForSlot(caseId, slotIdx) {
1034
+ const c = CASES.find(x => x.id === caseId);
1035
+ const slots = SHUFFLE[caseId];
1036
+ if (!c || !slots) return null;
1037
+ const origIdx = slots[slotIdx];
1038
+ return c.videos[origIdx]?.model ?? null;
1039
+ }
1040
+
1041
+ function getModelRatings(caseId, model) {
1042
  const m = getCaseRatings(caseId);
1043
+ if (!m[model]) m[model] = {};
1044
+ return m[model];
1045
+ }
1046
+
1047
+ // Back-compat wrapper for the per-slot path in render code; resolves slot →
1048
+ // model and returns the matching ratings dict (auto-creates if absent).
1049
+ function getSlotRatings(caseId, slotIdx) {
1050
+ const model = modelForSlot(caseId, slotIdx);
1051
+ if (!model) return {};
1052
+ return getModelRatings(caseId, model);
1053
  }
1054
 
1055
  function setRating(caseId, slotIdx, qkey, score) {
1056
+ const model = modelForSlot(caseId, slotIdx);
1057
+ if (!model) return;
1058
+ getModelRatings(caseId, model)[qkey] = score;
1059
  saveState();
1060
  updateProgress();
1061
  updateCaseStatus(caseId);
 
1089
  id: c.id,
1090
  category: c.category || null,
1091
  prompt: c.prompt,
1092
+ videos: slots.map((origIdx, slotIdx) => {
1093
+ const v = c.videos[origIdx];
1094
+ return {
1095
+ slot: String.fromCharCode(65 + slotIdx),
1096
+ slot_index: slotIdx,
1097
+ original_index: origIdx,
1098
+ model: v.model,
1099
+ path: v.path,
1100
+ ratings: (RATINGS[c.id] || {})[v.model] || {},
1101
+ };
1102
+ }),
1103
  };
1104
  }),
1105
  };
 
1113
  for (const c of CASES) {
1114
  const slots = SHUFFLE[c.id] || c.videos.map((_, i) => i);
1115
  for (let slotIdx = 0; slotIdx < slots.length; slotIdx++) {
1116
+ const origIdx = slots[slotIdx];
1117
+ const model = c.videos[origIdx].model;
1118
+ const rs = (RATINGS[c.id] || {})[model] || {};
1119
  const slotLetter = String.fromCharCode(65 + slotIdx);
1120
  for (const qk of qkeys) {
1121
  expected++;
1122
  const v = rs[qk];
1123
  if (typeof v === "number" && v >= 1 && v <= 5) done++;
1124
+ else missing.push({ case_id: c.id, slot: slotLetter, q: qk, model });
1125
  }
1126
  }
1127
  }
 
1205
  id: c.id,
1206
  category: c.category || null,
1207
  prompt: c.prompt,
1208
+ videos: slots.map((origIdx, slotIdx) => {
1209
+ const v = c.videos[origIdx];
1210
+ return {
1211
+ slot: String.fromCharCode(65 + slotIdx),
1212
+ slot_index: slotIdx,
1213
+ original_index: origIdx,
1214
+ model: v.model,
1215
+ path: v.path,
1216
+ ratings: (RATINGS[c.id] || {})[v.model] || {},
1217
+ };
1218
+ }),
1219
  };
1220
  }),
1221
  };
 
1312
  const expected = slots.length * TOTAL_QUESTIONS;
1313
  let done = 0;
1314
  for (let s = 0; s < slots.length; s++) {
1315
+ const origIdx = slots[s];
1316
+ const model = caseCase.videos[origIdx]?.model;
1317
+ const r = (RATINGS[caseCase.id] || {})[model] || {};
1318
  done += Object.keys(r).length;
1319
  }
1320
  return { done, expected };
 
2112
  return {
2113
  id: c.id,
2114
  prompt: c.prompt,
2115
+ videos: slots.map((origIdx, slotIdx) => {
2116
+ const v = c.videos[origIdx];
2117
+ return {
2118
+ slot: String.fromCharCode(65 + slotIdx),
2119
+ slot_index: slotIdx,
2120
+ original_index: origIdx,
2121
+ model: v.model,
2122
+ path: v.path,
2123
+ ratings: (RATINGS[c.id] || {})[v.model] || {},
2124
+ };
2125
+ }),
2126
  };
2127
  }),
2128
  };
 
2219
  });
2220
  });
2221
 
2222
+ // One-time migration from v1 (RATINGS keyed by slot_idx) to v2 (keyed by
2223
+ // model name). Uses the v1 saved SHUFFLE to map slot → original_index →
2224
+ // model. Cases whose v1 SHUFFLE is missing or stale are skipped; the
2225
+ // rater would then see those cases as un-rated, but no existing rating
2226
+ // is misattributed to a different video.
2227
+ function migrateLegacyRatingsV1(legacy, allCases) {
2228
+ const oldRatings = legacy?.ratings || {};
2229
+ const oldShuffle = legacy?.shuffle || {};
2230
+ const out = {};
2231
+ let migrated = 0, skipped = 0;
2232
+ for (const [caseId, slotMap] of Object.entries(oldRatings)) {
2233
+ const caseObj = allCases.find(c => c.id === caseId);
2234
+ const shuffle = oldShuffle[caseId];
2235
+ if (!caseObj || !Array.isArray(shuffle)) {
2236
+ skipped++;
2237
+ continue;
2238
+ }
2239
+ out[caseId] = {};
2240
+ for (const [slotStr, qMap] of Object.entries(slotMap || {})) {
2241
+ const slotIdx = parseInt(slotStr, 10);
2242
+ if (!Number.isFinite(slotIdx)) continue;
2243
+ const origIdx = shuffle[slotIdx];
2244
+ const video = caseObj.videos[origIdx];
2245
+ if (!video || !video.model) continue;
2246
+ // Merge in case the same model already has entries (shouldn't happen
2247
+ // for the v1 format but defensive).
2248
+ out[caseId][video.model] = Object.assign(
2249
+ out[caseId][video.model] || {}, qMap);
2250
+ migrated++;
2251
+ }
2252
+ }
2253
+ return { ratings: out, migrated, skipped };
2254
+ }
2255
+
2256
  (async function init() {
2257
  // Restore LANG first so the empty-app + registration overlay show the right
2258
  // language even before the rest of the state is wired up.
 
2292
  RATINGS = prior.ratings || {};
2293
  SHUFFLE = prior.shuffle || {};
2294
  } else {
2295
+ // No v2 state. Try a one-time migration from v1.
2296
+ const legacyRaw = localStorage.getItem(STORAGE_KEY_LEGACY);
2297
+ if (legacyRaw) {
2298
+ try {
2299
+ const legacy = JSON.parse(legacyRaw);
2300
+ RATER = Object.assign(blankRater(), legacy.rater || {});
2301
+ anonymize = legacy.anonymize !== false;
2302
+ const { ratings, migrated, skipped } = migrateLegacyRatingsV1(legacy, CASES);
2303
+ RATINGS = ratings;
2304
+ // SHUFFLE is recomputed every load anyway; don't carry the v1 order.
2305
+ SHUFFLE = {};
2306
+ console.info(
2307
+ `[user-study] migrated v1 → v2: ${migrated} ratings carried over, ${skipped} cases skipped`);
2308
+ localStorage.removeItem(STORAGE_KEY_LEGACY);
2309
+ } catch (e) {
2310
+ console.warn("[user-study] v1 migration failed:", e);
2311
+ RATER = blankRater();
2312
+ }
2313
+ } else {
2314
+ RATER = blankRater();
2315
+ }
2316
  }
2317
 
2318
  // Non-admins always see anonymized view — they can't toggle.