Download backend/services/duplicateService.js from essay003/prmsu: direct link, hf CLI and curl.
- Browser
- Download file 5.47 kB
-
https://huggingface.co/spaces/essay003/prmsu/resolve/main/backend/services/duplicateService.js
- Command line
-
hf download hf://spaces/essay003/prmsu/backend/services/duplicateService.js
-
curl -L -o duplicateService.js https://huggingface.co/spaces/essay003/prmsu/resolve/main/backend/services/duplicateService.js
5.47 kB
| import crypto from "node:crypto"; | |
| import { db } from "../config/firebase.js"; | |
| const EVALUATIONS_COLLECTION = "evaluations"; | |
| /** | |
| * Identical / near-identical answer detection. | |
| * | |
| * Two layers, because students who copy rarely paste verbatim: | |
| * | |
| * 1. Exact match - SHA-256 of the normalised text. Instant, zero false positives. | |
| * 2. Near match - Jaccard similarity over 5-word shingles. Catches the usual edits | |
| * (swapped words, reordered sentences, changed intro) while staying | |
| * cheap enough to run against recent submissions on every grade. | |
| * | |
| * Shingling is used rather than plain word overlap because word overlap alone flags any | |
| * two essays on the same prompt - they share topic vocabulary by definition. Requiring | |
| * shared 5-word runs means the wording itself has to match, not just the subject. | |
| */ | |
| const SHINGLE = 5; | |
| const NEAR_THRESHOLD = 0.35; // share of 5-word runs in common | |
| const COMPARE_LIMIT = 300; // most recent submissions to compare against | |
| export const normalizeAnswer = (text) => | |
| String(text || "") | |
| .toLowerCase() | |
| .replace(/[^a-z0-9\s]/g, " ") | |
| .replace(/\s+/g, " ") | |
| .trim(); | |
| export const answerHash = (text) => | |
| crypto.createHash("sha256").update(normalizeAnswer(text)).digest("hex"); | |
| /** | |
| * What a viewer may see of a duplicate result. Students learn that a match exists, never | |
| * whose work it matched: other students' names and record ids stay with teachers. The | |
| * full result is always what gets saved. | |
| */ | |
| export const duplicateForViewer = (duplicateCheck, isTeacher) => { | |
| if (isTeacher || !duplicateCheck?.matches) return duplicateCheck; | |
| return { | |
| ...duplicateCheck, | |
| matches: duplicateCheck.matches.map((m) => ({ | |
| subject: m.subject || null, | |
| createdAt: m.createdAt || null, | |
| })), | |
| }; | |
| }; | |
| const shingles = (text) => { | |
| const words = normalizeAnswer(text).split(" ").filter(Boolean); | |
| const set = new Set(); | |
| for (let i = 0; i + SHINGLE <= words.length; i += 1) { | |
| set.add(words.slice(i, i + SHINGLE).join(" ")); | |
| } | |
| return set; | |
| }; | |
| const jaccard = (a, b) => { | |
| if (a.size === 0 || b.size === 0) return 0; | |
| let shared = 0; | |
| const [small, large] = a.size <= b.size ? [a, b] : [b, a]; | |
| for (const s of small) if (large.has(s)) shared += 1; | |
| return shared / (a.size + b.size - shared); | |
| }; | |
| /** | |
| * Compare one answer against previously stored evaluations. | |
| * | |
| * Scoped to the same subject when one is given, so an English essay is not compared | |
| * against a Science answer. Never compares a student against their own past work - | |
| * resubmitting your own essay is not plagiarism. | |
| */ | |
| export const checkDuplicate = async ({ answer, subject = null, excludeUid = null }) => { | |
| const text = String(answer || ""); | |
| if (normalizeAnswer(text).split(" ").length < 20) { | |
| return { checked: false, reason: "Answer too short to compare" }; | |
| } | |
| const hash = answerHash(text); | |
| // Layer 1 - exact match, a single indexed lookup. | |
| let exact = await db | |
| .collection(EVALUATIONS_COLLECTION) | |
| .where("answerHash", "==", hash) | |
| .limit(5) | |
| .get(); | |
| const exactHits = exact.docs | |
| .map((d) => ({ id: d.id, ...d.data() })) | |
| .filter((d) => !excludeUid || d.userUid !== excludeUid); | |
| if (exactHits.length) { | |
| return { | |
| checked: true, | |
| isDuplicate: true, | |
| matchType: "exact", | |
| similarity: 1, | |
| matchCount: exactHits.length, | |
| matches: exactHits.slice(0, 3).map((d) => ({ | |
| evaluationId: d.id, | |
| studentName: d.studentName || null, | |
| subject: d.subject || null, | |
| createdAt: d.createdAt?.toDate?.().toISOString() || null, | |
| })), | |
| }; | |
| } | |
| // Layer 2 - near match against recent submissions. | |
| let q = db.collection(EVALUATIONS_COLLECTION).orderBy("createdAt", "desc").limit(COMPARE_LIMIT); | |
| if (subject) q = db.collection(EVALUATIONS_COLLECTION) | |
| .where("subject", "==", subject) | |
| .orderBy("createdAt", "desc") | |
| .limit(COMPARE_LIMIT); | |
| let snap; | |
| try { | |
| snap = await q.get(); | |
| } catch (error) { | |
| // A missing composite index must not break grading - degrade to exact-match only. | |
| return { | |
| checked: true, | |
| isDuplicate: false, | |
| matchType: "exact-only", | |
| note: "Near-duplicate scan skipped: " + error.message, | |
| }; | |
| } | |
| const mine = shingles(text); | |
| let best = null; | |
| for (const doc of snap.docs) { | |
| const data = doc.data(); | |
| if (excludeUid && data.userUid === excludeUid) continue; | |
| if (!data.answer) continue; | |
| const score = jaccard(mine, shingles(data.answer)); | |
| if (!best || score > best.similarity) { | |
| best = { | |
| evaluationId: doc.id, | |
| similarity: score, | |
| studentName: data.studentName || null, | |
| subject: data.subject || null, | |
| createdAt: data.createdAt?.toDate?.().toISOString() || null, | |
| }; | |
| } | |
| } | |
| if (best && best.similarity >= NEAR_THRESHOLD) { | |
| return { | |
| checked: true, | |
| isDuplicate: true, | |
| matchType: "near", | |
| similarity: Math.round(best.similarity * 1000) / 1000, | |
| matchCount: 1, | |
| matches: [best], | |
| comparedAgainst: snap.size, | |
| }; | |
| } | |
| return { | |
| checked: true, | |
| isDuplicate: false, | |
| matchType: "none", | |
| similarity: best ? Math.round(best.similarity * 1000) / 1000 : 0, | |
| comparedAgainst: snap.size, | |
| }; | |
| }; | |