File size: 7,180 Bytes
54137cd
f1d9169
 
54137cd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f1d9169
 
54137cd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f1d9169
54137cd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Source Overlap</title>
<style>
  :root{
    --paper:#f7f6f2; --ink:#1c1f24; --muted:#6b6f76; --line:#d9d7d0;
    --flag:#b3261e; --flagbg:#fbe9e7; --ok:#2e6b3a; --okbg:#e8f2ea; --warn:#8a6d00; --warnbg:#fbf3d9;
  }
  *{box-sizing:border-box}
  body{margin:0;background:var(--paper);color:var(--ink);font:16px/1.5 Georgia,"Times New Roman",serif}
  main{max-width:820px;margin:0 auto;padding:40px 20px 80px}
  h1{font-size:28px;font-weight:normal;margin:0 0 6px}
  p.lead{color:var(--muted);margin:0 0 32px;max-width:60ch}
  .src{display:grid;grid-template-columns:32px 1fr;gap:12px;margin-bottom:14px;align-items:start}
  .src span{font-size:14px;color:var(--muted);padding-top:10px}
  textarea{width:100%;min-height:64px;padding:10px 12px;border:1px solid var(--line);background:#fff;
    font:15px/1.45 Georgia,serif;color:var(--ink);resize:vertical}
  textarea:focus{outline:2px solid var(--ink);outline-offset:1px}
  .row{display:flex;gap:12px;align-items:center;margin:18px 0 32px}
  button{padding:10px 18px;border:1px solid var(--ink);background:var(--ink);color:#fff;font:15px Georgia,serif;cursor:pointer}
  button.quiet{background:transparent;color:var(--ink)}
  button:disabled{opacity:.5;cursor:default}
  #status{font-size:14px;color:var(--muted)}
  #out{display:none}
  .verdict{padding:14px 16px;border-left:4px solid;margin-bottom:24px}
  .verdict.flag{border-color:var(--flag);background:var(--flagbg)}
  .verdict.ok{border-color:var(--ok);background:var(--okbg)}
  .pair{display:grid;grid-template-columns:56px 1fr;gap:12px;padding:10px 0;border-top:1px solid var(--line)}
  .pair b{font-weight:normal;font-variant-numeric:tabular-nums}
  .pair.high b{color:var(--flag)} .pair.mid b{color:var(--warn)}
  .pair small{color:var(--muted)}
  table{border-collapse:collapse;margin-top:28px;font-size:14px;font-variant-numeric:tabular-nums}
  td,th{padding:6px 10px;border:1px solid var(--line);text-align:center;min-width:52px}
  th{font-weight:normal;color:var(--muted)}
  td.high{background:var(--flagbg);color:var(--flag)} td.mid{background:var(--warnbg)}
  .how{margin-top:40px;font-size:14px;color:var(--muted);max-width:64ch}
</style>
</head>
<body>
<main>
  <h1>Source overlap</h1>
  <p class="lead">Paste several reports about the same event. This estimates which ones are restating one another rather than adding independent information. Runs in your browser; nothing is uploaded.</p>

  <div id="inputs"></div>

  <div class="row">
    <button id="run">Compare</button>
    <button id="add" class="quiet">Add a source</button>
    <span id="status">Loading model…</span>
  </div>

  <section id="out">
    <div id="verdict" class="verdict"></div>
    <div id="pairs"></div>
    <table id="grid"></table>
  </section>

  <p class="how">How it works: each snippet is turned into a vector with a small sentence-embedding model (all-MiniLM-L6-v2), and every pair is scored by cosine similarity. Above 0.85 is likely the same claim reworded; 0.70–0.85 is worth checking. It measures wording, not provenance — two genuinely independent reports of the same fact will also score high. Treat a flag as a prompt to trace the citations, not as proof.</p>
</main>

<script type="module">
import { pipeline } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.3.3";

const sample = [
  "Local Gazette: Zeta Corp will close Factory 7 by spring, according to an employee who asked not to be named.",
  "Industry Weekly: Zeta's Factory 7 is expected to shut down in the coming months, the Local Gazette reported this week.",
  "Manufacturing Blog: Multiple reports now indicate Factory 7 is closing, citing the Gazette and Industry Weekly.",
  "Zeta Corp quarterly filing: Production at Factory 7 will be reduced approximately 25% in the next fiscal year.",
  "Regional TV: Zeta Corp announced a new apprenticeship program at its Factory 7 site on Tuesday."
];

const inputs = document.getElementById("inputs");
const status = document.getElementById("status");
const run = document.getElementById("run");
let n = 0;

function addInput(text = "") {
  n++;
  const wrap = document.createElement("div");
  wrap.className = "src";
  wrap.innerHTML = `<span>${n}</span><textarea placeholder="Paste a report, headline, or paragraph">${text}</textarea>`;
  inputs.appendChild(wrap);
}
sample.forEach(addInput);
document.getElementById("add").onclick = () => addInput();

run.disabled = true;
const embed = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2", {
  progress_callback: p => { if (p.status === "progress") status.textContent = `Loading model… ${Math.round(p.progress)}%`; }
});
status.textContent = "Ready";
run.disabled = false;

const cos = (a, b) => a.reduce((s, x, i) => s + x * b[i], 0);

run.onclick = async () => {
  const texts = [...inputs.querySelectorAll("textarea")].map(t => t.value.trim());
  const idx = texts.map((t, i) => t ? i : -1).filter(i => i >= 0);
  if (idx.length < 2) { status.textContent = "Need at least two sources."; return; }

  run.disabled = true; status.textContent = "Comparing…";
  const out = await embed(idx.map(i => texts[i]), { pooling: "mean", normalize: true });
  const vecs = out.tolist();

  const pairs = [];
  for (let a = 0; a < idx.length; a++)
    for (let b = a + 1; b < idx.length; b++)
      pairs.push({ a: idx[a] + 1, b: idx[b] + 1, s: cos(vecs[a], vecs[b]) });
  pairs.sort((x, y) => y.s - x.s);

  const high = pairs.filter(p => p.s >= 0.85), mid = pairs.filter(p => p.s >= 0.70 && p.s < 0.85);
  const v = document.getElementById("verdict");
  if (high.length) {
    const linked = new Set(high.flatMap(p => [p.a, p.b]));
    v.className = "verdict flag";
    v.textContent = `${linked.size} of ${idx.length} sources look like the same claim reworded (${[...linked].sort().join(", ")}). Independent corroboration may be weaker than the count suggests.`;
  } else {
    v.className = "verdict ok";
    v.textContent = `No pair is near-duplicate. ${mid.length ? mid.length + " pair(s) overlap partially and are worth tracing." : "These read as distinct reports."}`;
  }

  document.getElementById("pairs").innerHTML = pairs.map(p =>
    `<div class="pair ${p.s >= 0.85 ? "high" : p.s >= 0.70 ? "mid" : ""}"><b>${p.s.toFixed(2)}</b><div>Source ${p.a} and source ${p.b}<br><small>${p.s >= 0.85 ? "likely restatement" : p.s >= 0.70 ? "partial overlap" : "distinct"}</small></div></div>`
  ).join("");

  const g = document.getElementById("grid");
  let html = "<tr><th></th>" + idx.map(i => `<th>${i + 1}</th>`).join("") + "</tr>";
  for (let a = 0; a < idx.length; a++) {
    html += `<tr><th>${idx[a] + 1}</th>`;
    for (let b = 0; b < idx.length; b++) {
      const s = a === b ? 1 : cos(vecs[a], vecs[b]);
      html += `<td class="${a !== b && s >= 0.85 ? "high" : a !== b && s >= 0.70 ? "mid" : ""}">${a === b ? "—" : s.toFixed(2)}</td>`;
    }
    html += "</tr>";
  }
  g.innerHTML = html;

  document.getElementById("out").style.display = "block";
  status.textContent = "Done"; run.disabled = false;
};
</script>
</body>
</html>