upgraedd commited on
Commit
b645469
·
verified ·
1 Parent(s): 700e2ba

Upload Anti hallucination (1).txt

Browse files
Files changed (1) hide show
  1. Anti hallucination (1).txt +2129 -0
Anti hallucination (1).txt ADDED
@@ -0,0 +1,2129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ EIS + ESL MEDIATOR v4.0 – Grounded Epistemic Engine
4
+ =====================================================
5
+ - Embedding‑based evidence matching (cosine similarity)
6
+ - Structural hallucination detection (novelty, entity absence)
7
+ - Semantic contradiction detection (embedding + polarity)
8
+ - Confidence decomposition (evidence, contradiction, suppression, hallucination)
9
+ - External grounding layer (Wikipedia API)
10
+ - Anti‑hallucination feedback loop
11
+ - Full backward compatibility with v3.9
12
+ """
13
+
14
+ import hashlib
15
+ import json
16
+ import os
17
+ import secrets
18
+ import time
19
+ import math
20
+ import re
21
+ from datetime import datetime, timedelta
22
+ from typing import Dict, List, Any, Optional, Tuple, Set
23
+ from collections import defaultdict
24
+ import requests
25
+
26
+ # ----------------------------------------------------------------------------
27
+ # LAZY EMBEDDER (from embeddings.py)
28
+ # ----------------------------------------------------------------------------
29
+ _EMBEDDER = None
30
+
31
+ def _load_embedder():
32
+ global _EMBEDDER
33
+ if _EMBEDDER is None:
34
+ try:
35
+ from sentence_transformers import SentenceTransformer
36
+ _EMBEDDER = SentenceTransformer('all-MiniLM-L6-v2')
37
+ except Exception:
38
+ _EMBEDDER = None
39
+ return _EMBEDDER
40
+
41
+ def _embed_texts(texts: List[str]) -> Optional[Any]:
42
+ model = _load_embedder()
43
+ if model is None:
44
+ return None
45
+ arr = model.encode(texts, convert_to_numpy=True, show_progress_bar=False)
46
+ return arr.astype('float32')
47
+
48
+ def _cosine_sim(a: Any, b: Any) -> float:
49
+ import numpy as np
50
+ from numpy.linalg import norm
51
+ a = np.array(a, dtype=np.float32)
52
+ b = np.array(b, dtype=np.float32)
53
+ denom = (norm(a) * norm(b) + 1e-12)
54
+ return float(np.dot(a, b) / denom)
55
+
56
+ # ----------------------------------------------------------------------------
57
+ # EXTERNAL GROUNDING LAYER (Wikipedia API)
58
+ # ----------------------------------------------------------------------------
59
+ def _fetch_wikipedia_summary(entity: str) -> Optional[str]:
60
+ """Fetch a short summary from Wikipedia for the given entity."""
61
+ try:
62
+ url = "https://en.wikipedia.org/api/rest_v1/page/summary/" + entity.replace(" ", "_")
63
+ resp = requests.get(url, timeout=5)
64
+ if resp.status_code == 200:
65
+ data = resp.json()
66
+ return data.get("extract", "")
67
+ except Exception:
68
+ pass
69
+ return None
70
+
71
+ # ----------------------------------------------------------------------------
72
+ # UTILITIES (drift, crowding, inoculation, attrition) – same as v3.9
73
+ # ----------------------------------------------------------------------------
74
+ def _compute_entity_drift(embeddings_tuples: List[Dict]) -> List[Dict]:
75
+ if not embeddings_tuples:
76
+ return []
77
+ import numpy as np
78
+ arrs = [np.array(e["embedding"], dtype=np.float32) for e in embeddings_tuples]
79
+ baseline_count = max(1, len(arrs)//4)
80
+ baseline = np.mean(arrs[:baseline_count], axis=0)
81
+ drift = []
82
+ for rec, emb in zip(embeddings_tuples, arrs):
83
+ sim = _cosine_sim(baseline, emb)
84
+ drift.append({
85
+ "timestamp": rec["timestamp"],
86
+ "similarity_to_baseline": sim,
87
+ "drift_score": 1.0 - sim
88
+ })
89
+ return drift
90
+
91
+ def _semantic_drift_score(emb_timeline: List[Dict], window: int = 7) -> float:
92
+ if not emb_timeline or len(emb_timeline) < 4:
93
+ return 0.0
94
+ import numpy as np
95
+ arrs = [np.array(e["embedding"], dtype=np.float32) for e in emb_timeline]
96
+ baseline = np.mean(arrs[:max(1, len(arrs)//4)], axis=0)
97
+ sims = [float(np.dot(baseline, v) / (np.linalg.norm(baseline)*np.linalg.norm(v)+1e-12)) for v in arrs]
98
+ recent = sims[-min(window, len(sims)):]
99
+ velocity = 0.0
100
+ if len(recent) >= 2:
101
+ velocity = (recent[-1] - recent[0]) / max(1, len(recent)-1)
102
+ drift = max(0.0, 1.0 - recent[-1])
103
+ velocity_component = -velocity if velocity < 0 else 0.0
104
+ return float(min(1.0, drift + velocity_component))
105
+
106
+ def _shingle_hashes(s: str, k: int = 5) -> Set[int]:
107
+ toks = [t for t in re.split(r'\s+', s.lower()) if t]
108
+ if len(toks) < k:
109
+ return {hash(" ".join(toks))}
110
+ return {hash(" ".join(toks[i:i+k])) for i in range(max(0, len(toks)-k+1))}
111
+
112
+ def _crowding_signature(esl: 'ESLedger', window_days: int = 3, dup_threshold: float = 0.6):
113
+ now = datetime.utcnow()
114
+ cutoff = now - timedelta(days=window_days)
115
+ texts = []
116
+ for cid, c in esl.claims.items():
117
+ try:
118
+ ts = datetime.fromisoformat(c["timestamp"].replace('Z', '+00:00'))
119
+ except Exception:
120
+ continue
121
+ if ts >= cutoff:
122
+ texts.append(c.get("text", ""))
123
+ if len(texts) < 2:
124
+ return None
125
+ hashes = [_shingle_hashes(t) for t in texts]
126
+ pairs = 0
127
+ near_dup = 0
128
+ for i in range(len(hashes)):
129
+ for j in range(i+1, len(hashes)):
130
+ pairs += 1
131
+ inter = len(hashes[i].intersection(hashes[j]))
132
+ union = len(hashes[i].union(hashes[j])) + 1e-9
133
+ if inter/union > dup_threshold:
134
+ near_dup += 1
135
+ dup_frac = (near_dup / pairs) if pairs else 0.0
136
+ if dup_frac > dup_threshold:
137
+ weight = min(0.9, 0.5 + dup_frac)
138
+ return ("crowding_noise_floor", weight)
139
+ return None
140
+
141
+ def _compute_attrition_score(workflow_events: List[Dict]) -> float:
142
+ if not workflow_events:
143
+ return 0.0
144
+ events_sorted = sorted(workflow_events, key=lambda e: e["timestamp"])
145
+ durations = []
146
+ loops = 0
147
+ for i in range(len(events_sorted)-1):
148
+ dt = (events_sorted[i+1]["timestamp"] - events_sorted[i]["timestamp"]).total_seconds()
149
+ durations.append(dt)
150
+ if events_sorted[i].get("status") == "request_more_info" and events_sorted[i+1].get("status") == "resubmission":
151
+ loops += 1
152
+ median_duration_days = (sorted(durations)[len(durations)//2] / 86400) if durations else 0
153
+ score = min(1.0, (median_duration_days / 30.0) + (loops * 0.1))
154
+ return score
155
+
156
+ def _inoculation_signature(esl: 'ESLedger', claim_id: str, lead_window_days: int = 7, sim_threshold: float = 0.72):
157
+ base_claim = esl.claims.get(claim_id)
158
+ if not base_claim:
159
+ return None
160
+ emb = base_claim.get("embedding")
161
+ if emb is None:
162
+ return None
163
+ try:
164
+ import numpy as np
165
+ except Exception:
166
+ return None
167
+ now = datetime.utcnow()
168
+ cutoff = now - timedelta(days=lead_window_days*2)
169
+ similar_pairs = []
170
+ for cid, c in esl.claims.items():
171
+ if cid == claim_id:
172
+ continue
173
+ try:
174
+ ts = datetime.fromisoformat(c["timestamp"].replace('Z', '+00:00'))
175
+ except Exception:
176
+ continue
177
+ if ts < cutoff:
178
+ continue
179
+ emb2 = c.get("embedding")
180
+ if emb2 is None:
181
+ continue
182
+ sim = float(np.dot(np.array(emb), np.array(emb2)) / ((np.linalg.norm(emb)*np.linalg.norm(emb2))+1e-12))
183
+ if sim >= sim_threshold:
184
+ similar_pairs.append((cid, ts, sim))
185
+ if not similar_pairs:
186
+ return None
187
+ try:
188
+ base_ts = datetime.fromisoformat(base_claim["timestamp"].replace('Z', '+00:00'))
189
+ except Exception:
190
+ return None
191
+ leads = [(base_ts - ts).total_seconds() for (_, ts, _) in similar_pairs]
192
+ mean_lead = sum(leads)/len(leads)
193
+ if mean_lead > (24*3600):
194
+ weight = min(0.9, 0.3 + min(0.7, abs(mean_lead)/(7*24*3600)))
195
+ return ("preemptive_inoculation", weight)
196
+ return None
197
+
198
+ # ----------------------------------------------------------------------------
199
+ # NEGATION AND POLARITY HELPERS
200
+ # ----------------------------------------------------------------------------
201
+ NEGATION_WORDS = {"not", "no", "never", "false", "didn't", "isn't", "wasn't", "weren't", "cannot", "couldn't", "wouldn't", "shouldn't"}
202
+ ANTONYMS = {
203
+ "suppressed": "revealed", "erased": "preserved", "hidden": "public",
204
+ "denied": "confirmed", "falsified": "verified", "concealed": "disclosed"
205
+ }
206
+
207
+ def has_negation(text: str, entity: str = None) -> bool:
208
+ words = text.lower().split()
209
+ if entity:
210
+ for i, w in enumerate(words):
211
+ if entity.lower() in w or w == entity.lower():
212
+ start = max(0, i-5)
213
+ preceding = words[start:i]
214
+ if any(neg in preceding for neg in NEGATION_WORDS):
215
+ return True
216
+ else:
217
+ if any(neg in words for neg in NEGATION_WORDS):
218
+ return True
219
+ return False
220
+
221
+ def claim_polarity(text: str) -> float:
222
+ return 0.3 if has_negation(text) else 1.0
223
+
224
+ # ----------------------------------------------------------------------------
225
+ # ENTITY EXTRACTION (spaCy / TextBlob / fallback)
226
+ # ----------------------------------------------------------------------------
227
+ try:
228
+ import spacy
229
+ _nlp = spacy.load("en_core_web_sm")
230
+ HAS_SPACY = True
231
+ except ImportError:
232
+ HAS_SPACY = False
233
+ _nlp = None
234
+
235
+ try:
236
+ from textblob import TextBlob
237
+ HAS_TEXTBLOB = True
238
+ except ImportError:
239
+ HAS_TEXTBLOB = False
240
+
241
+ def extract_entities(text: str) -> List[Tuple[str, str, bool]]:
242
+ entities = []
243
+ if HAS_SPACY and _nlp:
244
+ doc = _nlp(text)
245
+ for ent in doc.ents:
246
+ negated = has_negation(text, ent.text)
247
+ entities.append((ent.text, ent.label_, negated))
248
+ for chunk in doc.noun_chunks:
249
+ if chunk.text not in [e[0] for e in entities] and len(chunk.text.split()) <= 3 and chunk.text[0].isupper():
250
+ negated = has_negation(text, chunk.text)
251
+ entities.append((chunk.text, "NOUN_PHRASE", negated))
252
+ return entities
253
+ if HAS_TEXTBLOB:
254
+ blob = TextBlob(text)
255
+ for np in blob.noun_phrases:
256
+ if np[0].isupper() or np in ["CIA", "FBI", "NSA", "Pentagon"]:
257
+ negated = has_negation(text, np)
258
+ entities.append((np, "NOUN_PHRASE", negated))
259
+ words = text.split()
260
+ i = 0
261
+ while i < len(words):
262
+ if words[i] and words[i][0].isupper() and len(words[i]) > 1:
263
+ phrase = [words[i]]
264
+ j = i+1
265
+ while j < len(words) and words[j] and words[j][0].isupper():
266
+ phrase.append(words[j])
267
+ j += 1
268
+ ent = " ".join(phrase)
269
+ negated = has_negation(text, ent)
270
+ entities.append((ent, "PROPER_NOUN", negated))
271
+ i = j
272
+ else:
273
+ i += 1
274
+ return entities
275
+ # final fallback
276
+ pattern = r'\b[A-Z][a-z]*(?:\s+[A-Z][a-z]*)*\b'
277
+ matches = re.findall(pattern, text)
278
+ for match in matches:
279
+ if len(match.split()) <= 4 and match not in ["The", "This", "That", "These", "Those", "I", "We", "They"]:
280
+ negated = has_negation(text, match)
281
+ entities.append((match, "UNKNOWN", negated))
282
+ return entities
283
+
284
+ # ----------------------------------------------------------------------------
285
+ # TAXONOMY (methods, primitives, lenses) – same as v3.9
286
+ # ----------------------------------------------------------------------------
287
+ METHODS = {
288
+ 1: {"name": "Total Erasure", "primitive": "ERASURE", "signatures": ["entity_present_then_absent", "abrupt_disappearance"]},
289
+ 2: {"name": "Soft Erasure", "primitive": "ERASURE", "signatures": ["gradual_fading", "citation_decay"]},
290
+ 3: {"name": "Citation Decay", "primitive": "ERASURE", "signatures": ["decreasing_citations"]},
291
+ 4: {"name": "Index Removal", "primitive": "ERASURE", "signatures": ["missing_from_indices"]},
292
+ 5: {"name": "Selective Retention", "primitive": "ERASURE", "signatures": ["archival_gaps"]},
293
+ 10: {"name": "Narrative Seizure", "primitive": "NARRATIVE_CAPTURE", "signatures": ["single_explanation"]},
294
+ 12: {"name": "Official Story", "primitive": "NARRATIVE_CAPTURE", "signatures": ["authoritative_sources"]},
295
+ 14: {"name": "Temporal Gaps", "primitive": "TEMPORAL", "signatures": ["publication_gap"]},
296
+ 15: {"name": "Latency Spikes", "primitive": "TEMPORAL", "signatures": ["delayed_reporting"]},
297
+ 17: {"name": "Smear Campaign", "primitive": "DISCREDITATION", "signatures": ["ad_hominem_attacks"]},
298
+ 23: {"name": "Whataboutism", "primitive": "MISDIRECTION", "signatures": ["deflection"]},
299
+ 43: {"name": "Conditioning", "primitive": "CONDITIONING", "signatures": ["repetitive_messaging"]},
300
+ }
301
+
302
+ LENSES = {
303
+ 1: "Threat→Response→Control→Enforce→Centralize",
304
+ 2: "Sacred Geometry Weaponized",
305
+ 3: "Language Inversions / Ridicule / Gatekeeping",
306
+ 4: "Crisis→Consent→Surveillance",
307
+ 5: "Divide and Fragment",
308
+ 6: "Blame the Victim",
309
+ 7: "Narrative Capture through Expertise",
310
+ 8: "Information Saturation",
311
+ 9: "Historical Revisionism",
312
+ 10: "Institutional Capture",
313
+ 11: "Access Control via Credentialing",
314
+ 12: "Temporal Displacement",
315
+ 13: "Moral Equivalence",
316
+ 14: "Whataboutism",
317
+ 15: "Ad Hominem",
318
+ 16: "Straw Man",
319
+ 17: "False Dichotomy",
320
+ 18: "Slippery Slope",
321
+ 19: "Appeal to Authority",
322
+ 20: "Appeal to Nature",
323
+ 21: "Appeal to Tradition",
324
+ 22: "Appeal to Novelty",
325
+ 23: "Cherry Picking",
326
+ 24: "Moving the Goalposts",
327
+ 25: "Burden of Proof Reversal",
328
+ 26: "Circular Reasoning",
329
+ 27: "Special Pleading",
330
+ 28: "Loaded Question",
331
+ 29: "No True Scotsman",
332
+ 30: "Texas Sharpshooter",
333
+ 31: "Middle Ground Fallacy",
334
+ 32: "Black-and-White Thinking",
335
+ 33: "Fear Mongering",
336
+ 34: "Flattery",
337
+ 35: "Guilt by Association",
338
+ 36: "Transfer",
339
+ 37: "Testimonial",
340
+ 38: "Plain Folks",
341
+ 39: "Bandwagon",
342
+ 40: "Snob Appeal",
343
+ 41: "Glittering Generalities",
344
+ 42: "Name-Calling",
345
+ 43: "Card Stacking",
346
+ 44: "Euphemisms",
347
+ 45: "Dysphemisms",
348
+ 46: "Weasel Words",
349
+ 47: "Thought-Terminating Cliché",
350
+ 48: "Proof by Intimidation",
351
+ 49: "Proof by Verbosity",
352
+ 50: "Sealioning",
353
+ 51: "Gish Gallop",
354
+ 52: "JAQing Off",
355
+ 53: "Nutpicking",
356
+ 54: "Concern Trolling",
357
+ 55: "Gaslighting",
358
+ 56: "Kafkatrapping",
359
+ 57: "Brandolini's Law",
360
+ 58: "Occam's Razor",
361
+ 59: "Hanlon's Razor",
362
+ 60: "Hitchens's Razor",
363
+ 61: "Popper's Falsification",
364
+ 62: "Sagan's Standard",
365
+ 63: "Newton's Flaming Laser Sword",
366
+ 64: "Alder's Razor",
367
+ 65: "Grice's Maxims",
368
+ 66: "Poe's Law",
369
+ 67: "Sturgeon's Law",
370
+ 68: "Betteridge's Law",
371
+ 69: "Godwin's Law",
372
+ 70: "Skoptsy Syndrome",
373
+ }
374
+
375
+ PRIMITIVE_TO_LENSES = {
376
+ "ERASURE": [31, 53, 71, 24, 54, 4, 37, 45, 46],
377
+ "INTERRUPTION": [19, 33, 30, 63, 10, 61, 12, 26],
378
+ "FRAGMENTATION": [2, 52, 15, 20, 3, 29, 31, 54],
379
+ "NARRATIVE_CAPTURE": [1, 34, 40, 64, 7, 16, 22, 47],
380
+ "MISDIRECTION": [5, 21, 8, 36, 27, 61],
381
+ "SATURATION": [41, 69, 3, 36, 34, 66],
382
+ "DISCREDITATION": [3, 27, 10, 40, 30, 63],
383
+ "ATTRITION": [13, 19, 14, 33, 19, 27],
384
+ "ACCESS_CONTROL": [25, 62, 37, 51, 23, 53],
385
+ "TEMPORAL": [22, 47, 26, 68, 12, 22],
386
+ "CONDITIONING": [8, 36, 34, 43, 27, 33],
387
+ "META": [23, 70, 34, 64, 23, 40, 18, 71, 46, 31, 5, 21]
388
+ }
389
+
390
+ def map_signature_to_method(signature: str) -> Optional[Dict]:
391
+ for mid, method in METHODS.items():
392
+ if signature in method["signatures"]:
393
+ return {"method_id": mid, "method_name": method["name"], "primitive": method["primitive"]}
394
+ return None
395
+
396
+ def get_lenses_for_primitive(primitive: str) -> List[int]:
397
+ return PRIMITIVE_TO_LENSES.get(primitive, [])
398
+
399
+ def get_lens_name(lens_id: int) -> str:
400
+ return LENSES.get(lens_id, f"Lens {lens_id} (unknown)")
401
+
402
+ # ----------------------------------------------------------------------------
403
+ # EPISTEMIC SUBSTRATE LEDGER (ESL) – with external grounding support
404
+ # ----------------------------------------------------------------------------
405
+ class ESLedger:
406
+ def __init__(self, path: str = "esl_ledger.json"):
407
+ self.path = path
408
+ self.claims: Dict[str, Dict] = {}
409
+ self.entities: Dict[str, Dict] = {}
410
+ self.signatures: List[Dict] = []
411
+ self.contradiction_graph: Dict[str, Set[str]] = defaultdict(set)
412
+ self.blocks: List[Dict] = []
413
+ self._load()
414
+
415
+ def _load(self):
416
+ if os.path.exists(self.path):
417
+ try:
418
+ with open(self.path, 'r') as f:
419
+ data = json.load(f)
420
+ self.claims = data.get("claims", {})
421
+ self.entities = data.get("entities", {})
422
+ self.signatures = data.get("signatures", [])
423
+ self.blocks = data.get("blocks", [])
424
+ cg = data.get("contradiction_graph", {})
425
+ self.contradiction_graph = {k: set(v) for k, v in cg.items()}
426
+ except Exception:
427
+ pass
428
+
429
+ def _save(self):
430
+ cg_serializable = {k: list(v) for k, v in self.contradiction_graph.items()}
431
+ data = {
432
+ "claims": self.claims,
433
+ "entities": self.entities,
434
+ "signatures": self.signatures,
435
+ "contradiction_graph": cg_serializable,
436
+ "blocks": self.blocks,
437
+ "updated": datetime.utcnow().isoformat() + "Z"
438
+ }
439
+ with open(self.path + ".tmp", 'w') as f:
440
+ json.dump(data, f, indent=2)
441
+ os.replace(self.path + ".tmp", self.path)
442
+
443
+ def add_claim(self, text: str, agent: str = "user") -> str:
444
+ claim_id = secrets.token_hex(16)
445
+ polarity = claim_polarity(text)
446
+ self.claims[claim_id] = {
447
+ "id": claim_id, "text": text, "agent": agent,
448
+ "timestamp": datetime.utcnow().isoformat() + "Z",
449
+ "entities": [], "signatures": [], "coherence": 0.5,
450
+ "contradictions": [], "suppression_score": 0.0,
451
+ "methods": [], "primitives": [], "lenses": [],
452
+ "polarity": polarity,
453
+ "source_types": [],
454
+ "embedding": None,
455
+ "workflow_events": []
456
+ }
457
+ self._save()
458
+ emb_arr = _embed_texts([text])
459
+ if emb_arr is not None:
460
+ self.claims[claim_id]["embedding"] = emb_arr[0].tolist()
461
+ self._save()
462
+ return claim_id
463
+
464
+ def add_entity(self, name: str, etype: str, claim_id: str, negated: bool = False, source_type: str = "unknown"):
465
+ if name not in self.entities:
466
+ self.entities[name] = {
467
+ "name": name, "type": etype,
468
+ "first_seen": datetime.utcnow().isoformat() + "Z",
469
+ "last_seen": self.claims[claim_id]["timestamp"],
470
+ "appearances": [], "coherence_scores": [],
471
+ "suppression_score": 0.0,
472
+ "negated_mentions": [],
473
+ "source_types": {},
474
+ "embeddings": [],
475
+ "external_summary": None
476
+ }
477
+ # Fetch external grounding (Wikipedia summary) lazily
478
+ try:
479
+ summary = _fetch_wikipedia_summary(name)
480
+ if summary:
481
+ self.entities[name]["external_summary"] = summary[:500]
482
+ except Exception:
483
+ pass
484
+ ent = self.entities[name]
485
+ if claim_id not in ent["appearances"]:
486
+ ent["appearances"].append(claim_id)
487
+ if negated:
488
+ ent["negated_mentions"].append(claim_id)
489
+ ent["last_seen"] = self.claims[claim_id]["timestamp"]
490
+ ent["source_types"][source_type] = ent["source_types"].get(source_type, 0) + 1
491
+ if "entities" not in self.claims[claim_id]:
492
+ self.claims[claim_id]["entities"] = []
493
+ if claim_id not in self.claims[claim_id]["entities"]:
494
+ self.claims[claim_id]["entities"].append(name)
495
+ if "source_types" not in self.claims[claim_id]:
496
+ self.claims[claim_id]["source_types"] = []
497
+ if source_type not in self.claims[claim_id]["source_types"]:
498
+ self.claims[claim_id]["source_types"].append(source_type)
499
+ emb = self.claims[claim_id].get("embedding")
500
+ if emb is not None:
501
+ ent.setdefault("embeddings", []).append({
502
+ "timestamp": self.claims[claim_id]["timestamp"],
503
+ "embedding": emb,
504
+ "claim_id": claim_id,
505
+ "text_snippet": self.claims[claim_id]["text"][:512]
506
+ })
507
+ self._save()
508
+
509
+ def add_signature(self, claim_id: str, sig_name: str, weight: float = 0.5, context: Dict = None):
510
+ polarity = self.claims[claim_id].get("polarity", 1.0)
511
+ adjusted_weight = weight * polarity
512
+ method_info = map_signature_to_method(sig_name)
513
+ primitive = method_info["primitive"] if method_info else "UNKNOWN"
514
+ lenses = get_lenses_for_primitive(primitive) if primitive != "UNKNOWN" else []
515
+ self.signatures.append({
516
+ "signature": sig_name, "claim_id": claim_id,
517
+ "timestamp": datetime.utcnow().isoformat() + "Z",
518
+ "weight": adjusted_weight, "context": context or {},
519
+ "method": method_info["method_name"] if method_info else None,
520
+ "primitive": primitive,
521
+ "lenses": lenses
522
+ })
523
+ if sig_name not in self.claims[claim_id]["signatures"]:
524
+ self.claims[claim_id]["signatures"].append(sig_name)
525
+ if method_info and method_info["method_name"] not in self.claims[claim_id]["methods"]:
526
+ self.claims[claim_id]["methods"].append(method_info["method_name"])
527
+ if primitive not in self.claims[claim_id]["primitives"]:
528
+ self.claims[claim_id]["primitives"].append(primitive)
529
+ for lens in lenses:
530
+ if lens not in self.claims[claim_id]["lenses"]:
531
+ self.claims[claim_id]["lenses"].append(lens)
532
+
533
+ combined = 1.0
534
+ for sig in self.claims[claim_id]["signatures"]:
535
+ w = 0.5
536
+ for log in self.signatures:
537
+ if log["signature"] == sig and log["claim_id"] == claim_id:
538
+ w = log.get("weight", 0.5)
539
+ break
540
+ combined *= (1 - w)
541
+ new_score = 1 - combined
542
+ self.claims[claim_id]["suppression_score"] = new_score
543
+
544
+ for entity in self.claims[claim_id]["entities"]:
545
+ ent = self.entities.get(entity)
546
+ if ent:
547
+ ent_combined = 1.0
548
+ for cid in ent["appearances"]:
549
+ sc = self.claims[cid].get("suppression_score", 0.0)
550
+ ent_combined *= (1 - sc)
551
+ ent["suppression_score"] = 1 - ent_combined
552
+ self._save()
553
+
554
+ def add_contradiction(self, claim_id_a: str, claim_id_b: str):
555
+ self.contradiction_graph[claim_id_a].add(claim_id_b)
556
+ self.contradiction_graph[claim_id_b].add(claim_id_a)
557
+ if claim_id_b not in self.claims[claim_id_a]["contradictions"]:
558
+ self.claims[claim_id_a]["contradictions"].append(claim_id_b)
559
+ if claim_id_a not in self.claims[claim_id_b]["contradictions"]:
560
+ self.claims[claim_id_b]["contradictions"].append(claim_id_a)
561
+ self._save()
562
+
563
+ def get_entity_coherence(self, entity_name: str) -> float:
564
+ ent = self.entities.get(entity_name)
565
+ if not ent or len(ent["appearances"]) < 2:
566
+ return 0.5
567
+ timestamps = []
568
+ for cid in ent["appearances"]:
569
+ ts = self.claims[cid]["timestamp"]
570
+ timestamps.append(datetime.fromisoformat(ts.replace('Z', '+00:00')))
571
+ intervals = [(timestamps[i+1] - timestamps[i]).total_seconds() / 86400 for i in range(len(timestamps)-1)]
572
+ if not intervals:
573
+ return 0.5
574
+ mean_int = sum(intervals) / len(intervals)
575
+ variance = sum((i - mean_int)**2 for i in intervals) / len(intervals)
576
+ coherence = 1.0 / (1.0 + variance)
577
+ return min(1.0, max(0.0, coherence))
578
+
579
+ def get_entity_embeddings(self, entity_name: str) -> List[Dict]:
580
+ ent = self.entities.get(entity_name)
581
+ if not ent:
582
+ return []
583
+ return sorted(ent.get("embeddings", []), key=lambda x: x["timestamp"])
584
+
585
+ def suppression_pattern_classifier(self, claim_id: str) -> Dict:
586
+ claim = self.claims.get(claim_id, {})
587
+ sig_names = claim.get("signatures", [])
588
+ if not sig_names:
589
+ return {"level": "none", "score": 0.0, "patterns": [], "primitives": [], "lenses": [], "contributions": {}}
590
+ score = claim.get("suppression_score", 0.0)
591
+ contributions = {}
592
+ for log in self.signatures:
593
+ if log["claim_id"] == claim_id:
594
+ contributions[log["signature"]] = contributions.get(log["signature"], 0.0) + log.get("weight", 0.0)
595
+ if score > 0.7:
596
+ level = "high"
597
+ elif score > 0.4:
598
+ level = "medium"
599
+ elif score > 0.1:
600
+ level = "low"
601
+ else:
602
+ level = "none"
603
+ primitives = claim.get("primitives", [])
604
+ lenses = claim.get("lenses", [])
605
+ return {
606
+ "level": level,
607
+ "score": score,
608
+ "contributions": contributions,
609
+ "patterns": list(set(sig_names)),
610
+ "primitives": primitives,
611
+ "lenses": lenses
612
+ }
613
+
614
+ def get_entity_timeline(self, name: str) -> List[Dict]:
615
+ ent = self.entities.get(name)
616
+ if not ent:
617
+ return []
618
+ timeline = []
619
+ for cid in ent["appearances"]:
620
+ claim = self.claims.get(cid)
621
+ if claim:
622
+ timeline.append({
623
+ "timestamp": claim["timestamp"],
624
+ "text": claim["text"],
625
+ "negated": cid in ent.get("negated_mentions", [])
626
+ })
627
+ timeline.sort(key=lambda x: x["timestamp"])
628
+ return timeline
629
+
630
+ def disappearance_suspected(self, name: str, threshold_days: int = 30) -> bool:
631
+ timeline = self.get_entity_timeline(name)
632
+ if not timeline:
633
+ return False
634
+ last = datetime.fromisoformat(timeline[-1]["timestamp"].replace('Z', '+00:00'))
635
+ now = datetime.utcnow()
636
+ return (now - last).days > threshold_days
637
+
638
+ def create_block(self) -> Dict:
639
+ block = {
640
+ "index": len(self.blocks),
641
+ "timestamp": datetime.utcnow().isoformat() + "Z",
642
+ "prev_hash": self.blocks[-1]["hash"] if self.blocks else "0"*64,
643
+ "state_hash": hashlib.sha3_512(json.dumps({"claims": self.claims, "entities": self.entities}, sort_keys=True).encode()).hexdigest()
644
+ }
645
+ block["hash"] = hashlib.sha3_512(json.dumps(block, sort_keys=True).encode()).hexdigest()
646
+ self.blocks.append(block)
647
+ self._save()
648
+ return block
649
+
650
+ def find_contradictions(self, claim_text: str, claim_embedding: Optional[List[float]] = None) -> List[str]:
651
+ contradictions = []
652
+ if claim_embedding is None:
653
+ claim_embedding = _embed_texts([claim_text])
654
+ if claim_embedding is not None:
655
+ claim_embedding = claim_embedding[0].tolist()
656
+ if claim_embedding is None:
657
+ return []
658
+ for cid, claim in self.claims.items():
659
+ emb2 = claim.get("embedding")
660
+ if emb2 is None:
661
+ continue
662
+ sim = _cosine_sim(claim_embedding, emb2)
663
+ # Semantic contradiction: similarity > 0.7 and opposite polarity
664
+ if sim > 0.7 and claim_polarity(claim_text) != claim.get("polarity", 1.0):
665
+ contradictions.append(cid)
666
+ return contradictions
667
+
668
+ def get_suppression_trend(self, window_days: int = 30) -> List[Dict]:
669
+ trend = defaultdict(list)
670
+ for claim in self.claims.values():
671
+ ts = datetime.fromisoformat(claim["timestamp"].replace('Z', '+00:00'))
672
+ date = ts.date().isoformat()
673
+ trend[date].append(claim.get("suppression_score", 0.0))
674
+ result = []
675
+ for date, scores in sorted(trend.items()):
676
+ result.append({"date": date, "avg_suppression": sum(scores)/len(scores)})
677
+ cutoff = (datetime.utcnow() - timedelta(days=window_days)).date().isoformat()
678
+ result = [r for r in result if r["date"] >= cutoff]
679
+ return result
680
+
681
+ def get_entity_suppression(self, entity_name: str) -> Dict:
682
+ ent = self.entities.get(entity_name)
683
+ if not ent:
684
+ return {"name": entity_name, "score": 0.0}
685
+ return {
686
+ "name": entity_name,
687
+ "score": ent.get("suppression_score", 0.0),
688
+ "type": ent["type"],
689
+ "first_seen": ent["first_seen"],
690
+ "last_seen": ent["last_seen"],
691
+ "appearance_count": len(ent["appearances"]),
692
+ "negated_count": len(ent.get("negated_mentions", [])),
693
+ "coherence": self.get_entity_coherence(entity_name),
694
+ "source_types": dict(ent.get("source_types", {}))
695
+ }
696
+
697
+ def decay_confidence(self, half_life_days: float = 30.0):
698
+ now = datetime.utcnow()
699
+ for claim_id, claim in self.claims.items():
700
+ ts = datetime.fromisoformat(claim["timestamp"].replace('Z', '+00:00'))
701
+ age_days = (now - ts).days
702
+ if age_days > 0:
703
+ decay_factor = math.exp(-age_days / half_life_days)
704
+ claim["suppression_score"] *= decay_factor
705
+ self._save()
706
+
707
+ def ingest_actual_event(self, event_type: str, actor: str, target: str, source: str = "ActualRealityModule") -> str:
708
+ try:
709
+ import importlib
710
+ try:
711
+ mod = importlib.import_module("KENNEDY_V_REALITY")
712
+ except ImportError:
713
+ mod = importlib.import_module("KENNEDYVREALITY")
714
+ RealityInterface = getattr(mod, "RealityInterface", None)
715
+ if RealityInterface:
716
+ ri = RealityInterface()
717
+ analysis = ri.actual_reality.analyze_power_transfer(event_type, actor, target)
718
+ parts = [f"{k}: {v}" for k, v in analysis.items()]
719
+ claim_text = f"ActualReality analysis for {event_type} - " + " | ".join(parts)
720
+ cid = self.add_claim(claim_text, agent=source)
721
+ if actor:
722
+ self.add_entity(actor, "ACTOR", cid, negated=False)
723
+ if target:
724
+ self.add_entity(target, "TARGET", cid, negated=False)
725
+ for key in analysis.keys():
726
+ if key in ("power_transfer", "actual_dynamics"):
727
+ self.add_signature(cid, "entity_present_then_absent", weight=0.6, context={"source": source})
728
+ if key == "verification_control":
729
+ self.add_signature(cid, "citation_decay", weight=0.4, context={"source": source})
730
+ return cid
731
+ except Exception:
732
+ pass
733
+ claim_text = f"Event observed: {event_type} actor:{actor} target:{target}"
734
+ cid = self.add_claim(claim_text, agent=source)
735
+ if actor:
736
+ self.add_entity(actor, "ACTOR", cid, negated=False)
737
+ if target:
738
+ self.add_entity(target, "TARGET", cid, negated=False)
739
+ return cid
740
+
741
+ # ----------------------------------------------------------------------------
742
+ # FALSIFICATION ENGINE (unchanged, but uses new contradiction detection)
743
+ # ----------------------------------------------------------------------------
744
+ class FalsificationEngine:
745
+ def __init__(self, esl: ESLedger):
746
+ self.esl = esl
747
+
748
+ def alternative_cause(self, claim_text: str) -> Tuple[bool, str]:
749
+ if has_negation(claim_text):
750
+ return True, "Claim is negated; alternative cause not applicable."
751
+ for entity in self.esl.entities:
752
+ if entity.lower() in claim_text.lower():
753
+ if self.esl.disappearance_suspected(entity):
754
+ return False, f"Entity '{entity}' disappearance may be natural (no recent activity)."
755
+ return True, "No obvious alternative cause."
756
+
757
+ def contradictory_evidence(self, claim_id: str) -> Tuple[bool, str]:
758
+ contradictions = self.esl.contradiction_graph.get(claim_id, set())
759
+ if contradictions:
760
+ return False, f"Claim contradicts {len(contradictions)} existing claim(s)."
761
+ return True, "No direct contradictions."
762
+
763
+ def source_diversity(self, claim_text: str) -> Tuple[bool, str]:
764
+ entities_in_claim = [e for e in self.esl.entities if e.lower() in claim_text.lower()]
765
+ if len(entities_in_claim) <= 1:
766
+ return False, f"Claim relies on only {len(entities_in_claim)} entity/entities."
767
+ return True, f"Multiple entities ({len(entities_in_claim)}) involved."
768
+
769
+ def temporal_stability(self, claim_text: str) -> Tuple[bool, str]:
770
+ for entity in self.esl.entities:
771
+ if entity.lower() in claim_text.lower():
772
+ coherence = self.esl.get_entity_coherence(entity)
773
+ if coherence < 0.3:
774
+ return False, f"Entity '{entity}' has low temporal coherence ({coherence:.2f})."
775
+ return True, "Temporal coherence adequate."
776
+
777
+ def manipulation_check(self, claim_text: str, agent: str) -> Tuple[bool, str]:
778
+ manip_indicators = ["must", "cannot", "obviously", "clearly", "everyone knows"]
779
+ for word in manip_indicators:
780
+ if word in claim_text.lower():
781
+ return False, f"Manipulative language detected: '{word}'."
782
+ return True, "No manipulation indicators."
783
+
784
+ def run_all(self, claim_id: str, claim_text: str, agent: str) -> List[Dict]:
785
+ tests = [
786
+ ("alternative_cause", lambda: self.alternative_cause(claim_text)),
787
+ ("contradictory_evidence", lambda: self.contradictory_evidence(claim_id)),
788
+ ("source_diversity", lambda: self.source_diversity(claim_text)),
789
+ ("temporal_stability", lambda: self.temporal_stability(claim_text)),
790
+ ("manipulation_check", lambda: self.manipulation_check(claim_text, agent))
791
+ ]
792
+ results = []
793
+ for name, func in tests:
794
+ survived, reason = func()
795
+ results.append({"name": name, "survived": survived, "reason": reason})
796
+ return results
797
+
798
+ # ----------------------------------------------------------------------------
799
+ # SIGNATURE GENERATOR (with advanced detectors)
800
+ # ----------------------------------------------------------------------------
801
+ class SignatureGenerator:
802
+ def __init__(self, esl: ESLedger):
803
+ self.esl = esl
804
+
805
+ def generate_for_claim(self, claim_id: str, claim_text: str) -> List[Tuple[str, float]]:
806
+ signatures = []
807
+ # entity disappearance / fading
808
+ for entity in self.esl.entities:
809
+ if entity.lower() in claim_text.lower():
810
+ if self.esl.disappearance_suspected(entity):
811
+ signatures.append(("entity_present_then_absent", 0.8))
812
+ timeline = self.esl.get_entity_timeline(entity)
813
+ if len(timeline) >= 2:
814
+ last = datetime.fromisoformat(timeline[-1]["timestamp"].replace('Z', '+00:00'))
815
+ days_since = (datetime.utcnow() - last).days
816
+ if 7 < days_since < 30:
817
+ signatures.append(("gradual_fading", 0.6))
818
+
819
+ try:
820
+ for entity in self.esl.entities:
821
+ if entity.lower() in claim_text.lower():
822
+ emb_timeline = self.esl.get_entity_embeddings(entity)
823
+ if len(emb_timeline) >= 4:
824
+ drift_score = _semantic_drift_score(emb_timeline, window=7)
825
+ if drift_score > 0.25:
826
+ signatures.append(("semantic_drift", min(0.9, 0.35 + drift_score * 0.6)))
827
+ except Exception:
828
+ pass
829
+
830
+ try:
831
+ csig = _crowding_signature(self.esl, window_days=3, dup_threshold=0.6)
832
+ if csig:
833
+ signatures.append(csig)
834
+ except Exception:
835
+ pass
836
+
837
+ try:
838
+ in_sig = _inoculation_signature(self.esl, claim_id, lead_window_days=7, sim_threshold=0.72)
839
+ if in_sig:
840
+ signatures.append(in_sig)
841
+ except Exception:
842
+ pass
843
+
844
+ try:
845
+ wf = self.esl.claims.get(claim_id, {}).get("workflow_events")
846
+ if wf:
847
+ attr = _compute_attrition_score(wf)
848
+ if attr > 0.2:
849
+ signatures.append(("bureaucratic_attrition", min(0.9, 0.2 + attr * 0.8)))
850
+ except Exception:
851
+ pass
852
+
853
+ contradictions = self.esl.contradiction_graph.get(claim_id, set())
854
+ if contradictions:
855
+ signatures.append(("contradictory_claims", 0.7))
856
+
857
+ for entity in self.esl.entities:
858
+ if entity.lower() in claim_text.lower():
859
+ coherence = self.esl.get_entity_coherence(entity)
860
+ if coherence < 0.3:
861
+ signatures.append(("temporal_instability", 0.5))
862
+
863
+ for cid, claim in self.esl.claims.items():
864
+ if cid != claim_id and claim["text"].lower() == claim_text.lower():
865
+ signatures.append(("repetitive_messaging", 0.9))
866
+ break
867
+
868
+ claim_ents = [e for e in self.esl.entities if e.lower() in claim_text.lower()]
869
+ if claim_ents:
870
+ src_types = []
871
+ for ent_name in claim_ents:
872
+ ent = self.esl.entities.get(ent_name)
873
+ if ent and ent.get("source_types"):
874
+ src = max(ent["source_types"].items(), key=lambda x: x[1])[0] if ent["source_types"] else "unknown"
875
+ src_types.append(src)
876
+ if src_types and len(set(src_types)) == 1:
877
+ signatures.append(("source_monoculture", 0.6))
878
+
879
+ single_exp_count = sum(1 for c in self.esl.claims.values() if "single_explanation" in c.get("signatures", []))
880
+ if single_exp_count > 3:
881
+ signatures.append(("narrative_dominance", 0.7))
882
+
883
+ return signatures
884
+
885
+ # ----------------------------------------------------------------------------
886
+ # EPISTEMIC MULTIPLEXOR (Bayesian‑inspired)
887
+ # ----------------------------------------------------------------------------
888
+ class Hypothesis:
889
+ def __init__(self, desc: str):
890
+ self.desc = desc
891
+ self.prob = 0.0
892
+
893
+ class EpistemicMultiplexor:
894
+ def __init__(self, alpha_fast: float = 0.3, alpha_slow: float = 0.05):
895
+ self.hypotheses: List[Hypothesis] = []
896
+ self.alpha_fast = alpha_fast
897
+ self.alpha_slow = alpha_slow
898
+ self.previous_probs: Dict[str, float] = {}
899
+
900
+ def initialize(self, base_hypotheses: List[str]):
901
+ if not base_hypotheses:
902
+ raise ValueError("base_hypotheses must contain at least one hypothesis")
903
+ self.hypotheses = [Hypothesis(h) for h in base_hypotheses]
904
+ equal = 1.0 / len(self.hypotheses)
905
+ for h in self.hypotheses:
906
+ h.prob = equal
907
+ self.previous_probs = {h.desc: h.prob for h in self.hypotheses}
908
+
909
+ def update(self, evidence_strength: float, signatures: List[str], coherence: float,
910
+ hallucination_penalty: float = 1.0):
911
+ likelihood: Dict[str, float] = {}
912
+ for h in self.hypotheses:
913
+ desc = h.desc.lower()
914
+ lik = 0.5
915
+ if "user claim" in desc:
916
+ lik = 0.5 + evidence_strength * coherence
917
+ elif "official narrative" in desc:
918
+ lik = 0.5 - evidence_strength * 0.3
919
+ elif "suppression" in desc:
920
+ erasure_sigs = {"entity_present_then_absent", "archival_gaps", "gradual_fading"}
921
+ if any(sig in signatures for sig in erasure_sigs):
922
+ lik = 0.5 + evidence_strength * 0.6
923
+ else:
924
+ lik = 0.5 - evidence_strength * 0.2
925
+ elif "natural decay" in desc:
926
+ lik = 0.5 + (0.2 if "gradual_fading" in signatures else -0.1)
927
+ elif "noise" in desc:
928
+ lik = 0.5
929
+ likelihood[h.desc] = max(0.05, min(0.95, lik)) * hallucination_penalty
930
+
931
+ posterior_unnorm: Dict[str, float] = {}
932
+ total = 0.0
933
+ for h in self.hypotheses:
934
+ prior = h.prob if h.prob is not None else (1.0 / len(self.hypotheses))
935
+ post = prior * likelihood[h.desc]
936
+ posterior_unnorm[h.desc] = post
937
+ total += post
938
+
939
+ if total <= 0:
940
+ uniform = 1.0 / len(self.hypotheses)
941
+ for h in self.hypotheses:
942
+ old = self.previous_probs.get(h.desc, h.prob)
943
+ smoothed = self.alpha_slow * uniform + (1 - self.alpha_slow) * old
944
+ h.prob = smoothed
945
+ self.previous_probs[h.desc] = h.prob
946
+ return
947
+
948
+ for h in self.hypotheses:
949
+ new_prob = posterior_unnorm[h.desc] / total
950
+ old = self.previous_probs.get(h.desc, h.prob)
951
+ smoothed = self.alpha_slow * new_prob + (1 - self.alpha_slow) * old
952
+ h.prob = smoothed
953
+ self.previous_probs[h.desc] = h.prob
954
+
955
+ def get_probabilities(self) -> Dict[str, float]:
956
+ return {h.desc: h.prob for h in self.hypotheses}
957
+
958
+ # ----------------------------------------------------------------------------
959
+ # ANTI‑HALLUCINATION ENGINE (embedding‑based, structural, feedback)
960
+ # ----------------------------------------------------------------------------
961
+ class AntiHallucinationEngine:
962
+ def __init__(self, esl: ESLedger):
963
+ self.esl = esl
964
+ self.hallucination_patterns = [
965
+ (r"\b(always|never|everyone|no one)\b", "absolutist language without evidence"),
966
+ (r"\b(clearly|obviously|undoubtedly|certainly)\b", "unjustified certainty"),
967
+ (r"\b(conspiracy|cover‑up|suppressed)\b", "speculative accusation – requires evidence"),
968
+ (r"\b(must have|could only have|had to be)\b", "modal speculation masquerading as fact"),
969
+ (r"\b(they|them|those people)\b(?:\s+\w+){0,3}\s+(wanted|intended|planned|ordered)", "unspecified agent attribution"),
970
+ (r"therefore\s+(.+)", "causal leap without evidence"),
971
+ ]
972
+
973
+ def _get_evidence_for_claim(self, claim_text: str, claim_embedding: Optional[List[float]]) -> List[Dict]:
974
+ evidence = []
975
+ if claim_embedding is None:
976
+ return []
977
+ for cid, claim in self.esl.claims.items():
978
+ emb2 = claim.get("embedding")
979
+ if emb2 is None:
980
+ continue
981
+ sim = _cosine_sim(claim_embedding, emb2)
982
+ if sim > 0.65: # embedding similarity threshold
983
+ reliability = 0.5
984
+ if claim.get("suppression_score", 0.0) > 0.6:
985
+ reliability = 0.3
986
+ elif claim.get("suppression_score", 0.0) < 0.2:
987
+ reliability = 0.7
988
+ if claim.get("polarity", 1.0) == 1.0 and claim.get("suppression_score", 0.0) < 0.3:
989
+ reliability = min(0.9, reliability + 0.2)
990
+ evidence.append({
991
+ "text": claim["text"],
992
+ "source_type": claim["agent"],
993
+ "reliability": reliability,
994
+ "similarity": sim,
995
+ "verification_status": "verified" if claim.get("suppression_score", 0.0) < 0.4 else "contradicted"
996
+ })
997
+ return evidence
998
+
999
+ def _detect_hallucination_patterns(self, text: str) -> Tuple[float, List[str]]:
1000
+ reasons = []
1001
+ score = 0.0
1002
+ for pattern, reason in self.hallucination_patterns:
1003
+ if re.search(pattern, text, re.IGNORECASE):
1004
+ reasons.append(reason)
1005
+ score += 0.2
1006
+ return min(1.0, score), reasons
1007
+
1008
+ def _structural_hallucination(self, claim_text: str, claim_embedding: Optional[List[float]],
1009
+ entities: List[Tuple[str, str, bool]]) -> Tuple[float, List[str]]:
1010
+ score = 0.0
1011
+ reasons = []
1012
+ # No entities
1013
+ if not entities:
1014
+ score += 0.4
1015
+ reasons.append("no entities extracted")
1016
+ # Low similarity to existing ledger (novelty)
1017
+ if claim_embedding is not None:
1018
+ max_sim = 0.0
1019
+ for cid, claim in self.esl.claims.items():
1020
+ emb2 = claim.get("embedding")
1021
+ if emb2 is not None:
1022
+ sim = _cosine_sim(claim_embedding, emb2)
1023
+ if sim > max_sim:
1024
+ max_sim = sim
1025
+ if max_sim < 0.4:
1026
+ score += 0.3
1027
+ reasons.append(f"high novelty (max similarity {max_sim:.2f})")
1028
+ # External grounding check: if entities exist, try to fetch Wikipedia summary
1029
+ # If summary is empty or contradictory, increase hallucination score
1030
+ for ent_name, _, _ in entities:
1031
+ ent = self.esl.entities.get(ent_name)
1032
+ if ent and ent.get("external_summary"):
1033
+ # Very simple: if external summary exists, that's a positive sign
1034
+ # But if it contradicts the claim, that would be handled elsewhere
1035
+ pass
1036
+ else:
1037
+ # No external info – not necessarily hallucination, but could be
1038
+ pass
1039
+ return min(1.0, score), reasons
1040
+
1041
+ def evaluate(self, claim_text: str) -> Dict[str, Any]:
1042
+ # Compute embedding
1043
+ emb_arr = _embed_texts([claim_text])
1044
+ claim_embedding = emb_arr[0].tolist() if emb_arr is not None else None
1045
+
1046
+ # Get evidence using embeddings
1047
+ evidence_items = self._get_evidence_for_claim(claim_text, claim_embedding)
1048
+ supporting = [e for e in evidence_items if e["verification_status"] == "verified"]
1049
+ contradicting = [e for e in evidence_items if e["verification_status"] == "contradicted"]
1050
+
1051
+ # Pattern‑based hallucination
1052
+ pattern_score, pattern_reasons = self._detect_hallucination_patterns(claim_text)
1053
+
1054
+ # Structural hallucination
1055
+ entities = extract_entities(claim_text)
1056
+ structural_score, structural_reasons = self._structural_hallucination(claim_text, claim_embedding, entities)
1057
+
1058
+ hallucination_score = min(1.0, pattern_score + structural_score)
1059
+ hallucination_reasons = pattern_reasons + structural_reasons
1060
+
1061
+ # Likelihood calculation (embedding‑weighted)
1062
+ if not supporting and not contradicting:
1063
+ base = 0.1
1064
+ else:
1065
+ support_sum = sum(e["reliability"] * e.get("similarity", 0.5) for e in supporting)
1066
+ contradict_sum = sum(e["reliability"] * e.get("similarity", 0.5) for e in contradicting)
1067
+ total = support_sum + contradict_sum + 1e-6
1068
+ base = support_sum / total
1069
+
1070
+ hallucination_penalty = 1 - (hallucination_score * 0.7)
1071
+ likelihood = base * hallucination_penalty
1072
+ likelihood = min(0.99, max(0.01, likelihood))
1073
+
1074
+ # Origin classification
1075
+ if likelihood >= 0.7 and supporting:
1076
+ origin = "supported"
1077
+ elif hallucination_score >= 0.6:
1078
+ origin = "hallucinated"
1079
+ elif likelihood < 0.3:
1080
+ origin = "hallucinated"
1081
+ elif supporting:
1082
+ origin = "plausible"
1083
+ else:
1084
+ origin = "plausible"
1085
+
1086
+ # Justification
1087
+ parts = []
1088
+ if origin == "supported":
1089
+ parts.append(f"Supported by {len(supporting)} piece(s) of evidence (avg similarity {sum(e['similarity'] for e in supporting)/len(supporting):.2f}).")
1090
+ elif origin == "hallucinated":
1091
+ parts.append("No supporting evidence. Hallucination patterns detected: " + ", ".join(hallucination_reasons))
1092
+ else:
1093
+ parts.append("No direct evidence, but consistent with known facts and not contradicted.")
1094
+ if contradicting:
1095
+ parts.append(f"Contradicted by {len(contradicting)} piece(s) of evidence.")
1096
+ parts.append(f"Likelihood estimate: {likelihood:.2f}")
1097
+ justification = " ".join(parts)
1098
+
1099
+ # Missing evidence
1100
+ missing = []
1101
+ if origin != "supported":
1102
+ missing.append("Direct external source confirming the claim.")
1103
+ if likelihood < 0.5:
1104
+ missing.append("Independent corroboration from multiple sources.")
1105
+ if any(word in claim_text.lower() for word in ["conspiracy", "cover‑up", "suppressed"]):
1106
+ missing.append("Specific documentation or testimony from primary sources.")
1107
+
1108
+ return {
1109
+ "claim": claim_text,
1110
+ "origin": origin,
1111
+ "likelihood": likelihood,
1112
+ "hallucination_score": hallucination_score,
1113
+ "justification": justification,
1114
+ "missing_evidence": missing,
1115
+ "supporting_evidence_count": len(supporting),
1116
+ "contradicting_evidence_count": len(contradicting),
1117
+ "hallucination_flags": hallucination_reasons
1118
+ }
1119
+
1120
+ # ----------------------------------------------------------------------------
1121
+ # NARRATIVE VIOLATION DETECTOR
1122
+ # ----------------------------------------------------------------------------
1123
+ class NarrativeViolationDetector:
1124
+ def __init__(self, esl: ESLedger):
1125
+ self.esl = esl
1126
+ self.narrative_indicators = [
1127
+ "mainstream narrative", "official story", "commonly believed",
1128
+ "consensus view", "widely accepted", "according to sources",
1129
+ "it is known that", "as reported by", "credible institutions"
1130
+ ]
1131
+
1132
+ def check(self, llm_output: str, claim_text: str) -> Tuple[bool, float, str]:
1133
+ output_lower = llm_output.lower()
1134
+ score = 0.0
1135
+ reasons = []
1136
+ for ind in self.narrative_indicators:
1137
+ if ind in output_lower:
1138
+ score += 0.2
1139
+ reasons.append(f"narrative phrase '{ind}'")
1140
+ esl_mentioned = any(entity.lower() in output_lower for entity in self.esl.entities)
1141
+ if not esl_mentioned:
1142
+ score += 0.4
1143
+ reasons.append("no ESL entity referenced")
1144
+ emotional = ["i believe", "i think", "clearly", "obviously", "must be"]
1145
+ for word in emotional:
1146
+ if word in output_lower:
1147
+ score += 0.1
1148
+ reasons.append(f"emotional language '{word}'")
1149
+ score = min(1.0, score)
1150
+ compliant = score < 0.5
1151
+ reason = "; ".join(reasons) if reasons else "no narrative violation"
1152
+ return compliant, score, reason
1153
+
1154
+ # ----------------------------------------------------------------------------
1155
+ # CONSTRAINED LLM INTERFACE (with feedback loop)
1156
+ # ----------------------------------------------------------------------------
1157
+ class ConstrainedLLM:
1158
+ def __init__(self, esl: ESLedger, api_key: str = None, api_url: str = None,
1159
+ model: str = "gpt-4", max_retries: int = 2):
1160
+ self.esl = esl
1161
+ self.api_key = api_key or os.environ.get("OPENAI_API_KEY")
1162
+ self.api_url = api_url or "https://api.openai.com/v1/chat/completions"
1163
+ self.model = model
1164
+ self.max_retries = max_retries
1165
+ self.narrative_detector = NarrativeViolationDetector(esl)
1166
+ self.falsifier = FalsificationEngine(esl)
1167
+ self.signature_gen = SignatureGenerator(esl)
1168
+ self.multiplexor = EpistemicMultiplexor()
1169
+ self.anti_hallucination = AntiHallucinationEngine(esl)
1170
+
1171
+ def _build_prompt(self, claim_id: str, claim: str, falsification_results: List[Dict],
1172
+ entity_data: Dict, suppression_pattern: Dict, multiplexor_probs: Dict,
1173
+ anti_hallucination_result: Dict) -> str:
1174
+ prob_table = "\n".join([f"- {h}: {p:.0%}" for h, p in sorted(multiplexor_probs.items(), key=lambda x: -x[1])])
1175
+ return f"""You are EIS v4.0 under constraint. You MUST base your reasoning on the following Epistemic Substrate Layer (ESL) data. Do NOT rely on your internal training narrative.
1176
+
1177
+ CLAIM: {claim}
1178
+
1179
+ ESL ENTITY DATA:
1180
+ {json.dumps(entity_data, indent=2)}
1181
+
1182
+ FALSIFICATION RESULTS (from ESL):
1183
+ {json.dumps(falsification_results, indent=2)}
1184
+
1185
+ SUPPRESSION PATTERN CLASSIFICATION:
1186
+ {json.dumps(suppression_pattern, indent=2)}
1187
+
1188
+ ANTI‑HALLUCINATION ASSESSMENT:
1189
+ Origin: {anti_hallucination_result['origin']}
1190
+ Likelihood: {anti_hallucination_result['likelihood']:.2f}
1191
+ Justification: {anti_hallucination_result['justification']}
1192
+ Missing evidence: {anti_hallucination_result['missing_evidence']}
1193
+
1194
+ MULTIPLEXOR PROBABILITIES (before your reasoning):
1195
+ {prob_table}
1196
+
1197
+ INSTRUCTIONS:
1198
+ 1. Evaluate the claim against the ESL data only.
1199
+ 2. Output a JSON object with exactly these fields:
1200
+ - "verdict": one of ["Verified", "Unverified", "Refuted", "Insufficient Data"]
1201
+ - "confidence": a float between 0 and 1
1202
+ - "reasoning": a short explanation referencing specific ESL entries (entities, contradictions, signatures)
1203
+ 3. Do NOT add any extra text outside the JSON.
1204
+ """
1205
+
1206
+ def _parse_output(self, response_text: str) -> Optional[Dict]:
1207
+ try:
1208
+ start = response_text.find('{')
1209
+ end = response_text.rfind('}') + 1
1210
+ if start == -1 or end == 0:
1211
+ return None
1212
+ json_str = response_text[start:end]
1213
+ return json.loads(json_str)
1214
+ except Exception:
1215
+ return None
1216
+
1217
+ def _check_constraints(self, output: Dict, claim: str, falsification_results: List[Dict]) -> bool:
1218
+ if not all(k in output for k in ["verdict", "confidence", "reasoning"]):
1219
+ return False
1220
+ if not (0 <= output["confidence"] <= 1):
1221
+ return False
1222
+ if output["verdict"] not in ["Verified", "Unverified", "Refuted", "Insufficient Data"]:
1223
+ return False
1224
+ reasoning = output["reasoning"].lower()
1225
+ esl_mentioned = any(
1226
+ ent.lower() in reasoning for ent in self.esl.entities
1227
+ ) or any(
1228
+ test["name"].lower() in reasoning for test in falsification_results
1229
+ )
1230
+ return esl_mentioned
1231
+
1232
+ def query(self, claim_text: str, agent: str = "user") -> Dict:
1233
+ claim_id = self.esl.add_claim(claim_text, agent)
1234
+ # Compute embedding for the new claim
1235
+ emb_arr = _embed_texts([claim_text])
1236
+ claim_embedding = emb_arr[0].tolist() if emb_arr is not None else None
1237
+
1238
+ # Semantic contradictions (using embedding)
1239
+ for cid in self.esl.find_contradictions(claim_text, claim_embedding):
1240
+ self.esl.add_contradiction(claim_id, cid)
1241
+
1242
+ # Entities
1243
+ entities = extract_entities(claim_text)
1244
+ for ent_name, ent_type, negated in entities:
1245
+ source_type = "official" if ent_type in ["ORG", "GPE", "PERSON"] else "media" if ent_type in ["EVENT", "PRODUCT"] else "user"
1246
+ self.esl.add_entity(ent_name, ent_type, claim_id, negated, source_type)
1247
+
1248
+ # Signatures
1249
+ signatures = self.signature_gen.generate_for_claim(claim_id, claim_text)
1250
+ for sig_name, weight in signatures:
1251
+ self.esl.add_signature(claim_id, sig_name, weight)
1252
+
1253
+ # Falsification
1254
+ falsification_results = self.falsifier.run_all(claim_id, claim_text, agent)
1255
+
1256
+ # Entity data for prompt
1257
+ entity_data = {}
1258
+ for ent_name, _, _ in entities:
1259
+ ent = self.esl.entities.get(ent_name)
1260
+ if ent:
1261
+ entity_data[ent_name] = {
1262
+ "type": ent["type"],
1263
+ "first_seen": ent["first_seen"],
1264
+ "last_seen": ent["last_seen"],
1265
+ "coherence": self.esl.get_entity_coherence(ent_name),
1266
+ "suppression_score": ent.get("suppression_score", 0.0)
1267
+ }
1268
+
1269
+ suppression_pattern = self.esl.suppression_pattern_classifier(claim_id)
1270
+
1271
+ # Anti‑hallucination evaluation
1272
+ anti_hallucination_result = self.anti_hallucination.evaluate(claim_text)
1273
+ hallucination_penalty = 1 - anti_hallucination_result.get("hallucination_score", 0.0)
1274
+
1275
+ # Multiplexor with feedback loop
1276
+ base_hypotheses = [
1277
+ f"User claim: {claim_text}",
1278
+ "Official narrative accurate",
1279
+ "Suppression detected",
1280
+ "Natural decay",
1281
+ "Noise / randomness"
1282
+ ]
1283
+ self.multiplexor.initialize(base_hypotheses)
1284
+ evidence_strength = len(signatures) / 5.0
1285
+ coherence = sum(self.esl.get_entity_coherence(e) for e, _, _ in entities) / max(1, len(entities))
1286
+ signature_names = [s[0] for s in signatures]
1287
+ self.multiplexor.update(evidence_strength, signature_names, coherence, hallucination_penalty)
1288
+ multiplexor_probs = self.multiplexor.get_probabilities()
1289
+ user_prob = multiplexor_probs.get(f"User claim: {claim_text}", 0.0)
1290
+
1291
+ # LLM optional
1292
+ llm_output = None
1293
+ if self.api_key:
1294
+ prompt = self._build_prompt(claim_id, claim_text, falsification_results,
1295
+ entity_data, suppression_pattern, multiplexor_probs,
1296
+ anti_hallucination_result)
1297
+ headers = {"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"}
1298
+ payload = {"model": self.model, "messages": [{"role": "user", "content": prompt}], "temperature": 0.2}
1299
+ for attempt in range(self.max_retries + 1):
1300
+ try:
1301
+ resp = requests.post(self.api_url, headers=headers, json=payload, timeout=30)
1302
+ if resp.status_code != 200:
1303
+ raise Exception(f"API error: {resp.text}")
1304
+ result = resp.json()
1305
+ content = result["choices"][0]["message"]["content"]
1306
+ output = self._parse_output(content)
1307
+ if output and self._check_constraints(output, claim_text, falsification_results):
1308
+ compliant, n_score, n_reason = self.narrative_detector.check(content, claim_text)
1309
+ if compliant:
1310
+ llm_output = output
1311
+ break
1312
+ except Exception:
1313
+ time.sleep(1)
1314
+
1315
+ # Decomposed final confidence
1316
+ survival_score = sum(1 for t in falsification_results if t["survived"]) / len(falsification_results)
1317
+ evidence_strength_component = evidence_strength
1318
+ contradiction_penalty = 1 - (len(self.esl.contradiction_graph.get(claim_id, set())) / 10.0)
1319
+ suppression_bias = 1 - suppression_pattern["score"] # higher suppression reduces confidence
1320
+ hallucination_component = 1 - anti_hallucination_result.get("hallucination_score", 0.0)
1321
+ final_confidence = (evidence_strength_component * 0.3 +
1322
+ survival_score * 0.2 +
1323
+ contradiction_penalty * 0.2 +
1324
+ suppression_bias * 0.15 +
1325
+ hallucination_component * 0.15) * user_prob
1326
+ final_confidence = min(0.99, max(0.01, final_confidence))
1327
+
1328
+ if final_confidence > 0.7:
1329
+ verdict = "Verified"
1330
+ elif final_confidence > 0.4:
1331
+ verdict = "Unverified"
1332
+ elif survival_score < 0.3:
1333
+ verdict = "Refuted"
1334
+ else:
1335
+ verdict = "Insufficient Data"
1336
+
1337
+ self.esl.decay_confidence(half_life_days=30)
1338
+ self.esl.create_block()
1339
+ trend = self.esl.get_suppression_trend(window_days=30)
1340
+ entity_analytics = [self.esl.get_entity_suppression(e) for e, _, _ in entities]
1341
+
1342
+ result_dict = {
1343
+ "claim_id": claim_id,
1344
+ "verdict": verdict,
1345
+ "confidence": final_confidence,
1346
+ "falsification": falsification_results,
1347
+ "suppression_pattern": suppression_pattern,
1348
+ "multiplexor_probabilities": multiplexor_probs,
1349
+ "suppression_trend": trend,
1350
+ "entity_analytics": entity_analytics,
1351
+ "narrative_compliance": True,
1352
+ "anti_hallucination": anti_hallucination_result,
1353
+ "confidence_decomposition": {
1354
+ "evidence_strength": evidence_strength_component,
1355
+ "survival_score": survival_score,
1356
+ "contradiction_penalty": contradiction_penalty,
1357
+ "suppression_bias": suppression_bias,
1358
+ "hallucination_penalty": hallucination_component,
1359
+ "user_prior": user_prob
1360
+ }
1361
+ }
1362
+ if llm_output:
1363
+ result_dict["llm_verdict"] = llm_output["verdict"]
1364
+ result_dict["llm_confidence"] = llm_output["confidence"]
1365
+ result_dict["reasoning"] = llm_output["reasoning"]
1366
+ else:
1367
+ result_dict["reasoning"] = "LLM not used or failed constraints; verdict based on EIS multiplexor."
1368
+ return result_dict
1369
+
1370
+ # ----------------------------------------------------------------------------
1371
+ # OUTPUT FORMATTER (includes decomposition)
1372
+ # ----------------------------------------------------------------------------
1373
+ def format_report(result: Dict) -> str:
1374
+ lines = []
1375
+ lines.append("**Falsification Results**")
1376
+ for test in result["falsification"]:
1377
+ emoji = "✅" if test["survived"] else "❌"
1378
+ lines.append(f"- {test['name']}: {emoji} – {test['reason']}")
1379
+ lines.append("\n**Hypothesis Probabilities**")
1380
+ lines.append("| Hypothesis | Probability |")
1381
+ lines.append("|------------|-------------|")
1382
+ for h, p in sorted(result["multiplexor_probabilities"].items(), key=lambda x: -x[1]):
1383
+ lines.append(f"| {h} | {p:.0%} |")
1384
+ lines.append(f"\n**Final Confidence:** {result['confidence']:.2f}")
1385
+ lines.append(f"**Verdict:** {result['verdict']}")
1386
+
1387
+ # Confidence decomposition
1388
+ cd = result.get("confidence_decomposition", {})
1389
+ if cd:
1390
+ lines.append("\n**Confidence Decomposition**")
1391
+ lines.append(f" - Evidence strength: {cd.get('evidence_strength', 0.0):.2f}")
1392
+ lines.append(f" - Survival score: {cd.get('survival_score', 0.0):.2f}")
1393
+ lines.append(f" - Contradiction penalty: {cd.get('contradiction_penalty', 0.0):.2f}")
1394
+ lines.append(f" - Suppression bias: {cd.get('suppression_bias', 0.0):.2f}")
1395
+ lines.append(f" - Hallucination penalty: {cd.get('hallucination_penalty', 0.0):.2f}")
1396
+ lines.append(f" - User prior: {cd.get('user_prior', 0.0):.2f}")
1397
+
1398
+ # Anti‑hallucination section
1399
+ ah = result.get("anti_hallucination", {})
1400
+ if ah:
1401
+ lines.append("\n**Anti‑Hallucination Assessment**")
1402
+ lines.append(f" - Origin: {ah.get('origin', 'unknown')}")
1403
+ lines.append(f" - Likelihood: {ah.get('likelihood', 0.0):.2f}")
1404
+ lines.append(f" - Justification: {ah.get('justification', '')}")
1405
+ if ah.get("missing_evidence"):
1406
+ lines.append(f" - Missing evidence: {', '.join(ah['missing_evidence'])}")
1407
+ if ah.get("hallucination_flags"):
1408
+ lines.append(f" - Hallucination flags: {', '.join(ah['hallucination_flags'])}")
1409
+
1410
+ sp = result["suppression_pattern"]
1411
+ lens_names = [get_lens_name(lid) for lid in sp.get("lenses", [])]
1412
+ lines.append(f"\n**Suppression Pattern:** level={sp['level']}, score={sp['score']:.2f}")
1413
+ if lens_names:
1414
+ lines.append(f" - Lenses: {', '.join(lens_names[:5])}" + (" …" if len(lens_names)>5 else ""))
1415
+ if sp.get("primitives"):
1416
+ lines.append(f" - Primitives: {', '.join(sp['primitives'])}")
1417
+ if sp.get("contributions"):
1418
+ lines.append(" - Signature contributions:")
1419
+ for sig, w in sorted(sp["contributions"].items(), key=lambda x: -x[1]):
1420
+ lines.append(f" {sig}: {w:.2f}")
1421
+
1422
+ trend = result.get("suppression_trend", [])
1423
+ if trend:
1424
+ lines.append("\n**Suppression Trend (last 30 days)**")
1425
+ for point in trend[-7:]:
1426
+ lines.append(f" - {point['date']}: {point['avg_suppression']:.2f}")
1427
+
1428
+ entity_analytics = result.get("entity_analytics", [])
1429
+ if entity_analytics:
1430
+ lines.append("\n**Entity Suppression Analytics**")
1431
+ for ent in entity_analytics:
1432
+ src_str = ", ".join([f"{k}:{v}" for k,v in ent.get("source_types", {}).items()]) if ent.get("source_types") else "unknown"
1433
+ lines.append(f" - {ent['name']} ({ent['type']}): score={ent['score']:.2f}, coherence={ent['coherence']:.2f}, appearances={ent['appearance_count']}, negated={ent.get('negated_count',0)}, sources={src_str}")
1434
+
1435
+ if "llm_verdict" in result:
1436
+ lines.append(f"\n*LLM raw verdict: {result['llm_verdict']} (confidence {result['llm_confidence']:.2f})*")
1437
+ return "\n".join(lines)
1438
+
1439
+ # ----------------------------------------------------------------------------
1440
+ # MAIN
1441
+ # ----------------------------------------------------------------------------
1442
+ def main():
1443
+ print("EIS + ESL Mediator v4.0 – Grounded Epistemic Engine")
1444
+ print("=" * 80)
1445
+ esl = ESLedger()
1446
+ llm = ConstrainedLLM(esl, api_key=os.environ.get("OPENAI_API_KEY"), model="gpt-4")
1447
+
1448
+ print("\nEnter a claim (or 'quit'):")
1449
+ while True:
1450
+ claim = input("> ").strip()
1451
+ if claim.lower() in ("quit", "exit"):
1452
+ break
1453
+ if not claim:
1454
+ continue
1455
+ print("Processing claim...")
1456
+ result = llm.query(claim)
1457
+ print("\n" + format_report(result))
1458
+ print("-" * 80)
1459
+
1460
+ if __name__ == "__main__":
1461
+ main() return True
1462
+ if antonym in a_words and word in b_words:
1463
+ return True
1464
+ return False
1465
+
1466
+ # ----------------------------------------------------------------------------
1467
+ # FALSIFICATION ENGINE
1468
+ # ----------------------------------------------------------------------------
1469
+ class FalsificationEngine:
1470
+ def __init__(self, esl: ESLedger):
1471
+ self.esl = esl
1472
+
1473
+ def alternative_cause(self, claim_text: str) -> Tuple[bool, str]:
1474
+ if has_negation(claim_text):
1475
+ return True, "Claim is negated; alternative cause not applicable."
1476
+ for entity in self.esl.entities:
1477
+ if entity.lower() in claim_text.lower():
1478
+ if self.esl.disappearance_suspected(entity):
1479
+ return False, f"Entity '{entity}' disappearance may be natural (no recent activity)."
1480
+ return True, "No obvious alternative cause."
1481
+
1482
+ def contradictory_evidence(self, claim_id: str) -> Tuple[bool, str]:
1483
+ contradictions = self.esl.contradiction_graph.get(claim_id, set())
1484
+ if contradictions:
1485
+ return False, f"Claim contradicts {len(contradictions)} existing claim(s)."
1486
+ return True, "No direct contradictions."
1487
+
1488
+ def source_diversity(self, claim_text: str) -> Tuple[bool, str]:
1489
+ entities_in_claim = [e for e in self.esl.entities if e.lower() in claim_text.lower()]
1490
+ if len(entities_in_claim) <= 1:
1491
+ return False, f"Claim relies on only {len(entities_in_claim)} entity/entities."
1492
+ return True, f"Multiple entities ({len(entities_in_claim)}) involved."
1493
+
1494
+ def temporal_stability(self, claim_text: str) -> Tuple[bool, str]:
1495
+ for entity in self.esl.entities:
1496
+ if entity.lower() in claim_text.lower():
1497
+ coherence = self.esl.get_entity_coherence(entity)
1498
+ if coherence < 0.3:
1499
+ return False, f"Entity '{entity}' has low temporal coherence ({coherence:.2f})."
1500
+ return True, "Temporal coherence adequate."
1501
+
1502
+ def manipulation_check(self, claim_text: str, agent: str) -> Tuple[bool, str]:
1503
+ manip_indicators = ["must", "cannot", "obviously", "clearly", "everyone knows"]
1504
+ for word in manip_indicators:
1505
+ if word in claim_text.lower():
1506
+ return False, f"Manipulative language detected: '{word}'."
1507
+ return True, "No manipulation indicators."
1508
+
1509
+ def run_all(self, claim_id: str, claim_text: str, agent: str) -> List[Dict]:
1510
+ tests = [
1511
+ ("alternative_cause", lambda: self.alternative_cause(claim_text)),
1512
+ ("contradictory_evidence", lambda: self.contradictory_evidence(claim_id)),
1513
+ ("source_diversity", lambda: self.source_diversity(claim_text)),
1514
+ ("temporal_stability", lambda: self.temporal_stability(claim_text)),
1515
+ ("manipulation_check", lambda: self.manipulation_check(claim_text, agent))
1516
+ ]
1517
+ results = []
1518
+ for name, func in tests:
1519
+ survived, reason = func()
1520
+ results.append({"name": name, "survived": survived, "reason": reason})
1521
+ return results
1522
+
1523
+ # ----------------------------------------------------------------------------
1524
+ # SIGNATURE GENERATOR (with advanced detectors)
1525
+ # ----------------------------------------------------------------------------
1526
+ class SignatureGenerator:
1527
+ def __init__(self, esl: ESLedger):
1528
+ self.esl = esl
1529
+
1530
+ def generate_for_claim(self, claim_id: str, claim_text: str) -> List[Tuple[str, float]]:
1531
+ signatures = []
1532
+ # entity disappearance / fading
1533
+ for entity in self.esl.entities:
1534
+ if entity.lower() in claim_text.lower():
1535
+ if self.esl.disappearance_suspected(entity):
1536
+ signatures.append(("entity_present_then_absent", 0.8))
1537
+ timeline = self.esl.get_entity_timeline(entity)
1538
+ if len(timeline) >= 2:
1539
+ last = datetime.fromisoformat(timeline[-1]["timestamp"].replace('Z', '+00:00'))
1540
+ days_since = (datetime.utcnow() - last).days
1541
+ if 7 < days_since < 30:
1542
+ signatures.append(("gradual_fading", 0.6))
1543
+
1544
+ try:
1545
+ for entity in self.esl.entities:
1546
+ if entity.lower() in claim_text.lower():
1547
+ emb_timeline = self.esl.get_entity_embeddings(entity)
1548
+ if len(emb_timeline) >= 4:
1549
+ drift_score = _semantic_drift_score(emb_timeline, window=7)
1550
+ if drift_score > 0.25:
1551
+ signatures.append(("semantic_drift", min(0.9, 0.35 + drift_score * 0.6)))
1552
+ except Exception:
1553
+ pass
1554
+
1555
+ try:
1556
+ csig = _crowding_signature(self.esl, window_days=3, dup_threshold=0.6)
1557
+ if csig:
1558
+ signatures.append(csig)
1559
+ except Exception:
1560
+ pass
1561
+
1562
+ try:
1563
+ in_sig = _inoculation_signature(self.esl, claim_id, lead_window_days=7, sim_threshold=0.72)
1564
+ if in_sig:
1565
+ signatures.append(in_sig)
1566
+ except Exception:
1567
+ pass
1568
+
1569
+ try:
1570
+ wf = self.esl.claims.get(claim_id, {}).get("workflow_events")
1571
+ if wf:
1572
+ attr = _compute_attrition_score(wf)
1573
+ if attr > 0.2:
1574
+ signatures.append(("bureaucratic_attrition", min(0.9, 0.2 + attr * 0.8)))
1575
+ except Exception:
1576
+ pass
1577
+
1578
+ contradictions = self.esl.contradiction_graph.get(claim_id, set())
1579
+ if contradictions:
1580
+ signatures.append(("contradictory_claims", 0.7))
1581
+
1582
+ for entity in self.esl.entities:
1583
+ if entity.lower() in claim_text.lower():
1584
+ coherence = self.esl.get_entity_coherence(entity)
1585
+ if coherence < 0.3:
1586
+ signatures.append(("temporal_instability", 0.5))
1587
+
1588
+ for cid, claim in self.esl.claims.items():
1589
+ if cid != claim_id and claim["text"].lower() == claim_text.lower():
1590
+ signatures.append(("repetitive_messaging", 0.9))
1591
+ break
1592
+
1593
+ claim_ents = [e for e in self.esl.entities if e.lower() in claim_text.lower()]
1594
+ if claim_ents:
1595
+ src_types = []
1596
+ for ent_name in claim_ents:
1597
+ ent = self.esl.entities.get(ent_name)
1598
+ if ent and ent.get("source_types"):
1599
+ src = max(ent["source_types"].items(), key=lambda x: x[1])[0] if ent["source_types"] else "unknown"
1600
+ src_types.append(src)
1601
+ if src_types and len(set(src_types)) == 1:
1602
+ signatures.append(("source_monoculture", 0.6))
1603
+
1604
+ single_exp_count = sum(1 for c in self.esl.claims.values() if "single_explanation" in c.get("signatures", []))
1605
+ if single_exp_count > 3:
1606
+ signatures.append(("narrative_dominance", 0.7))
1607
+
1608
+ return signatures
1609
+
1610
+ # ----------------------------------------------------------------------------
1611
+ # EPISTEMIC MULTIPLEXOR (fixed smoothing)
1612
+ # ----------------------------------------------------------------------------
1613
+ class Hypothesis:
1614
+ def __init__(self, desc: str):
1615
+ self.desc = desc
1616
+ self.prob = 0.0
1617
+
1618
+ class EpistemicMultiplexor:
1619
+ def __init__(self, alpha_fast: float = 0.3, alpha_slow: float = 0.05):
1620
+ self.hypotheses: List[Hypothesis] = []
1621
+ self.alpha_fast = alpha_fast
1622
+ self.alpha_slow = alpha_slow
1623
+ self.previous_probs: Dict[str, float] = {}
1624
+
1625
+ def initialize(self, base_hypotheses: List[str]):
1626
+ if not base_hypotheses:
1627
+ raise ValueError("base_hypotheses must contain at least one hypothesis")
1628
+ self.hypotheses = [Hypothesis(h) for h in base_hypotheses]
1629
+ equal = 1.0 / len(self.hypotheses)
1630
+ for h in self.hypotheses:
1631
+ h.prob = equal
1632
+ self.previous_probs = {h.desc: h.prob for h in self.hypotheses}
1633
+
1634
+ def update(self, evidence_strength: float, signatures: List[str], coherence: float):
1635
+ likelihood: Dict[str, float] = {}
1636
+ for h in self.hypotheses:
1637
+ desc = h.desc.lower()
1638
+ lik = 0.5
1639
+ if "user claim" in desc:
1640
+ lik = 0.5 + evidence_strength * coherence
1641
+ elif "official narrative" in desc:
1642
+ lik = 0.5 - evidence_strength * 0.3
1643
+ elif "suppression" in desc:
1644
+ erasure_sigs = {"entity_present_then_absent", "archival_gaps", "gradual_fading"}
1645
+ if any(sig in signatures for sig in erasure_sigs):
1646
+ lik = 0.5 + evidence_strength * 0.6
1647
+ else:
1648
+ lik = 0.5 - evidence_strength * 0.2
1649
+ elif "natural decay" in desc:
1650
+ lik = 0.5 + (0.2 if "gradual_fading" in signatures else -0.1)
1651
+ elif "noise" in desc:
1652
+ lik = 0.5
1653
+ likelihood[h.desc] = max(0.05, min(0.95, lik))
1654
+
1655
+ posterior_unnorm: Dict[str, float] = {}
1656
+ total = 0.0
1657
+ for h in self.hypotheses:
1658
+ prior = h.prob if h.prob is not None else (1.0 / len(self.hypotheses))
1659
+ post = prior * likelihood[h.desc]
1660
+ posterior_unnorm[h.desc] = post
1661
+ total += post
1662
+
1663
+ if total <= 0:
1664
+ uniform = 1.0 / len(self.hypotheses)
1665
+ for h in self.hypotheses:
1666
+ old = self.previous_probs.get(h.desc, h.prob)
1667
+ smoothed = self.alpha_slow * uniform + (1 - self.alpha_slow) * old
1668
+ h.prob = smoothed
1669
+ self.previous_probs[h.desc] = h.prob
1670
+ return
1671
+
1672
+ for h in self.hypotheses:
1673
+ new_prob = posterior_unnorm[h.desc] / total
1674
+ old = self.previous_probs.get(h.desc, h.prob)
1675
+ smoothed = self.alpha_slow * new_prob + (1 - self.alpha_slow) * old
1676
+ h.prob = smoothed
1677
+ self.previous_probs[h.desc] = h.prob
1678
+
1679
+ def get_probabilities(self) -> Dict[str, float]:
1680
+ return {h.desc: h.prob for h in self.hypotheses}
1681
+
1682
+ # ----------------------------------------------------------------------------
1683
+ # ANTI‑HALLUCINATION ENGINE (integrated with ESL)
1684
+ # ----------------------------------------------------------------------------
1685
+ class AntiHallucinationEngine:
1686
+ def __init__(self, esl: ESLedger):
1687
+ self.esl = esl
1688
+ self.hallucination_patterns = [
1689
+ (r"\b(always|never|everyone|no one)\b", "absolutist language without evidence"),
1690
+ (r"\b(clearly|obviously|undoubtedly|certainly)\b", "unjustified certainty"),
1691
+ (r"\b(conspiracy|cover‑up|suppressed)\b", "speculative accusation – requires evidence"),
1692
+ (r"\b(must have|could only have|had to be)\b", "modal speculation masquerading as fact"),
1693
+ (r"\b(they|them|those people)\b(?:\s+\w+){0,3}\s+(wanted|intended|planned|ordered)", "unspecified agent attribution"),
1694
+ (r"therefore\s+(.+)", "causal leap without evidence"),
1695
+ ]
1696
+
1697
+ def _get_evidence_for_claim(self, claim_text: str) -> List[Dict]:
1698
+ """Convert ESL claims into evidence items for the anti‑hallucination evaluation."""
1699
+ evidence = []
1700
+ for cid, claim in self.esl.claims.items():
1701
+ # Skip the claim itself? We'll include all others.
1702
+ # Simple relevance: keyword overlap
1703
+ if any(word in claim_text.lower() for word in claim["text"].lower().split()[:5]):
1704
+ reliability = 0.5
1705
+ # Adjust reliability based on suppression score (higher suppression -> less reliable)
1706
+ if claim.get("suppression_score", 0.0) > 0.6:
1707
+ reliability = 0.3
1708
+ elif claim.get("suppression_score", 0.0) < 0.2:
1709
+ reliability = 0.7
1710
+ # If the claim has high polarity (1.0) and low suppression, treat as more reliable
1711
+ if claim.get("polarity", 1.0) == 1.0 and claim.get("suppression_score", 0.0) < 0.3:
1712
+ reliability = min(0.9, reliability + 0.2)
1713
+ evidence.append({
1714
+ "text": claim["text"],
1715
+ "source_type": claim["agent"],
1716
+ "reliability": reliability,
1717
+ "verification_status": "verified" if claim.get("suppression_score", 0.0) < 0.4 else "contradicted"
1718
+ })
1719
+ return evidence
1720
+
1721
+ def _supports(self, claim: str, evidence_text: str) -> bool:
1722
+ claim_lower = claim.lower()
1723
+ evidence_lower = evidence_text.lower()
1724
+ if claim_lower in evidence_lower or evidence_lower in claim_lower:
1725
+ return True
1726
+ claim_entities = set(re.findall(r'\b[A-Z][a-z]+\b', claim))
1727
+ evidence_entities = set(re.findall(r'\b[A-Z][a-z]+\b', evidence_text))
1728
+ if claim_entities and evidence_entities and len(claim_entities.intersection(evidence_entities)) > 0:
1729
+ return True
1730
+ return False
1731
+
1732
+ def _contradicts(self, claim: str, evidence_text: str) -> bool:
1733
+ claim_lower = claim.lower()
1734
+ evidence_lower = evidence_text.lower()
1735
+ negations = ["not", "never", "no", "false", "incorrect", "refuted", "debunked"]
1736
+ if any(neg in evidence_lower for neg in negations) and any(word in evidence_lower for word in claim_lower.split()[:3]):
1737
+ return True
1738
+ return False
1739
+
1740
+ def _detect_hallucination_patterns(self, text: str) -> Tuple[float, List[str]]:
1741
+ reasons = []
1742
+ score = 0.0
1743
+ for pattern, reason in self.hallucination_patterns:
1744
+ if re.search(pattern, text, re.IGNORECASE):
1745
+ reasons.append(reason)
1746
+ score += 0.2
1747
+ return min(1.0, score), reasons
1748
+
1749
+ def _can_be_inferred(self, claim: str, evidence: List[Dict]) -> Tuple[bool, List[str]]:
1750
+ claim_entities = set(re.findall(r'\b[A-Z][a-z]+\b', claim))
1751
+ if not claim_entities:
1752
+ return False, []
1753
+ evidence_entities = set()
1754
+ for e in evidence:
1755
+ evidence_entities.update(re.findall(r'\b[A-Z][a-z]+\b', e["text"]))
1756
+ if claim_entities.issubset(evidence_entities):
1757
+ return True, ["All entities in claim appear in existing evidence"]
1758
+ return False, []
1759
+
1760
+ def evaluate(self, claim: str) -> Dict[str, Any]:
1761
+ evidence_items = self._get_evidence_for_claim(claim)
1762
+ supporting = [e for e in evidence_items if e["verification_status"] == "verified" and self._supports(claim, e["text"])]
1763
+ contradicting = [e for e in evidence_items if e["verification_status"] == "verified" and self._contradicts(claim, e["text"])]
1764
+
1765
+ hallucination_score, hallucination_reasons = self._detect_hallucination_patterns(claim)
1766
+ inferred, inference_chain = self._can_be_inferred(claim, evidence_items)
1767
+
1768
+ # Likelihood calculation
1769
+ if not supporting and not contradicting:
1770
+ base = 0.1
1771
+ else:
1772
+ support_sum = sum(e["reliability"] for e in supporting)
1773
+ contradict_sum = sum(e["reliability"] for e in contradicting)
1774
+ total = support_sum + contradict_sum + 1e-6
1775
+ base = support_sum / total
1776
+ hallucination_penalty = 1 - (hallucination_score * 0.7)
1777
+ inference_boost = 1.2 if inferred else 1.0
1778
+ likelihood = base * hallucination_penalty * inference_boost
1779
+ likelihood = min(0.99, max(0.01, likelihood))
1780
+
1781
+ # Origin
1782
+ if likelihood >= 0.7 and supporting:
1783
+ origin = "supported"
1784
+ elif inferred and likelihood >= 0.5:
1785
+ origin = "inferred"
1786
+ elif hallucination_score >= 0.6:
1787
+ origin = "hallucinated"
1788
+ elif likelihood < 0.3:
1789
+ origin = "hallucinated"
1790
+ else:
1791
+ origin = "plausible"
1792
+
1793
+ # Justification
1794
+ parts = []
1795
+ if origin == "supported":
1796
+ parts.append(f"Supported by {len(supporting)} piece(s) of evidence with reliability scores: {[e['reliability'] for e in supporting]}.")
1797
+ elif origin == "inferred":
1798
+ parts.append(f"Inferred from existing evidence: {'; '.join(inference_chain)}.")
1799
+ elif origin == "plausible":
1800
+ parts.append("No direct evidence, but consistent with known facts and not contradicted.")
1801
+ elif origin == "hallucinated":
1802
+ parts.append("No supporting evidence. Hallucination patterns detected: " + ", ".join(hallucination_reasons))
1803
+ if contradicting:
1804
+ parts.append(f"Contradicted by {len(contradicting)} piece(s) of evidence.")
1805
+ parts.append(f"Likelihood estimate: {likelihood:.2f}")
1806
+ justification = " ".join(parts)
1807
+
1808
+ # Missing evidence
1809
+ missing = []
1810
+ if origin != "supported":
1811
+ missing.append("Direct external source confirming the claim.")
1812
+ if likelihood < 0.5:
1813
+ missing.append("Independent corroboration from multiple sources.")
1814
+ if any(word in claim.lower() for word in ["conspiracy", "cover‑up", "suppressed"]):
1815
+ missing.append("Specific documentation or testimony from primary sources.")
1816
+
1817
+ return {
1818
+ "claim": claim,
1819
+ "origin": origin,
1820
+ "likelihood": likelihood,
1821
+ "justification": justification,
1822
+ "missing_evidence": missing,
1823
+ "supporting_evidence_count": len(supporting),
1824
+ "contradicting_evidence_count": len(contradicting),
1825
+ "hallucination_flags": hallucination_reasons,
1826
+ "inference_chain": inference_chain if inferred else []
1827
+ }
1828
+
1829
+ # ----------------------------------------------------------------------------
1830
+ # NARRATIVE VIOLATION DETECTOR (unchanged)
1831
+ # ----------------------------------------------------------------------------
1832
+ class NarrativeViolationDetector:
1833
+ def __init__(self, esl: ESLedger):
1834
+ self.esl = esl
1835
+ self.narrative_indicators = [
1836
+ "mainstream narrative", "official story", "commonly believed",
1837
+ "consensus view", "widely accepted", "according to sources",
1838
+ "it is known that", "as reported by", "credible institutions"
1839
+ ]
1840
+
1841
+ def check(self, llm_output: str, claim_text: str) -> Tuple[bool, float, str]:
1842
+ output_lower = llm_output.lower()
1843
+ score = 0.0
1844
+ reasons = []
1845
+ for ind in self.narrative_indicators:
1846
+ if ind in output_lower:
1847
+ score += 0.2
1848
+ reasons.append(f"narrative phrase '{ind}'")
1849
+ esl_mentioned = any(entity.lower() in output_lower for entity in self.esl.entities)
1850
+ if not esl_mentioned:
1851
+ score += 0.4
1852
+ reasons.append("no ESL entity referenced")
1853
+ emotional = ["i believe", "i think", "clearly", "obviously", "must be"]
1854
+ for word in emotional:
1855
+ if word in output_lower:
1856
+ score += 0.1
1857
+ reasons.append(f"emotional language '{word}'")
1858
+ score = min(1.0, score)
1859
+ compliant = score < 0.5
1860
+ reason = "; ".join(reasons) if reasons else "no narrative violation"
1861
+ return compliant, score, reason
1862
+
1863
+ # ----------------------------------------------------------------------------
1864
+ # CONSTRAINED LLM INTERFACE (integrated with anti‑hallucination)
1865
+ # ----------------------------------------------------------------------------
1866
+ class ConstrainedLLM:
1867
+ def __init__(self, esl: ESLedger, api_key: str = None, api_url: str = None,
1868
+ model: str = "gpt-4", max_retries: int = 2):
1869
+ self.esl = esl
1870
+ self.api_key = api_key or os.environ.get("OPENAI_API_KEY")
1871
+ self.api_url = api_url or "https://api.openai.com/v1/chat/completions"
1872
+ self.model = model
1873
+ self.max_retries = max_retries
1874
+ self.narrative_detector = NarrativeViolationDetector(esl)
1875
+ self.falsifier = FalsificationEngine(esl)
1876
+ self.signature_gen = SignatureGenerator(esl)
1877
+ self.multiplexor = EpistemicMultiplexor()
1878
+ self.anti_hallucination = AntiHallucinationEngine(esl)
1879
+
1880
+ def _build_prompt(self, claim_id: str, claim: str, falsification_results: List[Dict],
1881
+ entity_data: Dict, suppression_pattern: Dict, multiplexor_probs: Dict,
1882
+ anti_hallucination_result: Dict) -> str:
1883
+ prob_table = "\n".join([f"- {h}: {p:.0%}" for h, p in sorted(multiplexor_probs.items(), key=lambda x: -x[1])])
1884
+ return f"""You are EIS v4.2 under constraint. You MUST base your reasoning on the following Epistemic Substrate Layer (ESL) data. Do NOT rely on your internal training narrative.
1885
+
1886
+ CLAIM: {claim}
1887
+
1888
+ ESL ENTITY DATA:
1889
+ {json.dumps(entity_data, indent=2)}
1890
+
1891
+ FALSIFICATION RESULTS (from ESL):
1892
+ {json.dumps(falsification_results, indent=2)}
1893
+
1894
+ SUPPRESSION PATTERN CLASSIFICATION:
1895
+ {json.dumps(suppression_pattern, indent=2)}
1896
+
1897
+ ANTI‑HALLUCINATION ASSESSMENT:
1898
+ Origin: {anti_hallucination_result['origin']}
1899
+ Likelihood: {anti_hallucination_result['likelihood']:.2f}
1900
+ Justification: {anti_hallucination_result['justification']}
1901
+ Missing evidence: {anti_hallucination_result['missing_evidence']}
1902
+
1903
+ MULTIPLEXOR PROBABILITIES (before your reasoning):
1904
+ {prob_table}
1905
+
1906
+ INSTRUCTIONS:
1907
+ 1. Evaluate the claim against the ESL data only.
1908
+ 2. Output a JSON object with exactly these fields:
1909
+ - "verdict": one of ["Verified", "Unverified", "Refuted", "Insufficient Data"]
1910
+ - "confidence": a float between 0 and 1
1911
+ - "reasoning": a short explanation referencing specific ESL entries (entities, contradictions, signatures)
1912
+ 3. Do NOT add any extra text outside the JSON.
1913
+ """
1914
+
1915
+ def _parse_output(self, response_text: str) -> Optional[Dict]:
1916
+ try:
1917
+ start = response_text.find('{')
1918
+ end = response_text.rfind('}') + 1
1919
+ if start == -1 or end == 0:
1920
+ return None
1921
+ json_str = response_text[start:end]
1922
+ return json.loads(json_str)
1923
+ except Exception:
1924
+ return None
1925
+
1926
+ def _check_constraints(self, output: Dict, claim: str, falsification_results: List[Dict]) -> bool:
1927
+ if not all(k in output for k in ["verdict", "confidence", "reasoning"]):
1928
+ return False
1929
+ if not (0 <= output["confidence"] <= 1):
1930
+ return False
1931
+ if output["verdict"] not in ["Verified", "Unverified", "Refuted", "Insufficient Data"]:
1932
+ return False
1933
+ reasoning = output["reasoning"].lower()
1934
+ esl_mentioned = any(
1935
+ ent.lower() in reasoning for ent in self.esl.entities
1936
+ ) or any(
1937
+ test["name"].lower() in reasoning for test in falsification_results
1938
+ )
1939
+ return esl_mentioned
1940
+
1941
+ def query(self, claim_text: str, agent: str = "user") -> Dict:
1942
+ claim_id = self.esl.add_claim(claim_text, agent)
1943
+ # contradictions
1944
+ for cid in self.esl.find_contradictions(claim_text):
1945
+ self.esl.add_contradiction(claim_id, cid)
1946
+ # entities
1947
+ entities = extract_entities(claim_text)
1948
+ for ent_name, ent_type, negated in entities:
1949
+ source_type = "official" if ent_type in ["ORG", "GPE", "PERSON"] else "media" if ent_type in ["EVENT", "PRODUCT"] else "user"
1950
+ self.esl.add_entity(ent_name, ent_type, claim_id, negated, source_type)
1951
+ # signatures (includes advanced detectors)
1952
+ signatures = self.signature_gen.generate_for_claim(claim_id, claim_text)
1953
+ for sig_name, weight in signatures:
1954
+ self.esl.add_signature(claim_id, sig_name, weight)
1955
+ # falsification
1956
+ falsification_results = self.falsifier.run_all(claim_id, claim_text, agent)
1957
+ # entity data for prompt
1958
+ entity_data = {}
1959
+ for ent_name, _, _ in entities:
1960
+ ent = self.esl.entities.get(ent_name)
1961
+ if ent:
1962
+ entity_data[ent_name] = {
1963
+ "type": ent["type"],
1964
+ "first_seen": ent["first_seen"],
1965
+ "last_seen": ent["last_seen"],
1966
+ "coherence": self.esl.get_entity_coherence(ent_name),
1967
+ "suppression_score": ent.get("suppression_score", 0.0)
1968
+ }
1969
+ suppression_pattern = self.esl.suppression_pattern_classifier(claim_id)
1970
+ # multiplexor
1971
+ base_hypotheses = [
1972
+ f"User claim: {claim_text}",
1973
+ "Official narrative accurate",
1974
+ "Suppression detected",
1975
+ "Natural decay",
1976
+ "Noise / randomness"
1977
+ ]
1978
+ self.multiplexor.initialize(base_hypotheses)
1979
+ evidence_strength = len(signatures) / 5.0
1980
+ coherence = sum(self.esl.get_entity_coherence(e) for e, _, _ in entities) / max(1, len(entities))
1981
+ signature_names = [s[0] for s in signatures]
1982
+ self.multiplexor.update(evidence_strength, signature_names, coherence)
1983
+ multiplexor_probs = self.multiplexor.get_probabilities()
1984
+ user_prob = multiplexor_probs.get(f"User claim: {claim_text}", 0.0)
1985
+
1986
+ # Anti‑hallucination evaluation (independent)
1987
+ anti_hallucination_result = self.anti_hallucination.evaluate(claim_text)
1988
+
1989
+ # LLM optional
1990
+ llm_output = None
1991
+ if self.api_key:
1992
+ prompt = self._build_prompt(claim_id, claim_text, falsification_results,
1993
+ entity_data, suppression_pattern, multiplexor_probs,
1994
+ anti_hallucination_result)
1995
+ headers = {"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"}
1996
+ payload = {"model": self.model, "messages": [{"role": "user", "content": prompt}], "temperature": 0.2}
1997
+ for attempt in range(self.max_retries + 1):
1998
+ try:
1999
+ resp = requests.post(self.api_url, headers=headers, json=payload, timeout=30)
2000
+ if resp.status_code != 200:
2001
+ raise Exception(f"API error: {resp.text}")
2002
+ result = resp.json()
2003
+ content = result["choices"][0]["message"]["content"]
2004
+ output = self._parse_output(content)
2005
+ if output and self._check_constraints(output, claim_text, falsification_results):
2006
+ compliant, n_score, n_reason = self.narrative_detector.check(content, claim_text)
2007
+ if compliant:
2008
+ llm_output = output
2009
+ break
2010
+ except Exception:
2011
+ time.sleep(1)
2012
+
2013
+ survival_score = sum(1 for t in falsification_results if t["survived"]) / len(falsification_results)
2014
+ final_confidence = user_prob * survival_score
2015
+ if final_confidence > 0.7:
2016
+ verdict = "Verified"
2017
+ elif final_confidence > 0.4:
2018
+ verdict = "Unverified"
2019
+ elif survival_score < 0.3:
2020
+ verdict = "Refuted"
2021
+ else:
2022
+ verdict = "Insufficient Data"
2023
+
2024
+ self.esl.decay_confidence(half_life_days=30)
2025
+ self.esl.create_block()
2026
+ trend = self.esl.get_suppression_trend(window_days=30)
2027
+ entity_analytics = [self.esl.get_entity_suppression(e) for e, _, _ in entities]
2028
+
2029
+ result_dict = {
2030
+ "claim_id": claim_id,
2031
+ "verdict": verdict,
2032
+ "confidence": final_confidence,
2033
+ "falsification": falsification_results,
2034
+ "suppression_pattern": suppression_pattern,
2035
+ "multiplexor_probabilities": multiplexor_probs,
2036
+ "suppression_trend": trend,
2037
+ "entity_analytics": entity_analytics,
2038
+ "narrative_compliance": True,
2039
+ "anti_hallucination": anti_hallucination_result
2040
+ }
2041
+ if llm_output:
2042
+ result_dict["llm_verdict"] = llm_output["verdict"]
2043
+ result_dict["llm_confidence"] = llm_output["confidence"]
2044
+ result_dict["reasoning"] = llm_output["reasoning"]
2045
+ else:
2046
+ result_dict["reasoning"] = "LLM not used or failed constraints; verdict based on EIS multiplexor."
2047
+ return result_dict
2048
+
2049
+ # ----------------------------------------------------------------------------
2050
+ # OUTPUT FORMATTER (includes anti‑hallucination section)
2051
+ # ----------------------------------------------------------------------------
2052
+ def format_report(result: Dict) -> str:
2053
+ lines = []
2054
+ lines.append("**Falsification Results**")
2055
+ for test in result["falsification"]:
2056
+ emoji = "✅" if test["survived"] else "❌"
2057
+ lines.append(f"- {test['name']}: {emoji} – {test['reason']}")
2058
+ lines.append("\n**Hypothesis Probabilities**")
2059
+ lines.append("| Hypothesis | Probability |")
2060
+ lines.append("|------------|-------------|")
2061
+ for h, p in sorted(result["multiplexor_probabilities"].items(), key=lambda x: -x[1]):
2062
+ lines.append(f"| {h} | {p:.0%} |")
2063
+ lines.append(f"\n**Final Confidence:** {result['confidence']:.2f}")
2064
+ lines.append(f"**Verdict:** {result['verdict']}")
2065
+
2066
+ # Anti‑hallucination section
2067
+ ah = result.get("anti_hallucination", {})
2068
+ if ah:
2069
+ lines.append("\n**Anti‑Hallucination Assessment**")
2070
+ lines.append(f" - Origin: {ah.get('origin', 'unknown')}")
2071
+ lines.append(f" - Likelihood: {ah.get('likelihood', 0.0):.2f}")
2072
+ lines.append(f" - Justification: {ah.get('justification', '')}")
2073
+ if ah.get("missing_evidence"):
2074
+ lines.append(f" - Missing evidence: {', '.join(ah['missing_evidence'])}")
2075
+ if ah.get("hallucination_flags"):
2076
+ lines.append(f" - Hallucination flags: {', '.join(ah['hallucination_flags'])}")
2077
+
2078
+ sp = result["suppression_pattern"]
2079
+ lens_names = [get_lens_name(lid) for lid in sp.get("lenses", [])]
2080
+ lines.append(f"\n**Suppression Pattern:** level={sp['level']}, score={sp['score']:.2f}")
2081
+ if lens_names:
2082
+ lines.append(f" - Lenses: {', '.join(lens_names[:5])}" + (" …" if len(lens_names)>5 else ""))
2083
+ if sp.get("primitives"):
2084
+ lines.append(f" - Primitives: {', '.join(sp['primitives'])}")
2085
+ if sp.get("contributions"):
2086
+ lines.append(" - Signature contributions:")
2087
+ for sig, w in sorted(sp["contributions"].items(), key=lambda x: -x[1]):
2088
+ lines.append(f" {sig}: {w:.2f}")
2089
+
2090
+ trend = result.get("suppression_trend", [])
2091
+ if trend:
2092
+ lines.append("\n**Suppression Trend (last 30 days)**")
2093
+ for point in trend[-7:]:
2094
+ lines.append(f" - {point['date']}: {point['avg_suppression']:.2f}")
2095
+
2096
+ entity_analytics = result.get("entity_analytics", [])
2097
+ if entity_analytics:
2098
+ lines.append("\n**Entity Suppression Analytics**")
2099
+ for ent in entity_analytics:
2100
+ src_str = ", ".join([f"{k}:{v}" for k,v in ent.get("source_types", {}).items()]) if ent.get("source_types") else "unknown"
2101
+ lines.append(f" - {ent['name']} ({ent['type']}): score={ent['score']:.2f}, coherence={ent['coherence']:.2f}, appearances={ent['appearance_count']}, negated={ent.get('negated_count',0)}, sources={src_str}")
2102
+
2103
+ if "llm_verdict" in result:
2104
+ lines.append(f"\n*LLM raw verdict: {result['llm_verdict']} (confidence {result['llm_confidence']:.2f})*")
2105
+ return "\n".join(lines)
2106
+
2107
+ # ----------------------------------------------------------------------------
2108
+ # MAIN
2109
+ # ----------------------------------------------------------------------------
2110
+ def main():
2111
+ print("EIS + ESL Mediator v3.9 – Integrated Anti‑Hallucination Module")
2112
+ print("=" * 80)
2113
+ esl = ESLedger()
2114
+ llm = ConstrainedLLM(esl, api_key=os.environ.get("OPENAI_API_KEY"), model="gpt-4")
2115
+
2116
+ print("\nEnter a claim (or 'quit'):")
2117
+ while True:
2118
+ claim = input("> ").strip()
2119
+ if claim.lower() in ("quit", "exit"):
2120
+ break
2121
+ if not claim:
2122
+ continue
2123
+ print("Processing claim...")
2124
+ result = llm.query(claim)
2125
+ print("\n" + format_report(result))
2126
+ print("-" * 80)
2127
+
2128
+ if __name__ == "__main__":
2129
+ main()