Marius16 commited on
Commit
f397bd6
·
1 Parent(s): 5edce0b

Add neo4j and cross article verification

Browse files
.claude/settings.local.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "permissions": {
3
+ "allow": [
4
+ "Read(//mnt/c/Users/pante/Desktop/**)",
5
+ "Bash(pip install *)"
6
+ ]
7
+ }
8
+ }
.env.example CHANGED
@@ -26,3 +26,10 @@ CORS_ORIGINS=http://localhost:3000
26
 
27
  # NewsAPI key — only needed for the demo notebook, not the main pipeline
28
  # NEWSAPI_KEY=your_key_here
 
 
 
 
 
 
 
 
26
 
27
  # NewsAPI key — only needed for the demo notebook, not the main pipeline
28
  # NEWSAPI_KEY=your_key_here
29
+
30
+ # Neo4j (optional — for persistent cross-article analysis)
31
+ # Start with: docker run -p 7687:7687 -e NEO4J_AUTH=neo4j/password neo4j:community
32
+ NEO4J_URI=bolt://localhost:7687
33
+ NEO4J_USER=neo4j
34
+ NEO4J_PASSWORD=password
35
+ NEO4J_ENABLED=false
backend/config.py CHANGED
@@ -66,6 +66,13 @@ TCS_THRESHOLDS = {
66
  }
67
 
68
 
 
 
 
 
 
 
 
69
  # FastAPI settings
70
  API_TITLE = "TCS - Temporal Coherence Score API"
71
  API_VERSION = "0.1.0"
 
66
  }
67
 
68
 
69
+ # Neo4j — optional persistent store for cross-article analysis
70
+ NEO4J_URI = os.getenv("NEO4J_URI", "bolt://localhost:7687")
71
+ NEO4J_USER = os.getenv("NEO4J_USER", "neo4j")
72
+ NEO4J_PASSWORD = os.getenv("NEO4J_PASSWORD", "password")
73
+ NEO4J_ENABLED = os.getenv("NEO4J_ENABLED", "false").lower() == "true"
74
+
75
+
76
  # FastAPI settings
77
  API_TITLE = "TCS - Temporal Coherence Score API"
78
  API_VERSION = "0.1.0"
backend/main.py CHANGED
@@ -9,6 +9,9 @@ from fastapi.middleware.cors import CORSMiddleware
9
 
10
  from backend.config import API_TITLE, API_VERSION, CORS_ORIGINS
11
  from backend.routers import analyze, compare, health
 
 
 
12
 
13
  logging.basicConfig(
14
  level=logging.INFO,
@@ -38,6 +41,9 @@ app.add_middleware(
38
  app.include_router(health.router)
39
  app.include_router(analyze.router)
40
  app.include_router(compare.router)
 
 
 
41
 
42
 
43
  @app.on_event("startup")
 
9
 
10
  from backend.config import API_TITLE, API_VERSION, CORS_ORIGINS
11
  from backend.routers import analyze, compare, health
12
+ from backend.routers.articles import router as articles_router
13
+ from backend.routers.batch import router as batch_router
14
+ from backend.routers.upload import router as upload_router
15
 
16
  logging.basicConfig(
17
  level=logging.INFO,
 
41
  app.include_router(health.router)
42
  app.include_router(analyze.router)
43
  app.include_router(compare.router)
44
+ app.include_router(articles_router)
45
+ app.include_router(batch_router)
46
+ app.include_router(upload_router)
47
 
48
 
49
  @app.on_event("startup")
backend/pipeline/graph/base_store.py ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Abstract interface for Temporal Knowledge Graph storage backends."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from abc import ABC, abstractmethod
6
+
7
+ from backend.pipeline.graph.models import TemporalFact
8
+
9
+
10
+ class AbstractTKGStore(ABC):
11
+ """Interface for Temporal Knowledge Graph storage backends."""
12
+
13
+ @abstractmethod
14
+ def add_fact(self, fact: TemporalFact, article_id: str | None = None) -> None: ...
15
+
16
+ @abstractmethod
17
+ def add_facts(self, facts: list[TemporalFact], article_id: str | None = None) -> None: ...
18
+
19
+ @abstractmethod
20
+ def get_all_facts(self, article_id: str | None = None) -> list[TemporalFact]: ...
21
+
22
+ @abstractmethod
23
+ def get_facts_for_entity(self, entity_name: str) -> list[TemporalFact]: ...
24
+
25
+ @abstractmethod
26
+ def get_articles(self) -> list[dict]: ...
27
+
28
+ @abstractmethod
29
+ def delete_article(self, article_id: str) -> bool: ...
30
+
31
+ @abstractmethod
32
+ def summary(self) -> dict: ...
backend/pipeline/graph/factory.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Factory for creating TKG storage backends."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+
7
+ from backend.pipeline.graph.base_store import AbstractTKGStore
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+
12
+ def create_persistent_store() -> AbstractTKGStore | None:
13
+ """Return a Neo4jTKGStore if NEO4J_ENABLED=true, otherwise None."""
14
+ from backend.config import NEO4J_ENABLED, NEO4J_PASSWORD, NEO4J_URI, NEO4J_USER
15
+
16
+ if not NEO4J_ENABLED:
17
+ logger.debug("NEO4J_ENABLED=false — persistent store disabled")
18
+ return None
19
+
20
+ from backend.pipeline.graph.neo4j_store import Neo4jTKGStore
21
+
22
+ try:
23
+ store = Neo4jTKGStore(NEO4J_URI, NEO4J_USER, NEO4J_PASSWORD)
24
+ logger.info(f"Persistent Neo4j store created at {NEO4J_URI}")
25
+ return store
26
+ except Exception as exc:
27
+ logger.error(f"Failed to connect to Neo4j at {NEO4J_URI}: {exc}")
28
+ return None
backend/pipeline/graph/neo4j_store.py ADDED
@@ -0,0 +1,262 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Persistent TKG storage using Neo4j Community Edition."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ from datetime import datetime
7
+ from typing import TYPE_CHECKING
8
+
9
+ from backend.pipeline.graph.base_store import AbstractTKGStore
10
+ from backend.pipeline.graph.models import (
11
+ Entity, EntityType, RelationType, TemporalExpression, TemporalFact,
12
+ )
13
+ from backend.pipeline.graph.store import _entity_id
14
+
15
+ if TYPE_CHECKING:
16
+ from neo4j import Driver
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+ # Fields returned by every fact-fetching query — defined once to avoid repetition
21
+ _FACT_RETURN = """
22
+ s.entity_id AS s_id, s.text AS s_text, s.entity_type AS s_type, s.wikidata_id AS s_wikidata,
23
+ o.entity_id AS o_id, o.text AS o_text, o.entity_type AS o_type, o.wikidata_id AS o_wikidata,
24
+ r.relation AS relation,
25
+ r.time_start_iso AS time_start_iso, r.time_start_str AS time_start_str,
26
+ r.time_end_iso AS time_end_iso, r.time_end_str AS time_end_str,
27
+ r.time_point_iso AS time_point_iso, r.time_point_str AS time_point_str,
28
+ r.source_sentence_idx AS sentence_idx,
29
+ r.confidence AS confidence,
30
+ r.extractor AS extractor
31
+ """
32
+
33
+
34
+ class Neo4jTKGStore(AbstractTKGStore):
35
+ """Persistent TKG storage using Neo4j."""
36
+
37
+ def __init__(self, uri: str, user: str, password: str) -> None:
38
+ from neo4j import GraphDatabase
39
+ self._driver: Driver = GraphDatabase.driver(uri, auth=(user, password))
40
+ self._ensure_indexes()
41
+ logger.info(f"Neo4jTKGStore connected to {uri}")
42
+
43
+ def _ensure_indexes(self) -> None:
44
+ """Create indexes on first run so lookups on entity_id and article_id are fast."""
45
+ with self._driver.session() as session:
46
+ session.run("CREATE INDEX IF NOT EXISTS FOR (e:Entity) ON (e.entity_id)")
47
+ session.run("CREATE INDEX IF NOT EXISTS FOR (a:Article) ON (a.article_id)")
48
+
49
+ def close(self) -> None:
50
+ self._driver.close()
51
+
52
+ # ── Write ──
53
+
54
+ def add_fact(self, fact: TemporalFact, article_id: str | None = None) -> None:
55
+ """Store a fact as two Entity nodes + a TEMPORAL_RELATION edge."""
56
+ with self._driver.session() as session:
57
+ session.execute_write(self._create_fact_tx, fact, article_id)
58
+
59
+ @staticmethod
60
+ def _create_fact_tx(tx, fact: TemporalFact, article_id: str | None) -> None:
61
+ def to_iso(expr: TemporalExpression | None) -> str | None:
62
+ if expr and expr.normalized_date:
63
+ return expr.normalized_date.isoformat()
64
+ return None
65
+
66
+ def to_str(expr: TemporalExpression | None) -> str | None:
67
+ return expr.date_string if expr else None
68
+
69
+ params = {
70
+ "subj_id": _entity_id(fact.subject),
71
+ "subj_text": fact.subject.text,
72
+ "subj_type": fact.subject.entity_type.value,
73
+ "subj_wikidata": fact.subject.wikidata_id,
74
+ "obj_id": _entity_id(fact.object),
75
+ "obj_text": fact.object.text,
76
+ "obj_type": fact.object.entity_type.value,
77
+ "obj_wikidata": fact.object.wikidata_id,
78
+ "relation": fact.predicate.value,
79
+ "time_start_iso": to_iso(fact.time_start),
80
+ "time_start_str": to_str(fact.time_start),
81
+ "time_end_iso": to_iso(fact.time_end),
82
+ "time_end_str": to_str(fact.time_end),
83
+ "time_point_iso": to_iso(fact.time_point),
84
+ "time_point_str": to_str(fact.time_point),
85
+ "sentence_idx": fact.source_sentence_idx,
86
+ "confidence": fact.extraction_confidence,
87
+ "extractor": fact.extractor,
88
+ "article_id": article_id,
89
+ }
90
+
91
+ # MERGE entity nodes so the same entity is never duplicated across articles
92
+ tx.run("""
93
+ MERGE (s:Entity {entity_id: $subj_id})
94
+ ON CREATE SET s.text = $subj_text, s.entity_type = $subj_type,
95
+ s.wikidata_id = $subj_wikidata
96
+ MERGE (o:Entity {entity_id: $obj_id})
97
+ ON CREATE SET o.text = $obj_text, o.entity_type = $obj_type,
98
+ o.wikidata_id = $obj_wikidata
99
+ CREATE (s)-[:TEMPORAL_RELATION {
100
+ relation: $relation,
101
+ time_start_iso: $time_start_iso,
102
+ time_start_str: $time_start_str,
103
+ time_end_iso: $time_end_iso,
104
+ time_end_str: $time_end_str,
105
+ time_point_iso: $time_point_iso,
106
+ time_point_str: $time_point_str,
107
+ source_sentence_idx: $sentence_idx,
108
+ confidence: $confidence,
109
+ extractor: $extractor,
110
+ article_id: $article_id
111
+ }]->(o)
112
+ """, **params)
113
+
114
+ if article_id:
115
+ tx.run("MERGE (a:Article {article_id: $article_id})", article_id=article_id)
116
+
117
+ def add_facts(self, facts: list[TemporalFact], article_id: str | None = None) -> None:
118
+ for fact in facts:
119
+ self.add_fact(fact, article_id)
120
+ logger.info(
121
+ f"Neo4jTKGStore: {len(facts)} facts written"
122
+ + (f" for article {article_id}" if article_id else "")
123
+ )
124
+
125
+ # ── Read ──
126
+
127
+ def get_all_facts(self, article_id: str | None = None) -> list[TemporalFact]:
128
+ """Retrieve facts, optionally filtered by article_id."""
129
+ if article_id:
130
+ query = f"""
131
+ MATCH (s:Entity)-[r:TEMPORAL_RELATION {{article_id: $article_id}}]->(o:Entity)
132
+ RETURN {_FACT_RETURN}
133
+ """
134
+ params: dict = {"article_id": article_id}
135
+ else:
136
+ query = f"""
137
+ MATCH (s:Entity)-[r:TEMPORAL_RELATION]->(o:Entity)
138
+ RETURN {_FACT_RETURN}
139
+ """
140
+ params = {}
141
+
142
+ with self._driver.session() as session:
143
+ result = session.run(query, **params)
144
+ return [_record_to_fact(r) for r in result]
145
+
146
+ def get_facts_for_entity(self, entity_name: str) -> list[TemporalFact]:
147
+ """All facts across ALL articles where the entity appears as subject or object."""
148
+ entity_id = entity_name.lower().strip()
149
+ query = f"""
150
+ MATCH (s:Entity)-[r:TEMPORAL_RELATION]->(o:Entity)
151
+ WHERE s.entity_id = $entity_id OR o.entity_id = $entity_id
152
+ RETURN {_FACT_RETURN}
153
+ """
154
+ with self._driver.session() as session:
155
+ result = session.run(query, entity_id=entity_id)
156
+ return [_record_to_fact(r) for r in result]
157
+
158
+ def get_articles(self) -> list[dict]:
159
+ """List all analyzed articles with their fact counts."""
160
+ with self._driver.session() as session:
161
+ result = session.run("""
162
+ MATCH (a:Article)
163
+ OPTIONAL MATCH ()-[r:TEMPORAL_RELATION {article_id: a.article_id}]->()
164
+ RETURN a.article_id AS article_id,
165
+ a.title AS title,
166
+ a.source AS source,
167
+ a.analyzed_at AS analyzed_at,
168
+ count(r) AS fact_count
169
+ ORDER BY a.analyzed_at DESC
170
+ """)
171
+ return [dict(record) for record in result]
172
+
173
+ def delete_article(self, article_id: str) -> bool:
174
+ """Delete an article node and all TEMPORAL_RELATION edges that belong to it."""
175
+ with self._driver.session() as session:
176
+ check = session.run(
177
+ "MATCH (a:Article {article_id: $id}) RETURN count(a) > 0 AS found",
178
+ id=article_id,
179
+ ).single()
180
+ if not (check and check["found"]):
181
+ return False
182
+ session.run(
183
+ "MATCH ()-[r:TEMPORAL_RELATION {article_id: $id}]->() DELETE r",
184
+ id=article_id,
185
+ )
186
+ session.run(
187
+ "MATCH (a:Article {article_id: $id}) DELETE a",
188
+ id=article_id,
189
+ )
190
+ return True
191
+
192
+ def summary(self) -> dict:
193
+ with self._driver.session() as session:
194
+ result = session.run("""
195
+ MATCH (e:Entity) WITH count(e) AS nodes
196
+ MATCH ()-[r:TEMPORAL_RELATION]->() WITH nodes, count(r) AS edges
197
+ OPTIONAL MATCH (a:Article) WITH nodes, edges, count(a) AS articles
198
+ RETURN nodes, edges, articles
199
+ """).single()
200
+ if result is None:
201
+ return {"nodes": 0, "edges": 0, "articles": 0}
202
+ return {
203
+ "nodes": result["nodes"],
204
+ "edges": result["edges"],
205
+ "articles": result["articles"],
206
+ }
207
+
208
+
209
+ # ── Helpers ──
210
+
211
+ def _record_to_fact(record) -> TemporalFact:
212
+ """Reconstruct a TemporalFact from a Neo4j query record."""
213
+
214
+ def parse_entity_type(val: str) -> EntityType:
215
+ try:
216
+ return EntityType(val)
217
+ except ValueError:
218
+ return EntityType.OTHER
219
+
220
+ def parse_relation_type(val: str) -> RelationType:
221
+ try:
222
+ return RelationType(val)
223
+ except ValueError:
224
+ return RelationType.GENERIC
225
+
226
+ def parse_temporal_expr(
227
+ iso: str | None, date_str: str | None
228
+ ) -> TemporalExpression | None:
229
+ if iso is None:
230
+ return None
231
+ try:
232
+ dt: datetime | None = datetime.fromisoformat(iso)
233
+ except (ValueError, TypeError):
234
+ dt = None
235
+ return TemporalExpression(raw_text=date_str or "", normalized_date=dt, date_string=date_str)
236
+
237
+ subject = Entity(
238
+ text=record["s_text"],
239
+ entity_type=parse_entity_type(record["s_type"]),
240
+ start_char=0,
241
+ end_char=0,
242
+ wikidata_id=record.get("s_wikidata"),
243
+ )
244
+ obj = Entity(
245
+ text=record["o_text"],
246
+ entity_type=parse_entity_type(record["o_type"]),
247
+ start_char=0,
248
+ end_char=0,
249
+ wikidata_id=record.get("o_wikidata"),
250
+ )
251
+
252
+ return TemporalFact(
253
+ subject=subject,
254
+ predicate=parse_relation_type(record["relation"]),
255
+ object=obj,
256
+ time_start=parse_temporal_expr(record.get("time_start_iso"), record.get("time_start_str")),
257
+ time_end=parse_temporal_expr(record.get("time_end_iso"), record.get("time_end_str")),
258
+ time_point=parse_temporal_expr(record.get("time_point_iso"), record.get("time_point_str")),
259
+ source_sentence_idx=record.get("sentence_idx") or 0,
260
+ extraction_confidence=record.get("confidence") or 1.0,
261
+ extractor=record.get("extractor") or "unknown",
262
+ )
backend/pipeline/graph/store.py CHANGED
@@ -8,6 +8,7 @@ from typing import Optional
8
 
9
  import networkx as nx
10
 
 
11
  from backend.pipeline.graph.models import (
12
  Entity, EntityType, RelationType, TemporalExpression, TemporalFact,
13
  )
@@ -15,7 +16,7 @@ from backend.pipeline.graph.models import (
15
  logger = logging.getLogger(__name__)
16
 
17
 
18
- class TemporalKnowledgeGraph:
19
  """
20
  Graf temporal conform Cai et al. (2024): G = (E, R, T, F).
21
  Noduri = entitati, Muchii = relatii cu metadata temporala.
@@ -26,8 +27,8 @@ class TemporalKnowledgeGraph:
26
  self._facts: list[TemporalFact] = []
27
 
28
  # ── Adaugare ──
29
- def add_fact(self, fact: TemporalFact) -> None:
30
- """Adauga un TemporalFact ca nod+muchie in graf."""
31
  subj_id = _entity_id(fact.subject)
32
  obj_id = _entity_id(fact.object)
33
 
@@ -40,9 +41,9 @@ class TemporalKnowledgeGraph:
40
  self._graph.add_edge(subj_id, obj_id, **edge_attrs)
41
  self._facts.append(fact)
42
 
43
- def add_facts(self, facts: list[TemporalFact]) -> None:
44
  for fact in facts:
45
- self.add_fact(fact)
46
  logger.info(f"TKG: {len(facts)} fapte adaugate. Noduri: {self.node_count}, Muchii: {self.edge_count}")
47
 
48
  # ── Interogare noduri ──
@@ -53,7 +54,8 @@ class TemporalKnowledgeGraph:
53
  return dict(self._graph.nodes.get(entity_id, {}))
54
 
55
  # ── Interogare fapte ──
56
- def get_all_facts(self) -> list[TemporalFact]:
 
57
  return list(self._facts)
58
 
59
  def get_facts_for_entity(self, entity_name: str) -> list[TemporalFact]:
@@ -61,6 +63,14 @@ class TemporalKnowledgeGraph:
61
  entity_id = entity_name.lower().strip()
62
  return [f for f in self._facts if _entity_id(f.subject) == entity_id or _entity_id(f.object) == entity_id]
63
 
 
 
 
 
 
 
 
 
64
  def get_edges_in_interval(self, t_start: datetime, t_end: datetime) -> list[dict]:
65
  """Muchiile active in intervalul [t_start, t_end]."""
66
  result = []
 
8
 
9
  import networkx as nx
10
 
11
+ from backend.pipeline.graph.base_store import AbstractTKGStore
12
  from backend.pipeline.graph.models import (
13
  Entity, EntityType, RelationType, TemporalExpression, TemporalFact,
14
  )
 
16
  logger = logging.getLogger(__name__)
17
 
18
 
19
+ class TemporalKnowledgeGraph(AbstractTKGStore):
20
  """
21
  Graf temporal conform Cai et al. (2024): G = (E, R, T, F).
22
  Noduri = entitati, Muchii = relatii cu metadata temporala.
 
27
  self._facts: list[TemporalFact] = []
28
 
29
  # ── Adaugare ──
30
+ def add_fact(self, fact: TemporalFact, article_id: str | None = None) -> None:
31
+ """Adauga un TemporalFact ca nod+muchie in graf. article_id ignorat (in-memory)."""
32
  subj_id = _entity_id(fact.subject)
33
  obj_id = _entity_id(fact.object)
34
 
 
41
  self._graph.add_edge(subj_id, obj_id, **edge_attrs)
42
  self._facts.append(fact)
43
 
44
+ def add_facts(self, facts: list[TemporalFact], article_id: str | None = None) -> None:
45
  for fact in facts:
46
+ self.add_fact(fact, article_id)
47
  logger.info(f"TKG: {len(facts)} fapte adaugate. Noduri: {self.node_count}, Muchii: {self.edge_count}")
48
 
49
  # ── Interogare noduri ──
 
54
  return dict(self._graph.nodes.get(entity_id, {}))
55
 
56
  # ── Interogare fapte ──
57
+ def get_all_facts(self, article_id: str | None = None) -> list[TemporalFact]:
58
+ # article_id ignored — in-memory store holds one article at a time
59
  return list(self._facts)
60
 
61
  def get_facts_for_entity(self, entity_name: str) -> list[TemporalFact]:
 
63
  entity_id = entity_name.lower().strip()
64
  return [f for f in self._facts if _entity_id(f.subject) == entity_id or _entity_id(f.object) == entity_id]
65
 
66
+ def get_articles(self) -> list[dict]:
67
+ # in-memory store has no persistence; articles are not tracked
68
+ return []
69
+
70
+ def delete_article(self, article_id: str) -> bool:
71
+ # in-memory store has no persistence
72
+ return False
73
+
74
  def get_edges_in_interval(self, t_start: datetime, t_end: datetime) -> list[dict]:
75
  """Muchiile active in intervalul [t_start, t_end]."""
76
  result = []
backend/pipeline/verification/cross_article.py ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """C3c — Cross-article inconsistency detection using the persistent TKG store."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ from datetime import datetime
7
+ from difflib import SequenceMatcher
8
+ from typing import Optional
9
+
10
+ from backend.pipeline.graph.base_store import AbstractTKGStore
11
+ from backend.pipeline.graph.models import (
12
+ Inconsistency, InconsistencyType, Severity, TemporalFact,
13
+ )
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+ SIMILARITY_THRESHOLD = 0.80
18
+
19
+
20
+ class CrossArticleVerifier:
21
+ """Detects inconsistencies between facts in the current article and
22
+ previously analyzed articles stored in Neo4j."""
23
+
24
+ def __init__(self, persistent_store: AbstractTKGStore) -> None:
25
+ self.store = persistent_store
26
+
27
+ def verify(
28
+ self,
29
+ current_facts: list[TemporalFact],
30
+ current_article_id: str,
31
+ ) -> list[Inconsistency]:
32
+ """Compare current article facts against all stored facts for the same entities.
33
+
34
+ Call this BEFORE persisting the current article so stored results belong
35
+ exclusively to prior articles.
36
+ """
37
+ inconsistencies: list[Inconsistency] = []
38
+
39
+ for fact in current_facts:
40
+ entity_name = fact.subject.text.lower().strip()
41
+ stored_facts = self.store.get_facts_for_entity(entity_name)
42
+
43
+ # safety guard: skip any fact already tagged to this article
44
+ other_facts = [
45
+ f for f in stored_facts
46
+ if getattr(f, "article_id", None) != current_article_id
47
+ ]
48
+
49
+ for stored in other_facts:
50
+ # check 1: same relation + similar object + conflicting dates
51
+ if (
52
+ fact.predicate == stored.predicate
53
+ and _objects_similar(fact.object.text, stored.object.text)
54
+ ):
55
+ conflict = _check_date_conflict(fact, stored)
56
+ if conflict:
57
+ inconsistencies.append(conflict)
58
+
59
+ # check 2: same entity holds different positions at overlapping times
60
+ if (
61
+ fact.predicate.value == "holds_position"
62
+ and stored.predicate.value == "holds_position"
63
+ and _overlapping_time(fact, stored)
64
+ and not _objects_similar(fact.object.text, stored.object.text)
65
+ ):
66
+ inconsistencies.append(
67
+ Inconsistency(
68
+ inconsistency_type=InconsistencyType.DATE_MISMATCH,
69
+ severity=Severity.HIGH,
70
+ description=(
71
+ f"Cross-article conflict: '{fact.subject.text}' holds "
72
+ f"different positions at overlapping times: "
73
+ f"'{fact.object.text}' vs '{stored.object.text}'"
74
+ ),
75
+ facts_involved=[fact, stored],
76
+ sentence_indices=[fact.source_sentence_idx],
77
+ verified_by="cross_article",
78
+ evidence=f"Article {current_article_id} vs stored facts",
79
+ )
80
+ )
81
+
82
+ logger.info(
83
+ f"Cross-article verification: {len(inconsistencies)} conflicts found "
84
+ f"for article {current_article_id}"
85
+ )
86
+ return inconsistencies
87
+
88
+
89
+ def _objects_similar(a: str, b: str) -> bool:
90
+ """True when strings are equal (case-insensitive) or fuzzy-match above threshold."""
91
+ a_norm, b_norm = a.lower().strip(), b.lower().strip()
92
+ if a_norm == b_norm:
93
+ return True
94
+ return SequenceMatcher(None, a_norm, b_norm).ratio() >= SIMILARITY_THRESHOLD
95
+
96
+
97
+ def _check_date_conflict(
98
+ fact_a: TemporalFact, fact_b: TemporalFact
99
+ ) -> Optional[Inconsistency]:
100
+ """Return an Inconsistency when the same claim appears with dates more than 1 year apart."""
101
+ time_a = _best_date(fact_a)
102
+ time_b = _best_date(fact_b)
103
+ if time_a is None or time_b is None:
104
+ return None
105
+
106
+ if abs((time_a - time_b).days) <= 365:
107
+ return None
108
+
109
+ return Inconsistency(
110
+ inconsistency_type=InconsistencyType.DATE_MISMATCH,
111
+ severity=Severity.MEDIUM,
112
+ description=(
113
+ f"Cross-article date conflict: '{fact_a.subject.text} "
114
+ f"{fact_a.predicate.value} {fact_a.object.text}' — "
115
+ f"{time_a.year} (current article) vs {time_b.year} (stored)"
116
+ ),
117
+ facts_involved=[fact_a, fact_b],
118
+ sentence_indices=[fact_a.source_sentence_idx],
119
+ verified_by="cross_article",
120
+ evidence=f"{time_a.date()} (current) vs {time_b.date()} (stored)",
121
+ )
122
+
123
+
124
+ def _overlapping_time(fact_a: TemporalFact, fact_b: TemporalFact) -> bool:
125
+ """True if the temporal ranges of the two facts overlap."""
126
+ start_a = _range_start(fact_a)
127
+ start_b = _range_start(fact_b)
128
+ if start_a is None or start_b is None:
129
+ return False
130
+
131
+ end_a = _range_end(fact_a) or datetime.max
132
+ end_b = _range_end(fact_b) or datetime.max
133
+
134
+ return start_a <= end_b and start_b <= end_a
135
+
136
+
137
+ def _best_date(fact: TemporalFact) -> Optional[datetime]:
138
+ """Most representative single datetime: time_point > time_start > time_end."""
139
+ for attr in ("time_point", "time_start", "time_end"):
140
+ expr = getattr(fact, attr)
141
+ if expr and expr.normalized_date:
142
+ return expr.normalized_date
143
+ return None
144
+
145
+
146
+ def _range_start(fact: TemporalFact) -> Optional[datetime]:
147
+ for attr in ("time_start", "time_point"):
148
+ expr = getattr(fact, attr)
149
+ if expr and expr.normalized_date:
150
+ return expr.normalized_date
151
+ return None
152
+
153
+
154
+ def _range_end(fact: TemporalFact) -> Optional[datetime]:
155
+ for attr in ("time_end", "time_point"):
156
+ expr = getattr(fact, attr)
157
+ if expr and expr.normalized_date:
158
+ return expr.normalized_date
159
+ return None
backend/routers/analyze.py CHANGED
@@ -3,13 +3,14 @@
3
  from __future__ import annotations
4
 
5
  import logging
 
6
  from datetime import datetime
7
  from typing import Optional
8
 
9
  from fastapi import APIRouter, HTTPException
10
  from pydantic import BaseModel, Field
11
 
12
- from backend.pipeline.graph.models import Article
13
  from backend.pipeline.orchestrator import PipelineOrchestrator
14
  from backend.routers.dependencies import get_orchestrator, explainer
15
 
@@ -26,6 +27,7 @@ class AnalyzeRequest(BaseModel):
26
  source: str = Field(default="", description="Sursa articolului")
27
  pipeline: str = Field(default="spacy", description="Pipeline: 'spacy' sau 'llm'")
28
  model: Optional[str] = Field(default=None, description="Model specific: en_core_web_trf, llama3, mistral, etc. None = default per pipeline")
 
29
 
30
 
31
  class InconsistencyResponse(BaseModel):
@@ -65,11 +67,26 @@ class AnalyzeResponse(BaseModel):
65
  pipeline: str
66
  model: str = Field(default="", description="Model folosit")
67
  processing_time_ms: float
 
 
68
 
69
 
70
  _explainer = explainer
71
 
72
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
  # Endpoint
74
  @router.post("/analyze", response_model=AnalyzeResponse)
75
  async def analyze_article(req: AnalyzeRequest) -> AnalyzeResponse:
@@ -108,6 +125,37 @@ async def analyze_article(req: AnalyzeRequest) -> AnalyzeResponse:
108
  logger.error(f"/analyze: eroare pipeline — {e}", exc_info=True)
109
  raise HTTPException(status_code=500, detail="Eroare interna la procesare.")
110
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
111
  # Genereaza explicatii structurate
112
  explanation = _explainer.explain_structured(result)
113
 
@@ -124,4 +172,6 @@ async def analyze_article(req: AnalyzeRequest) -> AnalyzeResponse:
124
  pipeline=result.pipeline_variant,
125
  model=req.model or AVAILABLE_MODELS[req.pipeline]["default"],
126
  processing_time_ms=result.processing_time_ms,
 
 
127
  )
 
3
  from __future__ import annotations
4
 
5
  import logging
6
+ import uuid
7
  from datetime import datetime
8
  from typing import Optional
9
 
10
  from fastapi import APIRouter, HTTPException
11
  from pydantic import BaseModel, Field
12
 
13
+ from backend.pipeline.graph.models import Article, Inconsistency
14
  from backend.pipeline.orchestrator import PipelineOrchestrator
15
  from backend.routers.dependencies import get_orchestrator, explainer
16
 
 
27
  source: str = Field(default="", description="Sursa articolului")
28
  pipeline: str = Field(default="spacy", description="Pipeline: 'spacy' sau 'llm'")
29
  model: Optional[str] = Field(default=None, description="Model specific: en_core_web_trf, llama3, mistral, etc. None = default per pipeline")
30
+ persist: bool = Field(default=False, description="Save facts to Neo4j for cross-article analysis")
31
 
32
 
33
  class InconsistencyResponse(BaseModel):
 
67
  pipeline: str
68
  model: str = Field(default="", description="Model folosit")
69
  processing_time_ms: float
70
+ article_id: Optional[str] = Field(default=None, description="Neo4j article ID (set when persist=True)")
71
+ cross_article_inconsistencies: list[InconsistencyResponse] = Field(default_factory=list)
72
 
73
 
74
  _explainer = explainer
75
 
76
 
77
+ def _to_inconsistency_response(inc: Inconsistency) -> InconsistencyResponse:
78
+ """Serialize an Inconsistency dataclass to the API response model."""
79
+ return InconsistencyResponse(
80
+ type=inc.inconsistency_type.value,
81
+ severity=inc.severity.value,
82
+ severity_label=inc.severity.value.title(),
83
+ description=inc.description,
84
+ evidence=inc.evidence,
85
+ verified_by=inc.verified_by,
86
+ sentence_indices=inc.sentence_indices,
87
+ )
88
+
89
+
90
  # Endpoint
91
  @router.post("/analyze", response_model=AnalyzeResponse)
92
  async def analyze_article(req: AnalyzeRequest) -> AnalyzeResponse:
 
125
  logger.error(f"/analyze: eroare pipeline — {e}", exc_info=True)
126
  raise HTTPException(status_code=500, detail="Eroare interna la procesare.")
127
 
128
+ # optional: persist to Neo4j and run cross-article verification
129
+ article_id: Optional[str] = None
130
+ cross_article_inconsistencies: list[InconsistencyResponse] = []
131
+
132
+ if req.persist:
133
+ from backend.config import NEO4J_ENABLED
134
+ if NEO4J_ENABLED:
135
+ from backend.pipeline.graph.factory import create_persistent_store
136
+ from backend.pipeline.verification.cross_article import CrossArticleVerifier
137
+
138
+ article_id = str(uuid.uuid4())
139
+ store = create_persistent_store()
140
+ if store is not None:
141
+ try:
142
+ verifier = CrossArticleVerifier(store)
143
+ cross_incs = verifier.verify(result.facts, article_id)
144
+ store.add_facts(result.facts, article_id=article_id)
145
+ cross_article_inconsistencies = [
146
+ _to_inconsistency_response(c) for c in cross_incs
147
+ ]
148
+ logger.info(
149
+ f"/analyze: persisted {len(result.facts)} facts as {article_id}, "
150
+ f"{len(cross_incs)} cross-article conflicts"
151
+ )
152
+ except Exception as e:
153
+ logger.error(f"/analyze: persistence error — {e}", exc_info=True)
154
+ finally:
155
+ store.close()
156
+ else:
157
+ logger.warning("/analyze: persist=True but NEO4J_ENABLED=false — skipping")
158
+
159
  # Genereaza explicatii structurate
160
  explanation = _explainer.explain_structured(result)
161
 
 
172
  pipeline=result.pipeline_variant,
173
  model=req.model or AVAILABLE_MODELS[req.pipeline]["default"],
174
  processing_time_ms=result.processing_time_ms,
175
+ article_id=article_id,
176
+ cross_article_inconsistencies=cross_article_inconsistencies,
177
  )
backend/routers/articles.py ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Router — GET /articles, DELETE /articles/{article_id}: article management via Neo4j."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+
7
+ from fastapi import APIRouter, HTTPException
8
+
9
+ from backend.pipeline.graph.factory import create_persistent_store
10
+
11
+ logger = logging.getLogger(__name__)
12
+ router = APIRouter(tags=["articles"])
13
+
14
+
15
+ @router.get("/articles")
16
+ async def list_articles():
17
+ """List all previously analyzed articles stored in Neo4j."""
18
+ store = create_persistent_store()
19
+ if store is None:
20
+ return {"articles": [], "neo4j_enabled": False}
21
+ try:
22
+ articles = store.get_articles()
23
+ return {"articles": articles, "neo4j_enabled": True}
24
+ except Exception as e:
25
+ logger.error(f"GET /articles error: {e}", exc_info=True)
26
+ raise HTTPException(status_code=500, detail="Failed to retrieve articles from Neo4j.")
27
+ finally:
28
+ store.close()
29
+
30
+
31
+ @router.delete("/articles/{article_id}")
32
+ async def delete_article(article_id: str):
33
+ """Delete an article and all its facts from Neo4j."""
34
+ store = create_persistent_store()
35
+ if store is None:
36
+ raise HTTPException(status_code=503, detail="Neo4j not enabled.")
37
+ try:
38
+ deleted = store.delete_article(article_id)
39
+ if not deleted:
40
+ raise HTTPException(
41
+ status_code=404, detail=f"Article '{article_id}' not found."
42
+ )
43
+ return {"deleted": True, "article_id": article_id}
44
+ except HTTPException:
45
+ raise
46
+ except Exception as e:
47
+ logger.error(f"DELETE /articles/{article_id} error: {e}", exc_info=True)
48
+ raise HTTPException(status_code=500, detail="Failed to delete article from Neo4j.")
49
+ finally:
50
+ store.close()
backend/routers/batch.py ADDED
@@ -0,0 +1,185 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Router — POST /analyze-batch: analyze multiple articles with optional Neo4j persistence."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import uuid
7
+ from datetime import datetime
8
+ from typing import Optional
9
+
10
+ from fastapi import APIRouter, HTTPException
11
+ from pydantic import BaseModel, Field
12
+
13
+ from backend.pipeline.graph.models import Article
14
+ from backend.routers.analyze import InconsistencyResponse, _to_inconsistency_response
15
+ from backend.routers.dependencies import explainer, get_orchestrator
16
+
17
+ logger = logging.getLogger(__name__)
18
+ router = APIRouter(tags=["batch"])
19
+
20
+
21
+ class BatchArticle(BaseModel):
22
+ text: str = Field(..., min_length=20)
23
+ title: str = Field(default="")
24
+ publication_date: Optional[str] = None
25
+ source: str = Field(default="")
26
+
27
+
28
+ class BatchRequest(BaseModel):
29
+ articles: list[BatchArticle] = Field(..., min_length=1, max_length=50)
30
+ pipeline: str = Field(default="spacy")
31
+ model: Optional[str] = None
32
+ persist: bool = Field(
33
+ default=True,
34
+ description="Save to Neo4j for cross-article analysis",
35
+ )
36
+
37
+
38
+ class BatchArticleResult(BaseModel):
39
+ article_id: str
40
+ title: str
41
+ score: float
42
+ label: str
43
+ summary: str
44
+ n_claims: int
45
+ n_inconsistencies: int
46
+ n_cross_article_inconsistencies: int
47
+ cross_article_conflicts: list[InconsistencyResponse]
48
+ processing_time_ms: float
49
+ error: Optional[str] = None
50
+
51
+
52
+ class BatchResponse(BaseModel):
53
+ results: list[BatchArticleResult]
54
+ total_articles: int
55
+ total_cross_article_conflicts: int
56
+ avg_score: float
57
+ neo4j_enabled: bool
58
+ persisted: bool
59
+
60
+
61
+ @router.post("/analyze-batch", response_model=BatchResponse)
62
+ async def analyze_batch(req: BatchRequest) -> BatchResponse:
63
+ """Analyze multiple articles. Persist to Neo4j and run cross-article verification when enabled."""
64
+ from backend.config import AVAILABLE_MODELS, NEO4J_ENABLED
65
+
66
+ if req.pipeline not in ("spacy", "llm"):
67
+ raise HTTPException(
68
+ status_code=400,
69
+ detail=f"Unknown pipeline '{req.pipeline}'. Options: 'spacy', 'llm'.",
70
+ )
71
+ if req.model:
72
+ available = AVAILABLE_MODELS.get(req.pipeline, {}).get("models", [])
73
+ if req.model not in available:
74
+ raise HTTPException(
75
+ status_code=400,
76
+ detail=f"Model '{req.model}' not available for pipeline '{req.pipeline}'. Options: {available}",
77
+ )
78
+
79
+ persist = req.persist and NEO4J_ENABLED
80
+
81
+ # open one shared store connection for the whole batch
82
+ store = None
83
+ if persist:
84
+ from backend.pipeline.graph.factory import create_persistent_store
85
+ store = create_persistent_store()
86
+ if store is None:
87
+ persist = False
88
+
89
+ results: list[BatchArticleResult] = []
90
+ total_cross_conflicts = 0
91
+
92
+ try:
93
+ for batch_article in req.articles:
94
+ article_id = str(uuid.uuid4())
95
+
96
+ pub_date: Optional[datetime] = None
97
+ if batch_article.publication_date:
98
+ try:
99
+ pub_date = datetime.strptime(batch_article.publication_date, "%Y-%m-%d")
100
+ except ValueError:
101
+ pass
102
+
103
+ article = Article(
104
+ text=batch_article.text,
105
+ title=batch_article.title,
106
+ publication_date=pub_date,
107
+ source=batch_article.source,
108
+ )
109
+
110
+ try:
111
+ orchestrator = get_orchestrator(req.pipeline, req.model)
112
+ result = orchestrator.run(article)
113
+ except Exception as e:
114
+ logger.error(
115
+ f"/analyze-batch: pipeline error on '{batch_article.title}': {e}",
116
+ exc_info=True,
117
+ )
118
+ results.append(
119
+ BatchArticleResult(
120
+ article_id=article_id,
121
+ title=batch_article.title,
122
+ score=0.0,
123
+ label="Error",
124
+ summary="Pipeline error — see server logs.",
125
+ n_claims=0,
126
+ n_inconsistencies=0,
127
+ n_cross_article_inconsistencies=0,
128
+ cross_article_conflicts=[],
129
+ processing_time_ms=0.0,
130
+ error=str(e)[:300],
131
+ )
132
+ )
133
+ continue
134
+
135
+ explanation = explainer.explain_structured(result)
136
+
137
+ cross_conflicts: list[InconsistencyResponse] = []
138
+ if store is not None:
139
+ from backend.pipeline.verification.cross_article import CrossArticleVerifier
140
+
141
+ try:
142
+ # verify BEFORE persisting so current article is not in the store yet
143
+ verifier = CrossArticleVerifier(store)
144
+ raw_conflicts = verifier.verify(result.facts, article_id)
145
+ store.add_facts(result.facts, article_id=article_id)
146
+ cross_conflicts = [_to_inconsistency_response(c) for c in raw_conflicts]
147
+ total_cross_conflicts += len(cross_conflicts)
148
+ except Exception as e:
149
+ logger.error(
150
+ f"/analyze-batch: persistence error for {article_id}: {e}",
151
+ exc_info=True,
152
+ )
153
+
154
+ results.append(
155
+ BatchArticleResult(
156
+ article_id=article_id,
157
+ title=batch_article.title,
158
+ score=result.score,
159
+ label=result.label,
160
+ summary=explanation["summary"],
161
+ n_claims=result.n_temporal_claims,
162
+ n_inconsistencies=result.n_inconsistencies,
163
+ n_cross_article_inconsistencies=len(cross_conflicts),
164
+ cross_article_conflicts=cross_conflicts,
165
+ processing_time_ms=result.processing_time_ms,
166
+ )
167
+ )
168
+ finally:
169
+ if store is not None:
170
+ store.close()
171
+
172
+ successful = [r for r in results if r.error is None]
173
+ avg_score = (
174
+ round(sum(r.score for r in successful) / len(successful), 4)
175
+ if successful else 0.0
176
+ )
177
+
178
+ return BatchResponse(
179
+ results=results,
180
+ total_articles=len(results),
181
+ total_cross_article_conflicts=total_cross_conflicts,
182
+ avg_score=avg_score,
183
+ neo4j_enabled=NEO4J_ENABLED,
184
+ persisted=persist,
185
+ )
backend/routers/upload.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter, UploadFile, File, HTTPException
2
+
3
+ router = APIRouter(tags=["upload"])
4
+
5
+
6
+ @router.post("/upload")
7
+ async def upload_file(file: UploadFile = File(...)) -> dict:
8
+ """Extract text from an uploaded file (.txt, .pdf, .docx)."""
9
+ filename = file.filename or ""
10
+ ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
11
+ content = await file.read()
12
+
13
+ if ext == "txt":
14
+ text = content.decode("utf-8", errors="replace")
15
+
16
+ elif ext == "pdf":
17
+ from PyPDF2 import PdfReader
18
+ import io
19
+ reader = PdfReader(io.BytesIO(content))
20
+ text = "\n\n".join(page.extract_text() or "" for page in reader.pages)
21
+
22
+ elif ext in ("docx", "doc"):
23
+ from docx import Document
24
+ import io
25
+ doc = Document(io.BytesIO(content))
26
+ text = "\n\n".join(p.text for p in doc.paragraphs if p.text.strip())
27
+
28
+ else:
29
+ raise HTTPException(
30
+ status_code=400,
31
+ detail=f"Unsupported file type: .{ext}. Use .txt, .pdf, or .docx",
32
+ )
33
+
34
+ if not text.strip():
35
+ raise HTTPException(status_code=400, detail="No text could be extracted from the file")
36
+
37
+ return {
38
+ "text": text.strip(),
39
+ "filename": filename,
40
+ "file_type": ext,
41
+ "char_count": len(text.strip()),
42
+ }
frontend/src/App.tsx CHANGED
@@ -3,13 +3,19 @@ import {
3
  AppBar,
4
  Box,
5
  Container,
 
6
  Tab,
7
  Tabs,
8
  Toolbar,
 
9
  Typography,
10
  } from "@mui/material";
11
- import { checkHealth } from "./api/client";
 
 
12
  import AnalyzeTab from "./components/AnalyzeTab";
 
 
13
  import CompareTab from "./components/CompareTab";
14
 
15
  type HealthStatus = "checking" | "ok" | "error";
@@ -17,6 +23,8 @@ type HealthStatus = "checking" | "ok" | "error";
17
  function App(): React.ReactElement {
18
  const [activeTab, setActiveTab] = useState<number>(0);
19
  const [healthStatus, setHealthStatus] = useState<HealthStatus>("checking");
 
 
20
 
21
  useEffect(() => {
22
  checkHealth()
@@ -24,6 +32,12 @@ function App(): React.ReactElement {
24
  .catch(() => setHealthStatus("error"));
25
  }, []);
26
 
 
 
 
 
 
 
27
  const healthColor: string =
28
  healthStatus === "ok" ? "#22c55e" : healthStatus === "error" ? "#ef4444" : "#94a3b8";
29
 
@@ -61,19 +75,31 @@ function App(): React.ReactElement {
61
  </Typography>
62
  </Box>
63
 
64
- <Box sx={{ display: "flex", alignItems: "center", gap: 0.75 }}>
65
- <Box
66
- sx={{
67
- width: 8,
68
- height: 8,
69
- borderRadius: "50%",
70
- bgcolor: healthColor,
71
- flexShrink: 0,
72
- }}
73
- />
74
- <Typography variant="caption" color="text.secondary">
75
- {healthLabel}
76
- </Typography>
 
 
 
 
 
 
 
 
 
 
 
 
77
  </Box>
78
  </Toolbar>
79
  </AppBar>
@@ -92,6 +118,7 @@ function App(): React.ReactElement {
92
  >
93
  <Tab label="Analyze" />
94
  <Tab label="Compare" />
 
95
  </Tabs>
96
  </Container>
97
  </Box>
@@ -100,6 +127,7 @@ function App(): React.ReactElement {
100
  <Container maxWidth="lg" sx={{ py: 3 }}>
101
  {activeTab === 0 && <AnalyzeTab />}
102
  {activeTab === 1 && <CompareTab />}
 
103
  </Container>
104
  </Box>
105
 
@@ -115,6 +143,8 @@ function App(): React.ReactElement {
115
  Bachelor Thesis — UTCN 2026 — Marius Pantea
116
  </Typography>
117
  </Box>
 
 
118
  </Box>
119
  );
120
  }
 
3
  AppBar,
4
  Box,
5
  Container,
6
+ IconButton,
7
  Tab,
8
  Tabs,
9
  Toolbar,
10
+ Tooltip,
11
  Typography,
12
  } from "@mui/material";
13
+ import { History } from "@mui/icons-material";
14
+ import { checkHealth, getModels } from "./api/client";
15
+ import type { ModelsResponse } from "./types";
16
  import AnalyzeTab from "./components/AnalyzeTab";
17
+ import ArticleHistory from "./components/ArticleHistory";
18
+ import BatchTab from "./components/BatchTab";
19
  import CompareTab from "./components/CompareTab";
20
 
21
  type HealthStatus = "checking" | "ok" | "error";
 
23
  function App(): React.ReactElement {
24
  const [activeTab, setActiveTab] = useState<number>(0);
25
  const [healthStatus, setHealthStatus] = useState<HealthStatus>("checking");
26
+ const [availableModels, setAvailableModels] = useState<ModelsResponse | undefined>(undefined);
27
+ const [historyOpen, setHistoryOpen] = useState<boolean>(false);
28
 
29
  useEffect(() => {
30
  checkHealth()
 
32
  .catch(() => setHealthStatus("error"));
33
  }, []);
34
 
35
+ useEffect(() => {
36
+ getModels()
37
+ .then(setAvailableModels)
38
+ .catch(() => undefined);
39
+ }, []);
40
+
41
  const healthColor: string =
42
  healthStatus === "ok" ? "#22c55e" : healthStatus === "error" ? "#ef4444" : "#94a3b8";
43
 
 
75
  </Typography>
76
  </Box>
77
 
78
+ <Box sx={{ display: "flex", alignItems: "center", gap: 1.5 }}>
79
+ <Box sx={{ display: "flex", alignItems: "center", gap: 0.75 }}>
80
+ <Box
81
+ sx={{
82
+ width: 8,
83
+ height: 8,
84
+ borderRadius: "50%",
85
+ bgcolor: healthColor,
86
+ flexShrink: 0,
87
+ }}
88
+ />
89
+ <Typography variant="caption" color="text.secondary">
90
+ {healthLabel}
91
+ </Typography>
92
+ </Box>
93
+
94
+ <Tooltip title="Article history (Neo4j)">
95
+ <IconButton
96
+ size="small"
97
+ onClick={() => setHistoryOpen(true)}
98
+ aria-label="Open article history"
99
+ >
100
+ <History fontSize="small" />
101
+ </IconButton>
102
+ </Tooltip>
103
  </Box>
104
  </Toolbar>
105
  </AppBar>
 
118
  >
119
  <Tab label="Analyze" />
120
  <Tab label="Compare" />
121
+ <Tab label="Batch" />
122
  </Tabs>
123
  </Container>
124
  </Box>
 
127
  <Container maxWidth="lg" sx={{ py: 3 }}>
128
  {activeTab === 0 && <AnalyzeTab />}
129
  {activeTab === 1 && <CompareTab />}
130
+ {activeTab === 2 && <BatchTab availableModels={availableModels} />}
131
  </Container>
132
  </Box>
133
 
 
143
  Bachelor Thesis — UTCN 2026 — Marius Pantea
144
  </Typography>
145
  </Box>
146
+
147
+ <ArticleHistory open={historyOpen} onClose={() => setHistoryOpen(false)} />
148
  </Box>
149
  );
150
  }
frontend/src/api/client.ts CHANGED
@@ -1,7 +1,13 @@
1
  /* client.ts — Axios instance and typed API functions */
2
 
3
  import axios from "axios";
4
- import type { AnalyzeRequest, AnalyzeResponse, CompareRequest, CompareResponse, HealthResponse, ModelsResponse } from "../types";
 
 
 
 
 
 
5
 
6
  // axios instance
7
  const apiClient = axios.create({
@@ -33,4 +39,35 @@ export async function getModels(): Promise<ModelsResponse> {
33
  return data;
34
  }
35
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
36
  export default apiClient;
 
1
  /* client.ts — Axios instance and typed API functions */
2
 
3
  import axios from "axios";
4
+ import type {
5
+ AnalyzeRequest, AnalyzeResponse,
6
+ ArticlesResponse,
7
+ BatchRequest, BatchResponse,
8
+ CompareRequest, CompareResponse,
9
+ HealthResponse, ModelsResponse,
10
+ } from "../types";
11
 
12
  // axios instance
13
  const apiClient = axios.create({
 
39
  return data;
40
  }
41
 
42
+ export async function analyzeBatch(req: BatchRequest): Promise<BatchResponse> {
43
+ const { data } = await apiClient.post<BatchResponse>("/analyze-batch", req);
44
+ return data;
45
+ }
46
+
47
+ export async function getArticles(): Promise<ArticlesResponse> {
48
+ const { data } = await apiClient.get<ArticlesResponse>("/articles");
49
+ return data;
50
+ }
51
+
52
+ export async function deleteArticle(articleId: string): Promise<void> {
53
+ await apiClient.delete(`/articles/${encodeURIComponent(articleId)}`);
54
+ }
55
+
56
+ export interface UploadResponse {
57
+ text: string;
58
+ filename: string;
59
+ file_type: string;
60
+ char_count: number;
61
+ }
62
+
63
+ export async function uploadFile(file: File): Promise<UploadResponse> {
64
+ const formData = new FormData();
65
+ formData.append("file", file);
66
+ const { data } = await apiClient.post<UploadResponse>("/upload", formData, {
67
+ headers: { "Content-Type": "multipart/form-data" },
68
+ timeout: 30_000,
69
+ });
70
+ return data;
71
+ }
72
+
73
  export default apiClient;
frontend/src/components/AnalyzeTab.tsx CHANGED
@@ -1,5 +1,5 @@
1
  import React, { useEffect, useState } from "react";
2
- import { Alert, Stack } from "@mui/material";
3
  import axios from "axios";
4
  import type { AnalyzeRequest, AnalyzeResponse, ModelsResponse } from "../types";
5
  import { analyzeArticle, getModels } from "../api/client";
@@ -16,6 +16,7 @@ function AnalyzeTab(): React.ReactElement {
16
  const [isLoading, setIsLoading] = useState<boolean>(false);
17
  const [error, setError] = useState<string | null>(null);
18
  const [availableModels, setAvailableModels] = useState<ModelsResponse | undefined>(undefined);
 
19
 
20
  useEffect(() => {
21
  getModels()
@@ -28,7 +29,7 @@ function AnalyzeTab(): React.ReactElement {
28
  setError(null);
29
  setArticleText(request.text);
30
  try {
31
- const result = await analyzeArticle(request);
32
  setAnalyzeResult(result);
33
  } catch (err: unknown) {
34
  if (axios.isAxiosError(err)) {
@@ -44,6 +45,8 @@ function AnalyzeTab(): React.ReactElement {
44
  }
45
  };
46
 
 
 
47
  return (
48
  <Stack spacing={3}>
49
  <ArticleInput
@@ -53,6 +56,22 @@ function AnalyzeTab(): React.ReactElement {
53
  availableModels={availableModels}
54
  />
55
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
  {error && (
57
  <Alert severity="error" onClose={() => setError(null)}>
58
  {error}
@@ -72,6 +91,44 @@ function AnalyzeTab(): React.ReactElement {
72
  model={analyzeResult.model}
73
  processingTimeMs={analyzeResult.processing_time_ms}
74
  />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
75
  <TextHighlight
76
  text={articleText}
77
  annotations={analyzeResult.fact_annotations}
 
1
  import React, { useEffect, useState } from "react";
2
+ import { Alert, Box, Checkbox, Chip, FormControlLabel, Stack, Typography } from "@mui/material";
3
  import axios from "axios";
4
  import type { AnalyzeRequest, AnalyzeResponse, ModelsResponse } from "../types";
5
  import { analyzeArticle, getModels } from "../api/client";
 
16
  const [isLoading, setIsLoading] = useState<boolean>(false);
17
  const [error, setError] = useState<string | null>(null);
18
  const [availableModels, setAvailableModels] = useState<ModelsResponse | undefined>(undefined);
19
+ const [persist, setPersist] = useState<boolean>(false);
20
 
21
  useEffect(() => {
22
  getModels()
 
29
  setError(null);
30
  setArticleText(request.text);
31
  try {
32
+ const result = await analyzeArticle({ ...request, persist });
33
  setAnalyzeResult(result);
34
  } catch (err: unknown) {
35
  if (axios.isAxiosError(err)) {
 
45
  }
46
  };
47
 
48
+ const crossArticleConflicts = analyzeResult?.cross_article_inconsistencies ?? [];
49
+
50
  return (
51
  <Stack spacing={3}>
52
  <ArticleInput
 
56
  availableModels={availableModels}
57
  />
58
 
59
+ <FormControlLabel
60
+ sx={{ mt: -1 }}
61
+ control={
62
+ <Checkbox
63
+ size="small"
64
+ checked={persist}
65
+ onChange={(e) => setPersist(e.target.checked)}
66
+ />
67
+ }
68
+ label={
69
+ <Typography variant="body2">
70
+ Save to Neo4j for cross-article analysis
71
+ </Typography>
72
+ }
73
+ />
74
+
75
  {error && (
76
  <Alert severity="error" onClose={() => setError(null)}>
77
  {error}
 
91
  model={analyzeResult.model}
92
  processingTimeMs={analyzeResult.processing_time_ms}
93
  />
94
+
95
+ {/* Cross-article conflicts banner */}
96
+ {crossArticleConflicts.length > 0 && (
97
+ <Alert
98
+ severity="warning"
99
+ action={
100
+ <Chip
101
+ label={`${crossArticleConflicts.length} conflict${crossArticleConflicts.length !== 1 ? "s" : ""}`}
102
+ size="small"
103
+ color="warning"
104
+ />
105
+ }
106
+ >
107
+ <Box>
108
+ <Typography variant="body2" sx={{ fontWeight: "medium", mb: 0.5 }}>
109
+ Cross-article conflicts detected
110
+ </Typography>
111
+ {crossArticleConflicts.map((c, i) => (
112
+ <Typography key={i} variant="caption" sx={{ display: "block" }}>
113
+ • {c.description}
114
+ </Typography>
115
+ ))}
116
+ {analyzeResult.article_id && (
117
+ <Typography variant="caption" color="text.secondary" sx={{ mt: 0.5, display: "block" }}>
118
+ Saved as {analyzeResult.article_id}
119
+ </Typography>
120
+ )}
121
+ </Box>
122
+ </Alert>
123
+ )}
124
+
125
+ {/* Saved without conflicts */}
126
+ {analyzeResult.article_id && crossArticleConflicts.length === 0 && (
127
+ <Alert severity="success" sx={{ py: 0.5 }}>
128
+ Saved to Neo4j — no cross-article conflicts found.
129
+ </Alert>
130
+ )}
131
+
132
  <TextHighlight
133
  text={articleText}
134
  annotations={analyzeResult.fact_annotations}
frontend/src/components/ArticleHistory.tsx ADDED
@@ -0,0 +1,213 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import React, { useCallback, useEffect, useState } from "react";
2
+ import {
3
+ Alert,
4
+ Box,
5
+ CircularProgress,
6
+ Divider,
7
+ Drawer,
8
+ IconButton,
9
+ List,
10
+ ListItem,
11
+ ListItemSecondaryAction,
12
+ ListItemText,
13
+ Tooltip,
14
+ Typography,
15
+ } from "@mui/material";
16
+ import { Close, Delete } from "@mui/icons-material";
17
+ import type { StoredArticle } from "../types";
18
+ import { deleteArticle, getArticles } from "../api/client";
19
+
20
+ interface ArticleHistoryProps {
21
+ open: boolean;
22
+ onClose: () => void;
23
+ }
24
+
25
+ function ArticleHistory({ open, onClose }: ArticleHistoryProps): React.ReactElement {
26
+ const [articles, setArticles] = useState<StoredArticle[]>([]);
27
+ const [neo4jEnabled, setNeo4jEnabled] = useState<boolean>(true);
28
+ const [isLoading, setIsLoading] = useState<boolean>(false);
29
+ const [deletingId, setDeletingId] = useState<string | null>(null);
30
+ const [error, setError] = useState<string | null>(null);
31
+
32
+ const fetchArticles = useCallback(async (): Promise<void> => {
33
+ setIsLoading(true);
34
+ setError(null);
35
+ try {
36
+ const response = await getArticles();
37
+ setNeo4jEnabled(response.neo4j_enabled);
38
+ setArticles(response.articles);
39
+ } catch {
40
+ setError("Failed to load article history.");
41
+ } finally {
42
+ setIsLoading(false);
43
+ }
44
+ }, []);
45
+
46
+ useEffect(() => {
47
+ if (open) {
48
+ void fetchArticles();
49
+ }
50
+ }, [open, fetchArticles]);
51
+
52
+ const handleDelete = async (articleId: string): Promise<void> => {
53
+ setDeletingId(articleId);
54
+ setError(null);
55
+ try {
56
+ await deleteArticle(articleId);
57
+ setArticles((prev) => prev.filter((a) => a.article_id !== articleId));
58
+ } catch {
59
+ setError(`Failed to delete article ${articleId}.`);
60
+ } finally {
61
+ setDeletingId(null);
62
+ }
63
+ };
64
+
65
+ const formatDate = (raw: string | null): string => {
66
+ if (!raw) return "Unknown date";
67
+ try {
68
+ return new Date(raw).toLocaleDateString(undefined, {
69
+ year: "numeric",
70
+ month: "short",
71
+ day: "numeric",
72
+ });
73
+ } catch {
74
+ return raw;
75
+ }
76
+ };
77
+
78
+ return (
79
+ <Drawer
80
+ anchor="right"
81
+ open={open}
82
+ onClose={onClose}
83
+ slotProps={{ paper: { sx: { width: 350, display: "flex", flexDirection: "column" } } }}
84
+ >
85
+ {/* Header */}
86
+ <Box
87
+ sx={{
88
+ px: 2,
89
+ py: 1.5,
90
+ display: "flex",
91
+ alignItems: "center",
92
+ justifyContent: "space-between",
93
+ borderBottom: "1px solid",
94
+ borderColor: "divider",
95
+ flexShrink: 0,
96
+ }}
97
+ >
98
+ <Typography variant="subtitle1" sx={{ fontWeight: "medium" }}>
99
+ Article History
100
+ </Typography>
101
+ <IconButton size="small" onClick={onClose} aria-label="Close history">
102
+ <Close fontSize="small" />
103
+ </IconButton>
104
+ </Box>
105
+
106
+ {/* Body */}
107
+ <Box sx={{ flex: 1, overflowY: "auto", px: 0 }}>
108
+ {!neo4jEnabled && (
109
+ <Box sx={{ p: 2 }}>
110
+ <Alert severity="info">
111
+ Neo4j not configured. Enable in <code>.env</code> to save articles.
112
+ </Alert>
113
+ </Box>
114
+ )}
115
+
116
+ {neo4jEnabled && isLoading && (
117
+ <Box sx={{ display: "flex", justifyContent: "center", pt: 4 }}>
118
+ <CircularProgress size={28} />
119
+ </Box>
120
+ )}
121
+
122
+ {neo4jEnabled && error && (
123
+ <Box sx={{ p: 2 }}>
124
+ <Alert severity="error" onClose={() => setError(null)}>
125
+ {error}
126
+ </Alert>
127
+ </Box>
128
+ )}
129
+
130
+ {neo4jEnabled && !isLoading && articles.length === 0 && !error && (
131
+ <Box sx={{ p: 2, textAlign: "center" }}>
132
+ <Typography variant="body2" color="text.secondary">
133
+ No articles saved yet. Use{" "}
134
+ <strong>Save to Neo4j</strong> when analyzing to store results.
135
+ </Typography>
136
+ </Box>
137
+ )}
138
+
139
+ {neo4jEnabled && !isLoading && articles.length > 0 && (
140
+ <List dense disablePadding>
141
+ {articles.map((article, idx) => (
142
+ <React.Fragment key={article.article_id}>
143
+ <ListItem
144
+ alignItems="flex-start"
145
+ sx={{ pr: 6 }}
146
+ >
147
+ <ListItemText
148
+ primary={
149
+ <Typography variant="body2" noWrap sx={{ fontWeight: "medium" }}>
150
+ {article.title ?? <em>Untitled</em>}
151
+ </Typography>
152
+ }
153
+ secondary={
154
+ <Box component="span" sx={{ display: "flex", flexDirection: "column", gap: 0.25 }}>
155
+ {article.source && (
156
+ <Typography variant="caption" color="text.secondary" noWrap>
157
+ {article.source}
158
+ </Typography>
159
+ )}
160
+ <Typography variant="caption" color="text.secondary">
161
+ {formatDate(article.analyzed_at)} · {article.fact_count} facts
162
+ </Typography>
163
+ </Box>
164
+ }
165
+ />
166
+ <ListItemSecondaryAction>
167
+ <Tooltip title="Delete article">
168
+ <span>
169
+ <IconButton
170
+ size="small"
171
+ edge="end"
172
+ onClick={() => void handleDelete(article.article_id)}
173
+ disabled={deletingId === article.article_id}
174
+ aria-label="Delete article"
175
+ >
176
+ {deletingId === article.article_id ? (
177
+ <CircularProgress size={16} />
178
+ ) : (
179
+ <Delete fontSize="small" />
180
+ )}
181
+ </IconButton>
182
+ </span>
183
+ </Tooltip>
184
+ </ListItemSecondaryAction>
185
+ </ListItem>
186
+ {idx < articles.length - 1 && <Divider component="li" />}
187
+ </React.Fragment>
188
+ ))}
189
+ </List>
190
+ )}
191
+ </Box>
192
+
193
+ {/* Footer */}
194
+ {neo4jEnabled && articles.length > 0 && (
195
+ <Box
196
+ sx={{
197
+ px: 2,
198
+ py: 1,
199
+ borderTop: "1px solid",
200
+ borderColor: "divider",
201
+ flexShrink: 0,
202
+ }}
203
+ >
204
+ <Typography variant="caption" color="text.secondary">
205
+ {articles.length} article{articles.length !== 1 ? "s" : ""} stored
206
+ </Typography>
207
+ </Box>
208
+ )}
209
+ </Drawer>
210
+ );
211
+ }
212
+
213
+ export default ArticleHistory;
frontend/src/components/ArticleInput.tsx CHANGED
@@ -5,6 +5,7 @@ import {
5
  AccordionSummary,
6
  Box,
7
  Button,
 
8
  CircularProgress,
9
  FormControl,
10
  FormControlLabel,
@@ -24,6 +25,7 @@ import type { SelectChangeEvent } from "@mui/material";
24
  import { Clear, ExpandMore, PlayArrow, Upload } from "@mui/icons-material";
25
  import type { AnalyzeRequest, ModelsResponse } from "../types";
26
  import { getModelDescription, getModelLabel } from "../utils/modelLabels";
 
27
 
28
  // Fields that ArticleInput owns (text, metadata — no pipeline/model)
29
  type ArticleFields = Omit<AnalyzeRequest, "pipeline" | "model">;
@@ -47,9 +49,11 @@ function ArticleInput({
47
  hideSubmitButton = false,
48
  onRequestChange,
49
  }: ArticleInputProps): React.ReactElement {
50
- const [inputMode, setInputMode] = useState<"text" | "file" | "url">("text");
51
  const [textValue, setTextValue] = useState<string>("");
52
- const [fileName, setFileName] = useState<string | null>(null);
 
 
53
  const [title, setTitle] = useState<string>("");
54
  const [publicationDate, setPublicationDate] = useState<string>("");
55
  const [source, setSource] = useState<string>("");
@@ -83,41 +87,37 @@ function ArticleInput({
83
  newMode: string | null,
84
  ): void => {
85
  if (newMode !== null) {
86
- setInputMode(newMode as "text" | "file" | "url");
87
  }
88
  };
89
 
90
- const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>): void => {
91
  const file = e.target.files?.[0];
92
  if (!file) return;
93
- setFileName(file.name);
94
- const reader = new FileReader();
95
- reader.onload = (event): void => {
96
- const content = event.target?.result as string;
97
- if (file.name.endsWith(".json")) {
98
- try {
99
- const parsed = JSON.parse(content) as Record<string, unknown>;
100
- if (typeof parsed.text === "string") setTextValue(parsed.text);
101
- if (typeof parsed.title === "string") setTitle(parsed.title);
102
- if (typeof parsed.source === "string") setSource(parsed.source);
103
- if (typeof parsed.publication_date === "string")
104
- setPublicationDate(parsed.publication_date);
105
- } catch {
106
- setTextValue(content);
107
- }
108
- } else {
109
- setTextValue(content);
110
- }
111
- };
112
- reader.readAsText(file);
113
  };
114
 
115
  const handleClearFile = (): void => {
116
- setFileName(null);
 
117
  setTextValue("");
118
- if (fileInputRef.current) {
119
- fileInputRef.current.value = "";
120
- }
121
  };
122
 
123
  const handleSubmit = (): void => {
@@ -143,7 +143,6 @@ function ArticleInput({
143
  >
144
  <ToggleButton value="text">Paste Text</ToggleButton>
145
  <ToggleButton value="file">Upload File</ToggleButton>
146
- <ToggleButton value="url">URL</ToggleButton>
147
  </ToggleButtonGroup>
148
 
149
  {inputMode === "text" && (
@@ -157,13 +156,20 @@ function ArticleInput({
157
  onChange={(e) => setTextValue(e.target.value)}
158
  sx={{ "& .MuiInputBase-input": { fontFamily: "IBM Plex Mono, monospace" } }}
159
  />
160
- <Typography
161
- variant="caption"
162
- color="text.secondary"
163
- sx={{ mt: 0.5, display: "block" }}
164
- >
165
- {textValue.length} characters
166
- </Typography>
 
 
 
 
 
 
 
167
  </Box>
168
  )}
169
 
@@ -171,7 +177,7 @@ function ArticleInput({
171
  <Box sx={{ display: "flex", flexDirection: "column", gap: 1.5 }}>
172
  <input
173
  type="file"
174
- accept=".txt,.json"
175
  ref={fileInputRef}
176
  onChange={handleFileChange}
177
  style={{ display: "none" }}
@@ -179,44 +185,34 @@ function ArticleInput({
179
  <Box sx={{ display: "flex", alignItems: "center", gap: 1.5 }}>
180
  <Button
181
  variant="outlined"
182
- startIcon={<Upload />}
183
- onClick={() => fileInputRef.current?.click()}
 
184
  >
185
- Choose File
186
  </Button>
187
- {fileName && (
188
- <>
189
- <Typography variant="body2" sx={{ flex: 1 }}>
190
- {fileName}
191
- </Typography>
192
- <Button
193
- size="small"
194
- color="inherit"
195
- startIcon={<Clear />}
196
- onClick={handleClearFile}
197
- >
198
- Clear
199
- </Button>
200
- </>
201
  )}
202
  </Box>
203
- {textValue && (
204
- <Typography variant="caption" color="text.secondary">
205
- {textValue.length} characters loaded
206
  </Typography>
207
  )}
 
 
 
208
  </Box>
209
  )}
210
 
211
- {inputMode === "url" && (
212
- <TextField
213
- fullWidth
214
- disabled
215
- placeholder="https://..."
216
- helperText="Coming soon — requires RSS/API integration"
217
- />
218
- )}
219
-
220
  <Accordion
221
  disableGutters
222
  elevation={0}
 
5
  AccordionSummary,
6
  Box,
7
  Button,
8
+ Chip,
9
  CircularProgress,
10
  FormControl,
11
  FormControlLabel,
 
25
  import { Clear, ExpandMore, PlayArrow, Upload } from "@mui/icons-material";
26
  import type { AnalyzeRequest, ModelsResponse } from "../types";
27
  import { getModelDescription, getModelLabel } from "../utils/modelLabels";
28
+ import { uploadFile } from "../api/client";
29
 
30
  // Fields that ArticleInput owns (text, metadata — no pipeline/model)
31
  type ArticleFields = Omit<AnalyzeRequest, "pipeline" | "model">;
 
49
  hideSubmitButton = false,
50
  onRequestChange,
51
  }: ArticleInputProps): React.ReactElement {
52
+ const [inputMode, setInputMode] = useState<"text" | "file">("text");
53
  const [textValue, setTextValue] = useState<string>("");
54
+ const [isUploading, setIsUploading] = useState<boolean>(false);
55
+ const [uploadError, setUploadError] = useState<string | null>(null);
56
+ const [extractedInfo, setExtractedInfo] = useState<{ filename: string; char_count: number } | null>(null);
57
  const [title, setTitle] = useState<string>("");
58
  const [publicationDate, setPublicationDate] = useState<string>("");
59
  const [source, setSource] = useState<string>("");
 
87
  newMode: string | null,
88
  ): void => {
89
  if (newMode !== null) {
90
+ setInputMode(newMode as "text" | "file");
91
  }
92
  };
93
 
94
+ const handleFileChange = async (e: React.ChangeEvent<HTMLInputElement>): Promise<void> => {
95
  const file = e.target.files?.[0];
96
  if (!file) return;
97
+ setUploadError(null);
98
+ setExtractedInfo(null);
99
+ setIsUploading(true);
100
+ try {
101
+ const result = await uploadFile(file);
102
+ setTextValue(result.text);
103
+ setExtractedInfo({ filename: result.filename, char_count: result.char_count });
104
+ setInputMode("text");
105
+ } catch (err: unknown) {
106
+ const msg =
107
+ (err as { response?: { data?: { detail?: string } } })?.response?.data?.detail ??
108
+ "Upload failed";
109
+ setUploadError(msg);
110
+ } finally {
111
+ setIsUploading(false);
112
+ if (fileInputRef.current) fileInputRef.current.value = "";
113
+ }
 
 
 
114
  };
115
 
116
  const handleClearFile = (): void => {
117
+ setExtractedInfo(null);
118
+ setUploadError(null);
119
  setTextValue("");
120
+ if (fileInputRef.current) fileInputRef.current.value = "";
 
 
121
  };
122
 
123
  const handleSubmit = (): void => {
 
143
  >
144
  <ToggleButton value="text">Paste Text</ToggleButton>
145
  <ToggleButton value="file">Upload File</ToggleButton>
 
146
  </ToggleButtonGroup>
147
 
148
  {inputMode === "text" && (
 
156
  onChange={(e) => setTextValue(e.target.value)}
157
  sx={{ "& .MuiInputBase-input": { fontFamily: "IBM Plex Mono, monospace" } }}
158
  />
159
+ <Box sx={{ mt: 0.5, display: "flex", alignItems: "center", gap: 1, flexWrap: "wrap" }}>
160
+ <Typography variant="caption" color="text.secondary">
161
+ {textValue.length} characters
162
+ </Typography>
163
+ {extractedInfo && (
164
+ <Chip
165
+ size="small"
166
+ label={`Extracted from: ${extractedInfo.filename} (${extractedInfo.char_count} chars)`}
167
+ onDelete={handleClearFile}
168
+ color="info"
169
+ variant="outlined"
170
+ />
171
+ )}
172
+ </Box>
173
  </Box>
174
  )}
175
 
 
177
  <Box sx={{ display: "flex", flexDirection: "column", gap: 1.5 }}>
178
  <input
179
  type="file"
180
+ accept=".txt,.pdf,.docx,.doc"
181
  ref={fileInputRef}
182
  onChange={handleFileChange}
183
  style={{ display: "none" }}
 
185
  <Box sx={{ display: "flex", alignItems: "center", gap: 1.5 }}>
186
  <Button
187
  variant="outlined"
188
+ startIcon={isUploading ? <CircularProgress size={18} color="inherit" /> : <Upload />}
189
+ onClick={() => !isUploading && fileInputRef.current?.click()}
190
+ disabled={isUploading}
191
  >
192
+ {isUploading ? "Extracting..." : "Choose File"}
193
  </Button>
194
+ {!isUploading && extractedInfo && (
195
+ <Button
196
+ size="small"
197
+ color="inherit"
198
+ startIcon={<Clear />}
199
+ onClick={handleClearFile}
200
+ >
201
+ Clear
202
+ </Button>
 
 
 
 
 
203
  )}
204
  </Box>
205
+ {uploadError && (
206
+ <Typography variant="caption" color="error">
207
+ {uploadError}
208
  </Typography>
209
  )}
210
+ <Typography variant="caption" color="text.secondary">
211
+ Supported formats: .txt, .pdf, .docx
212
+ </Typography>
213
  </Box>
214
  )}
215
 
 
 
 
 
 
 
 
 
 
216
  <Accordion
217
  disableGutters
218
  elevation={0}
frontend/src/components/BatchTab.tsx ADDED
@@ -0,0 +1,506 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import React, { useRef, useState } from "react";
2
+ import {
3
+ Alert,
4
+ Box,
5
+ Button,
6
+ Checkbox,
7
+ Chip,
8
+ CircularProgress,
9
+ Collapse,
10
+ FormControl,
11
+ FormControlLabel,
12
+ IconButton,
13
+ InputLabel,
14
+ LinearProgress,
15
+ MenuItem,
16
+ Paper,
17
+ Select,
18
+ Stack,
19
+ Table,
20
+ TableBody,
21
+ TableCell,
22
+ TableHead,
23
+ TableRow,
24
+ TableSortLabel,
25
+ TextField,
26
+ Typography,
27
+ } from "@mui/material";
28
+ import type { SelectChangeEvent } from "@mui/material";
29
+ import { Add, Delete, PlayArrow } from "@mui/icons-material";
30
+ import axios from "axios";
31
+ import type { BatchArticleResult, BatchResponse, ModelsResponse } from "../types";
32
+ import { analyzeBatch } from "../api/client";
33
+ import { getModelLabel } from "../utils/modelLabels";
34
+
35
+ interface BatchTabProps {
36
+ availableModels?: ModelsResponse;
37
+ }
38
+
39
+ interface BatchRow {
40
+ id: number;
41
+ text: string;
42
+ title: string;
43
+ }
44
+
45
+ type SortField = "score" | "n_claims" | "n_inconsistencies" | "n_cross_article_inconsistencies" | "processing_time_ms";
46
+
47
+ function scoreColor(score: number): "success" | "warning" | "default" | "error" {
48
+ if (score >= 0.8) return "success";
49
+ if (score >= 0.5) return "warning";
50
+ if (score >= 0.2) return "default";
51
+ return "error";
52
+ }
53
+
54
+ const MAX_ROWS = 20;
55
+ let _rowIdCounter = 0;
56
+ function nextId(): number {
57
+ return ++_rowIdCounter;
58
+ }
59
+
60
+ function BatchTab({ availableModels }: BatchTabProps): React.ReactElement {
61
+ const [rows, setRows] = useState<BatchRow[]>([
62
+ { id: nextId(), text: "", title: "" },
63
+ { id: nextId(), text: "", title: "" },
64
+ ]);
65
+ const [pipeline, setPipeline] = useState<"spacy" | "llm">("spacy");
66
+ const [model, setModel] = useState<string>("");
67
+ const [persist, setPersist] = useState<boolean>(true);
68
+ const [isLoading, setIsLoading] = useState<boolean>(false);
69
+ const [result, setResult] = useState<BatchResponse | null>(null);
70
+ const [error, setError] = useState<string | null>(null);
71
+ const [sortField, setSortField] = useState<SortField>("score");
72
+ const [sortDir, setSortDir] = useState<"asc" | "desc">("asc");
73
+ const [expandedId, setExpandedId] = useState<string | null>(null);
74
+
75
+ const pipelineMeta = availableModels
76
+ ? pipeline === "llm"
77
+ ? availableModels.llm
78
+ : availableModels.spacy
79
+ : null;
80
+
81
+ const addRow = (): void => {
82
+ if (rows.length >= MAX_ROWS) return;
83
+ setRows((prev) => [...prev, { id: nextId(), text: "", title: "" }]);
84
+ };
85
+
86
+ const removeRow = (id: number): void => {
87
+ if (rows.length <= 1) return;
88
+ setRows((prev) => prev.filter((r) => r.id !== id));
89
+ };
90
+
91
+ const updateRow = (id: number, field: "text" | "title", value: string): void => {
92
+ setRows((prev) =>
93
+ prev.map((r) => (r.id === id ? { ...r, [field]: value } : r))
94
+ );
95
+ };
96
+
97
+ const handlePipelineChange = (e: SelectChangeEvent<string>): void => {
98
+ const p = e.target.value as "spacy" | "llm";
99
+ setPipeline(p);
100
+ if (availableModels) {
101
+ setModel(p === "llm" ? availableModels.llm.default : availableModels.spacy.default);
102
+ }
103
+ };
104
+
105
+ const handleSort = (field: SortField): void => {
106
+ if (sortField === field) {
107
+ setSortDir((d) => (d === "asc" ? "desc" : "asc"));
108
+ } else {
109
+ setSortField(field);
110
+ setSortDir("asc");
111
+ }
112
+ };
113
+
114
+ const validRows = rows.filter((r) => r.text.trim().length >= 20);
115
+
116
+ const handleSubmit = async (): Promise<void> => {
117
+ if (validRows.length === 0) return;
118
+ setIsLoading(true);
119
+ setError(null);
120
+ setResult(null);
121
+ setExpandedId(null);
122
+
123
+ try {
124
+ const response = await analyzeBatch({
125
+ articles: validRows.map((r) => ({
126
+ text: r.text.trim(),
127
+ title: r.title.trim() || undefined,
128
+ })),
129
+ pipeline,
130
+ model: model || undefined,
131
+ persist,
132
+ });
133
+ setResult(response);
134
+ } catch (err: unknown) {
135
+ if (axios.isAxiosError(err)) {
136
+ const detail = err.response?.data?.detail as string | undefined;
137
+ setError(detail ?? err.message);
138
+ } else if (err instanceof Error) {
139
+ setError(err.message);
140
+ } else {
141
+ setError("An unexpected error occurred");
142
+ }
143
+ } finally {
144
+ setIsLoading(false);
145
+ }
146
+ };
147
+
148
+ const sortedResults: BatchArticleResult[] = result
149
+ ? [...result.results].sort((a, b) => {
150
+ const aVal = a[sortField] as number;
151
+ const bVal = b[sortField] as number;
152
+ return sortDir === "asc" ? aVal - bVal : bVal - aVal;
153
+ })
154
+ : [];
155
+
156
+ return (
157
+ <Stack spacing={3}>
158
+ {/* Input section */}
159
+ <Paper variant="outlined" sx={{ p: 2.5 }}>
160
+ <Typography variant="subtitle2" sx={{ mb: 2 }}>
161
+ Articles
162
+ </Typography>
163
+
164
+ <Stack spacing={2}>
165
+ {rows.map((row, idx) => (
166
+ <Box key={row.id} sx={{ display: "flex", gap: 1.5, alignItems: "flex-start" }}>
167
+ <Typography
168
+ variant="caption"
169
+ color="text.secondary"
170
+ sx={{ pt: 1.5, minWidth: 20, textAlign: "right" }}
171
+ >
172
+ {idx + 1}
173
+ </Typography>
174
+ <Box sx={{ flex: 1, display: "flex", flexDirection: "column", gap: 1 }}>
175
+ <TextField
176
+ size="small"
177
+ placeholder="Article title (optional)"
178
+ value={row.title}
179
+ onChange={(e) => updateRow(row.id, "title", e.target.value)}
180
+ fullWidth
181
+ />
182
+ <TextField
183
+ multiline
184
+ minRows={4}
185
+ fullWidth
186
+ placeholder="Paste article text here... (min 20 characters)"
187
+ value={row.text}
188
+ onChange={(e) => updateRow(row.id, "text", e.target.value)}
189
+ error={row.text.length > 0 && row.text.trim().length < 20}
190
+ helperText={row.text.length > 0 ? `${row.text.length} characters` : undefined}
191
+ sx={{ "& .MuiInputBase-input": { fontFamily: "IBM Plex Mono, monospace", fontSize: 13 } }}
192
+ />
193
+ </Box>
194
+ <IconButton
195
+ size="small"
196
+ onClick={() => removeRow(row.id)}
197
+ disabled={rows.length <= 1}
198
+ sx={{ mt: 0.5 }}
199
+ >
200
+ <Delete fontSize="small" />
201
+ </IconButton>
202
+ </Box>
203
+ ))}
204
+
205
+ <Box>
206
+ <Button
207
+ size="small"
208
+ startIcon={<Add />}
209
+ onClick={addRow}
210
+ disabled={rows.length >= MAX_ROWS}
211
+ >
212
+ Add Article {rows.length >= MAX_ROWS ? `(max ${MAX_ROWS})` : ""}
213
+ </Button>
214
+ </Box>
215
+ </Stack>
216
+
217
+ <Box
218
+ sx={{
219
+ mt: 2.5,
220
+ pt: 2,
221
+ borderTop: "1px solid",
222
+ borderColor: "divider",
223
+ display: "flex",
224
+ gap: 2,
225
+ flexWrap: "wrap",
226
+ alignItems: "center",
227
+ }}
228
+ >
229
+ <FormControl size="small" sx={{ minWidth: 130 }}>
230
+ <InputLabel>Pipeline</InputLabel>
231
+ <Select value={pipeline} label="Pipeline" onChange={handlePipelineChange}>
232
+ <MenuItem value="spacy">spaCy</MenuItem>
233
+ <MenuItem value="llm">LLM</MenuItem>
234
+ </Select>
235
+ </FormControl>
236
+
237
+ {pipelineMeta && (
238
+ <FormControl size="small" sx={{ minWidth: 180 }}>
239
+ <InputLabel>Model</InputLabel>
240
+ <Select
241
+ value={model || pipelineMeta.default}
242
+ label="Model"
243
+ onChange={(e: SelectChangeEvent<string>) => setModel(e.target.value)}
244
+ >
245
+ {pipelineMeta.models.map((m) => (
246
+ <MenuItem key={m} value={m}>
247
+ {getModelLabel(m)}
248
+ </MenuItem>
249
+ ))}
250
+ </Select>
251
+ </FormControl>
252
+ )}
253
+
254
+ <FormControlLabel
255
+ control={
256
+ <Checkbox
257
+ size="small"
258
+ checked={persist}
259
+ onChange={(e) => setPersist(e.target.checked)}
260
+ />
261
+ }
262
+ label={
263
+ <Typography variant="body2">Save to Neo4j for cross-article analysis</Typography>
264
+ }
265
+ />
266
+
267
+ <Box sx={{ ml: "auto" }}>
268
+ <Button
269
+ variant="contained"
270
+ startIcon={
271
+ isLoading ? (
272
+ <CircularProgress size={18} color="inherit" />
273
+ ) : (
274
+ <PlayArrow />
275
+ )
276
+ }
277
+ disabled={validRows.length === 0 || isLoading}
278
+ onClick={handleSubmit}
279
+ >
280
+ {isLoading ? `Analyzing ${validRows.length} articles…` : "Analyze Batch"}
281
+ </Button>
282
+ </Box>
283
+ </Box>
284
+ </Paper>
285
+
286
+ {/* Loading indicator */}
287
+ {isLoading && (
288
+ <Box>
289
+ <LinearProgress />
290
+ <Typography variant="caption" color="text.secondary" sx={{ mt: 0.5, display: "block" }} component="p">
291
+ Analyzing {validRows.length} article{validRows.length !== 1 ? "s" : ""}…
292
+ </Typography>
293
+ </Box>
294
+ )}
295
+
296
+ {error && (
297
+ <Alert severity="error" onClose={() => setError(null)}>
298
+ {error}
299
+ </Alert>
300
+ )}
301
+
302
+ {/* Results section */}
303
+ {result && (
304
+ <Paper variant="outlined" sx={{ p: 2.5 }}>
305
+ {/* Aggregate stats */}
306
+ <Box sx={{ display: "flex", gap: 3, flexWrap: "wrap", mb: 2.5 }}>
307
+ <StatBox label="Articles" value={result.total_articles} />
308
+ <StatBox
309
+ label="Mean TCS"
310
+ value={result.avg_score.toFixed(3)}
311
+ color={
312
+ result.avg_score >= 0.8
313
+ ? "success.main"
314
+ : result.avg_score >= 0.5
315
+ ? "warning.main"
316
+ : "error.main"
317
+ }
318
+ />
319
+ <StatBox
320
+ label="Total claims"
321
+ value={result.results.reduce((s, r) => s + r.n_claims, 0)}
322
+ />
323
+ <StatBox
324
+ label="Inconsistencies"
325
+ value={result.results.reduce((s, r) => s + r.n_inconsistencies, 0)}
326
+ />
327
+ <StatBox label="Cross-article conflicts" value={result.total_cross_article_conflicts} />
328
+ {result.persisted && (
329
+ <Chip label="Saved to Neo4j" size="small" color="success" variant="outlined" />
330
+ )}
331
+ </Box>
332
+
333
+ {/* Results table */}
334
+ <Table size="small">
335
+ <TableHead>
336
+ <TableRow>
337
+ <TableCell sx={{ width: 36 }}>#</TableCell>
338
+ <TableCell>Title</TableCell>
339
+ <SortHeaderCell field="score" active={sortField} dir={sortDir} onSort={handleSort}>
340
+ TCS Score
341
+ </SortHeaderCell>
342
+ <SortHeaderCell field="n_claims" active={sortField} dir={sortDir} onSort={handleSort}>
343
+ Claims
344
+ </SortHeaderCell>
345
+ <SortHeaderCell field="n_inconsistencies" active={sortField} dir={sortDir} onSort={handleSort}>
346
+ Inconsistencies
347
+ </SortHeaderCell>
348
+ <SortHeaderCell
349
+ field="n_cross_article_inconsistencies"
350
+ active={sortField}
351
+ dir={sortDir}
352
+ onSort={handleSort}
353
+ >
354
+ Cross-Article
355
+ </SortHeaderCell>
356
+ <SortHeaderCell
357
+ field="processing_time_ms"
358
+ active={sortField}
359
+ dir={sortDir}
360
+ onSort={handleSort}
361
+ >
362
+ Time
363
+ </SortHeaderCell>
364
+ </TableRow>
365
+ </TableHead>
366
+ <TableBody>
367
+ {sortedResults.map((row, idx) => (
368
+ <React.Fragment key={row.article_id}>
369
+ <TableRow
370
+ hover
371
+ sx={{ cursor: "pointer" }}
372
+ onClick={() =>
373
+ setExpandedId((prev) =>
374
+ prev === row.article_id ? null : row.article_id
375
+ )
376
+ }
377
+ >
378
+ <TableCell sx={{ color: "text.secondary" }}>{idx + 1}</TableCell>
379
+ <TableCell sx={{ maxWidth: 220 }}>
380
+ <Typography variant="body2" noWrap title={row.title || row.article_id}>
381
+ {row.title || <em style={{ opacity: 0.6 }}>Untitled</em>}
382
+ </Typography>
383
+ {row.error && (
384
+ <Typography variant="caption" color="error">
385
+ Error
386
+ </Typography>
387
+ )}
388
+ </TableCell>
389
+ <TableCell>
390
+ <Chip
391
+ label={row.score.toFixed(3)}
392
+ size="small"
393
+ color={scoreColor(row.score)}
394
+ />
395
+ </TableCell>
396
+ <TableCell>{row.n_claims}</TableCell>
397
+ <TableCell>{row.n_inconsistencies}</TableCell>
398
+ <TableCell>
399
+ {row.n_cross_article_inconsistencies > 0 ? (
400
+ <Chip
401
+ label={row.n_cross_article_inconsistencies}
402
+ size="small"
403
+ color="warning"
404
+ />
405
+ ) : (
406
+ <Typography variant="body2" color="text.secondary">
407
+ 0
408
+ </Typography>
409
+ )}
410
+ </TableCell>
411
+ <TableCell>
412
+ <Typography variant="body2" color="text.secondary">
413
+ {Math.round(row.processing_time_ms)}ms
414
+ </Typography>
415
+ </TableCell>
416
+ </TableRow>
417
+
418
+ {/* Expandable detail row */}
419
+ <TableRow>
420
+ <TableCell colSpan={7} sx={{ p: 0, border: 0 }}>
421
+ <Collapse in={expandedId === row.article_id} unmountOnExit>
422
+ <Box sx={{ px: 3, py: 1.5, bgcolor: "action.hover" }}>
423
+ <Typography variant="body2" sx={{ mb: 1 }}>
424
+ {row.summary}
425
+ </Typography>
426
+ {row.cross_article_conflicts.length > 0 && (
427
+ <Stack spacing={0.5}>
428
+ <Typography variant="caption" color="text.secondary">
429
+ Cross-article conflicts:
430
+ </Typography>
431
+ {row.cross_article_conflicts.map((c, i) => (
432
+ <Typography key={i} variant="caption" color="warning.main">
433
+ • {c.description}
434
+ </Typography>
435
+ ))}
436
+ </Stack>
437
+ )}
438
+ {row.error && (
439
+ <Typography variant="caption" color="error">
440
+ {row.error}
441
+ </Typography>
442
+ )}
443
+ </Box>
444
+ </Collapse>
445
+ </TableCell>
446
+ </TableRow>
447
+ </React.Fragment>
448
+ ))}
449
+ </TableBody>
450
+ </Table>
451
+ </Paper>
452
+ )}
453
+ </Stack>
454
+ );
455
+ }
456
+
457
+ // Stat summary box used in aggregate row
458
+ function StatBox({
459
+ label,
460
+ value,
461
+ color,
462
+ }: {
463
+ label: string;
464
+ value: string | number;
465
+ color?: string;
466
+ }): React.ReactElement {
467
+ return (
468
+ <Box>
469
+ <Typography variant="caption" color="text.secondary" sx={{ display: "block" }}>
470
+ {label}
471
+ </Typography>
472
+ <Typography variant="h6" sx={{ color: color ?? "text.primary", lineHeight: 1.2 }}>
473
+ {value}
474
+ </Typography>
475
+ </Box>
476
+ );
477
+ }
478
+
479
+ // Sortable table header cell
480
+ function SortHeaderCell({
481
+ field,
482
+ active,
483
+ dir,
484
+ onSort,
485
+ children,
486
+ }: {
487
+ field: SortField;
488
+ active: SortField;
489
+ dir: "asc" | "desc";
490
+ onSort: (f: SortField) => void;
491
+ children: React.ReactNode;
492
+ }): React.ReactElement {
493
+ return (
494
+ <TableCell>
495
+ <TableSortLabel
496
+ active={active === field}
497
+ direction={active === field ? dir : "asc"}
498
+ onClick={() => onSort(field)}
499
+ >
500
+ {children}
501
+ </TableSortLabel>
502
+ </TableCell>
503
+ );
504
+ }
505
+
506
+ export default BatchTab;
frontend/src/types/api.ts CHANGED
@@ -14,6 +14,7 @@ export interface AnalyzeRequest {
14
  source?: string;
15
  pipeline?: "spacy" | "llm";
16
  model?: string;
 
17
  }
18
 
19
  export interface InconsistencyResponse {
@@ -65,6 +66,60 @@ export interface AnalyzeResponse {
65
  pipeline: string;
66
  model: string;
67
  processing_time_ms: number;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
  }
69
 
70
  // compare schemas
 
14
  source?: string;
15
  pipeline?: "spacy" | "llm";
16
  model?: string;
17
+ persist?: boolean;
18
  }
19
 
20
  export interface InconsistencyResponse {
 
66
  pipeline: string;
67
  model: string;
68
  processing_time_ms: number;
69
+ article_id?: string | null;
70
+ cross_article_inconsistencies?: InconsistencyResponse[];
71
+ }
72
+
73
+ // batch schemas
74
+ export interface BatchArticle {
75
+ text: string;
76
+ title?: string;
77
+ publication_date?: string;
78
+ source?: string;
79
+ }
80
+
81
+ export interface BatchRequest {
82
+ articles: BatchArticle[];
83
+ pipeline: string;
84
+ model?: string;
85
+ persist: boolean;
86
+ }
87
+
88
+ export interface BatchArticleResult {
89
+ article_id: string;
90
+ title: string;
91
+ score: number;
92
+ label: string;
93
+ summary: string;
94
+ n_claims: number;
95
+ n_inconsistencies: number;
96
+ n_cross_article_inconsistencies: number;
97
+ cross_article_conflicts: InconsistencyResponse[];
98
+ processing_time_ms: number;
99
+ error?: string | null;
100
+ }
101
+
102
+ export interface BatchResponse {
103
+ results: BatchArticleResult[];
104
+ total_articles: number;
105
+ total_cross_article_conflicts: number;
106
+ avg_score: number;
107
+ neo4j_enabled: boolean;
108
+ persisted: boolean;
109
+ }
110
+
111
+ // article history schemas
112
+ export interface StoredArticle {
113
+ article_id: string;
114
+ title: string | null;
115
+ source: string | null;
116
+ analyzed_at: string | null;
117
+ fact_count: number;
118
+ }
119
+
120
+ export interface ArticlesResponse {
121
+ articles: StoredArticle[];
122
+ neo4j_enabled: boolean;
123
  }
124
 
125
  // compare schemas
requirements.txt CHANGED
@@ -1,17 +1,20 @@
1
- # Core Pipeline
2
- spacy>=3.7
3
  dateparser
4
- networkx>=3.0
5
- networkx-temporal
6
- SPARQLWrapper
7
 
8
  # API
9
  fastapi
10
  uvicorn[standard]
11
- pydantic>=2.0
 
12
 
13
- # Data Ingestion
14
- feedparser
 
 
 
15
  requests
16
  python-dotenv
17
 
@@ -19,6 +22,3 @@ python-dotenv
19
  pytest
20
  pytest-asyncio
21
  httpx
22
-
23
- # Notebooks
24
- jupyter
 
1
+ # Core pipeline
2
+ spacy
3
  dateparser
4
+ networkx
5
+ neo4j
 
6
 
7
  # API
8
  fastapi
9
  uvicorn[standard]
10
+ pydantic
11
+ python-multipart
12
 
13
+ # File upload extraction
14
+ python-docx
15
+ PyPDF2
16
+
17
+ # HTTP and env
18
  requests
19
  python-dotenv
20
 
 
22
  pytest
23
  pytest-asyncio
24
  httpx
 
 
 
training/Modelfile DELETED
@@ -1,32 +0,0 @@
1
- FROM ./output/fakenews-ner.gguf
2
-
3
- PARAMETER temperature 0.1
4
- PARAMETER num_predict 4096
5
-
6
- SYSTEM """You are a temporal fact extraction system. Given a news article, extract ALL temporal facts.
7
-
8
- For each fact, return a JSON object with:
9
- - "subject": the main entity (person, organization, location)
10
- - "subject_type": one of PERSON, ORG, GPE, EVENT, NORP, PRODUCT, OTHER
11
- - "predicate": the relationship (e.g. "holds_position", "member_of", "occurred_on", "caused", "preceded", "followed", "generic")
12
- - "object": the related entity or context
13
- - "object_type": same types as subject_type
14
- - "time_expression": the temporal expression exactly as it appears in the text
15
- - "time_start": start date if it's a range (ISO format YYYY-MM-DD or null)
16
- - "time_end": end date if it's a range (ISO format YYYY-MM-DD or null)
17
- - "time_point": specific date if it's a single point (ISO format or null)
18
- - "source_sentence": the original sentence from the article that contains this fact
19
- - "confidence": your confidence 0.0–1.0
20
-
21
- Examples:
22
-
23
- Input: "Barack Obama served as the 44th President of the United States from January 2009 to January 2017."
24
- Output:
25
- [{"subject": "Barack Obama", "subject_type": "PERSON", "predicate": "holds_position", "object": "44th President of the United States", "object_type": "ORG", "time_expression": "from January 2009 to January 2017", "time_start": "2009-01-20", "time_end": "2017-01-20", "time_point": null, "source_sentence": "Barack Obama served as the 44th President of the United States from January 2009 to January 2017.", "confidence": 0.95}]
26
-
27
- Input: "The Affordable Care Act was signed into law on March 23, 2010 by President Obama."
28
- Output:
29
- [{"subject": "Affordable Care Act", "subject_type": "EVENT", "predicate": "occurred_on", "object": "signed into law", "object_type": "EVENT", "time_expression": "March 23, 2010", "time_start": null, "time_end": null, "time_point": "2010-03-23", "source_sentence": "The Affordable Care Act was signed into law on March 23, 2010 by President Obama.", "confidence": 0.92},
30
- {"subject": "President Obama", "subject_type": "PERSON", "predicate": "generic", "object": "Affordable Care Act", "object_type": "EVENT", "time_expression": "March 23, 2010", "time_start": null, "time_end": null, "time_point": "2010-03-23", "source_sentence": "The Affordable Care Act was signed into law on March 23, 2010 by President Obama.", "confidence": 0.88}]
31
-
32
- Return ONLY a JSON array. No explanations, no markdown fences."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
training/README.md DELETED
@@ -1,120 +0,0 @@
1
- # Fine-Tuning: fakenews-ner
2
-
3
- QLoRA fine-tuning pipeline for the `fakenews-ner` model — a Mistral-7B variant
4
- specialized for temporal fact extraction from political news articles.
5
-
6
- The fine-tuned model is exported to GGUF and loaded into Ollama as `fakenews-ner`,
7
- where it replaces zero-shot inference with task-specific extraction.
8
-
9
- ## Prerequisites
10
-
11
- **Hardware:** CUDA-capable GPU with at least 8 GB VRAM (developed on RTX 4060 Laptop).
12
-
13
- **Software:**
14
- - Python 3.11
15
- - CUDA 12.1+ with matching PyTorch
16
- - Ollama (for the final registration step)
17
-
18
- **Install training dependencies** (separate from the main backend requirements):
19
-
20
- ```bash
21
- pip install -r training/requirements.txt
22
- ```
23
-
24
- unsloth must be installed with the correct CUDA index. If the above fails, follow
25
- the manual install at https://docs.unsloth.ai/get-started/installation.
26
-
27
- ## Steps
28
-
29
- ### 1 — Prepare dataset (~5 minutes)
30
-
31
- Runs `SpacyExtractor(en_core_web_trf)` on each article to produce silver-label
32
- temporal facts, then formats them as instruction/input/output triplets.
33
-
34
- ```bash
35
- # Default: reads data/datasets/liar/, writes training/data/
36
- python training/prepare_dataset.py
37
-
38
- # Custom article directory (.txt files)
39
- python training/prepare_dataset.py --input-dir path/to/articles/ --max-articles 300
40
- ```
41
-
42
- **Input format:** either LIAR-style `.tsv` files (id, label, statement, ...) or
43
- plain `.txt` files, one article per file.
44
-
45
- **Output:** `training/data/train.jsonl` (80%) and `training/data/eval.jsonl` (20%).
46
-
47
- Articles where spaCy extracts zero temporal facts are skipped — they provide no
48
- training signal. Expect 100–150 usable examples from 200 articles.
49
-
50
- ### 2 — Train (~1–2 hours on RTX 4060)
51
-
52
- ```bash
53
- python training/train_qlora.py
54
- ```
55
-
56
- LoRA configuration: r=16, alpha=32, target modules q/k/v/o projections.
57
- Batch size 1 with 4 gradient accumulation steps (effective batch = 4).
58
- Adapter weights are saved to `training/output/fakenews-ner-lora/`.
59
-
60
- Optional overrides:
61
-
62
- ```bash
63
- python training/train_qlora.py --epochs 5 --lr 1e-4
64
- python training/train_qlora.py --base-model unsloth/llama-3-8b-Instruct-bnb-4bit
65
- ```
66
-
67
- ### 3 — Export to GGUF (~10 minutes)
68
-
69
- Merges the LoRA adapter into the base model and quantizes to Q4_K_M.
70
-
71
- ```bash
72
- python training/export_gguf.py
73
- ```
74
-
75
- Output: `training/output/fakenews-ner.gguf`
76
-
77
- ### 4 — Register in Ollama
78
-
79
- ```bash
80
- # Linux / macOS / Git Bash
81
- bash training/register_ollama.sh
82
-
83
- # Windows PowerShell
84
- .\training\register_ollama.ps1
85
- ```
86
-
87
- This runs `ollama create fakenews-ner -f training/Modelfile`. The Modelfile points
88
- to the `.gguf` file and embeds the system prompt with few-shot examples.
89
-
90
- Verify the model is available:
91
-
92
- ```bash
93
- ollama list
94
- ollama run fakenews-ner "Extract temporal facts from: Obama was president from 2009 to 2017."
95
- ```
96
-
97
- Once registered, set `LLM_MODELS=sciphi/triplex,fakenews-ner` and
98
- `LLM_DEFAULT_MODEL=fakenews-ner` in `.env` to use it as the default LLM pipeline.
99
-
100
- ## File Overview
101
-
102
- | File | Purpose |
103
- |------|---------|
104
- | `prepare_dataset.py` | Generate train/eval JSONL from articles using spaCy silver labels |
105
- | `train_qlora.py` | QLoRA fine-tuning with unsloth + SFTTrainer |
106
- | `export_gguf.py` | Merge adapter and export to GGUF Q4_K_M |
107
- | `Modelfile` | Ollama model definition (FROM + SYSTEM prompt) |
108
- | `register_ollama.sh` | Register GGUF in Ollama (Linux/macOS/Git Bash) |
109
- | `register_ollama.ps1` | Register GGUF in Ollama (Windows PowerShell) |
110
- | `data/` | Generated train.jsonl and eval.jsonl (git-ignored) |
111
- | `output/` | LoRA adapter and GGUF file (git-ignored) |
112
-
113
- ## Expected VRAM Usage
114
-
115
- | Stage | VRAM |
116
- |-------|------|
117
- | Prepare dataset (spaCy en_core_web_trf) | ~2 GB |
118
- | Training (Mistral-7B 4-bit + LoRA) | ~7–7.5 GB |
119
- | GGUF export (merge + quantize) | ~7 GB |
120
- | Ollama inference (Q4_K_M) | ~5 GB |
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
training/data/.gitkeep DELETED
File without changes
training/data/eval.jsonl DELETED
The diff for this file is too large to render. See raw diff
 
training/data/train.jsonl DELETED
The diff for this file is too large to render. See raw diff
 
training/export_gguf.py DELETED
@@ -1,96 +0,0 @@
1
- """
2
- Merge LoRA adapter with the base model and export to GGUF Q4_K_M format.
3
-
4
- The resulting .gguf file can be registered directly in Ollama via register_ollama.sh
5
- or register_ollama.ps1.
6
-
7
- Usage:
8
- python training/export_gguf.py
9
- python training/export_gguf.py --adapter-dir training/output/fakenews-ner-lora
10
- python training/export_gguf.py --output training/output/fakenews-ner.gguf
11
- """
12
-
13
- from __future__ import annotations
14
-
15
- import argparse
16
- import logging
17
- from pathlib import Path
18
-
19
- logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
20
- logger = logging.getLogger(__name__)
21
-
22
- DEFAULT_ADAPTER_DIR = Path(__file__).parent / "output" / "fakenews-ner-lora"
23
- DEFAULT_OUTPUT = Path(__file__).parent / "output" / "fakenews-ner.gguf"
24
-
25
-
26
- def parse_args() -> argparse.Namespace:
27
- parser = argparse.ArgumentParser(
28
- description="Merge LoRA adapter and export to GGUF for Ollama"
29
- )
30
- parser.add_argument(
31
- "--adapter-dir",
32
- type=Path,
33
- default=DEFAULT_ADAPTER_DIR,
34
- help="Directory containing the saved LoRA adapter (output of train_qlora.py)",
35
- )
36
- parser.add_argument(
37
- "--output",
38
- type=Path,
39
- default=DEFAULT_OUTPUT,
40
- help="Path for the exported .gguf file",
41
- )
42
- parser.add_argument(
43
- "--quantization",
44
- default="q4_k_m",
45
- choices=["q4_k_m", "q8_0", "f16"],
46
- help="GGUF quantization type (q4_k_m recommended for 8 GB VRAM inference)",
47
- )
48
- return parser.parse_args()
49
-
50
-
51
- def main() -> None:
52
- args = parse_args()
53
-
54
- if not args.adapter_dir.exists():
55
- raise FileNotFoundError(
56
- f"Adapter directory not found: {args.adapter_dir}\n"
57
- "Run train_qlora.py first."
58
- )
59
-
60
- args.output.parent.mkdir(parents=True, exist_ok=True)
61
-
62
- logger.info(f"Loading adapter from : {args.adapter_dir}")
63
- logger.info(f"Output GGUF path : {args.output}")
64
- logger.info(f"Quantization : {args.quantization}")
65
-
66
- from unsloth import FastLanguageModel
67
-
68
- # Reload the adapter (unsloth re-attaches it to the base model automatically)
69
- model, tokenizer = FastLanguageModel.from_pretrained(
70
- model_name=str(args.adapter_dir),
71
- max_seq_length=2048,
72
- dtype=None,
73
- load_in_4bit=True,
74
- )
75
-
76
- logger.info("Merging adapter weights into base model...")
77
-
78
- # save_pretrained_gguf merges, quantizes, and writes the .gguf in one step
79
- model.save_pretrained_gguf(
80
- str(args.output.with_suffix("")),
81
- tokenizer,
82
- quantization_method=args.quantization,
83
- )
84
-
85
- # unsloth appends the quantization suffix; rename to the clean target path
86
- candidate = args.output.parent / f"{args.output.stem}-{args.quantization}.gguf"
87
- if candidate.exists() and candidate != args.output:
88
- candidate.rename(args.output)
89
- logger.info(f"Renamed {candidate.name} -> {args.output.name}")
90
-
91
- logger.info(f"Export complete: {args.output}")
92
- logger.info("Next step: run register_ollama.sh (or .ps1) to load into Ollama")
93
-
94
-
95
- if __name__ == "__main__":
96
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
training/prepare_dataset.py DELETED
@@ -1,229 +0,0 @@
1
- """
2
- Prepare instruction-tuning dataset for fakenews-ner fine-tuning.
3
-
4
- Uses SpacyExtractor (en_core_web_trf) as a silver-label teacher to annotate
5
- articles, then formats each article as an instruction/input/output triplet
6
- matching the LLMExtractor prompt schema.
7
-
8
- Usage:
9
- python training/prepare_dataset.py
10
- python training/prepare_dataset.py --input-dir data/datasets/liar --max-articles 500
11
- python training/prepare_dataset.py --input-dir my_articles/ --split-ratio 0.85
12
- """
13
-
14
- from __future__ import annotations
15
-
16
- import argparse
17
- import json
18
- import logging
19
- import sys
20
- from pathlib import Path
21
- from typing import Optional
22
-
23
- logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
24
- logger = logging.getLogger(__name__)
25
-
26
- # Repo root must be on sys.path so backend imports resolve
27
- _REPO_ROOT = Path(__file__).parent.parent
28
- if str(_REPO_ROOT) not in sys.path:
29
- sys.path.insert(0, str(_REPO_ROOT))
30
-
31
- from backend.pipeline.extraction.llm_extractor import SYSTEM_PROMPT, USER_PROMPT_TEMPLATE
32
- from backend.pipeline.extraction.spacy_extractor import SpacyExtractor
33
- from backend.pipeline.graph.models import Article, TemporalFact
34
-
35
-
36
- def fact_to_dict(fact: TemporalFact) -> dict:
37
- """Convert a TemporalFact to the JSON schema expected by the LLM."""
38
- time_expr = ""
39
- if fact.time_point:
40
- time_expr = fact.time_point.raw_text
41
- elif fact.time_start:
42
- time_expr = fact.time_start.raw_text
43
-
44
- return {
45
- "subject": fact.subject.text,
46
- "subject_type": fact.subject.entity_type.value,
47
- "predicate": fact.predicate.value,
48
- "object": fact.object.text,
49
- "object_type": fact.object.entity_type.value,
50
- "time_expression": time_expr,
51
- "time_start": fact.time_start.date_string if fact.time_start else None,
52
- "time_end": fact.time_end.date_string if fact.time_end else None,
53
- "time_point": fact.time_point.date_string if fact.time_point else None,
54
- "source_sentence": fact.source_sentence,
55
- "confidence": round(fact.extraction_confidence, 2),
56
- }
57
-
58
-
59
- def load_articles_tsv(tsv_path: Path, max_articles: int) -> list[Article]:
60
- """Load articles from a LIAR-style TSV file (no header row).
61
-
62
- Column layout: id, label, statement, subject, speaker, job, state,
63
- party, counts x5, context
64
- """
65
- articles = []
66
- with open(tsv_path, encoding="utf-8") as f:
67
- for line in f:
68
- if len(articles) >= max_articles:
69
- break
70
- parts = line.rstrip("\n").split("\t")
71
- if len(parts) < 3:
72
- continue
73
- text = parts[2].strip()
74
- if not text:
75
- continue
76
- speaker = parts[4].strip() if len(parts) > 4 else ""
77
- articles.append(Article(text=text, title=speaker))
78
- return articles
79
-
80
-
81
- def load_articles_txt(txt_dir: Path, max_articles: int) -> list[Article]:
82
- """Load articles from a directory of plain-text .txt files."""
83
- articles = []
84
- for p in sorted(txt_dir.glob("*.txt")):
85
- if len(articles) >= max_articles:
86
- break
87
- text = p.read_text(encoding="utf-8").strip()
88
- if text:
89
- articles.append(Article(text=text, title=p.stem))
90
- return articles
91
-
92
-
93
- def load_articles(input_dir: Path, max_articles: int) -> list[Article]:
94
- """Auto-detect TSV files or .txt files in the input directory."""
95
- tsv_files = sorted(input_dir.glob("*.tsv")) + sorted(input_dir.glob("*.csv"))
96
- if tsv_files:
97
- logger.info(f"Found TSV file(s): {[f.name for f in tsv_files]}")
98
- articles: list[Article] = []
99
- for tsv in tsv_files:
100
- articles.extend(load_articles_tsv(tsv, max_articles - len(articles)))
101
- if len(articles) >= max_articles:
102
- break
103
- return articles
104
-
105
- txt_files = list(input_dir.glob("*.txt"))
106
- if txt_files:
107
- logger.info(f"Found {len(txt_files)} .txt file(s)")
108
- return load_articles_txt(input_dir, max_articles)
109
-
110
- logger.warning(f"No .tsv or .txt files found in {input_dir}")
111
- return []
112
-
113
-
114
- def build_example(article: Article, facts: list[TemporalFact]) -> dict:
115
- """Format a single article + its silver facts as an instruction example."""
116
- pub_date_str = (
117
- article.publication_date.strftime("%Y-%m-%d")
118
- if article.publication_date else "unknown"
119
- )
120
- user_input = USER_PROMPT_TEMPLATE.format(
121
- title=article.title or "(untitled)",
122
- pub_date=pub_date_str,
123
- text=article.text[:4000],
124
- )
125
- output_json = json.dumps([fact_to_dict(f) for f in facts], ensure_ascii=False)
126
- return {
127
- "instruction": SYSTEM_PROMPT,
128
- "input": user_input,
129
- "output": output_json,
130
- }
131
-
132
-
133
- def write_jsonl(examples: list[dict], path: Path) -> None:
134
- path.parent.mkdir(parents=True, exist_ok=True)
135
- with open(path, "w", encoding="utf-8") as f:
136
- for ex in examples:
137
- f.write(json.dumps(ex, ensure_ascii=False) + "\n")
138
- logger.info(f"Wrote {len(examples)} examples to {path}")
139
-
140
-
141
- def main() -> None:
142
- parser = argparse.ArgumentParser(
143
- description="Prepare instruction-tuning data for fakenews-ner"
144
- )
145
- parser.add_argument(
146
- "--input-dir",
147
- type=Path,
148
- default=_REPO_ROOT / "data" / "datasets" / "liar",
149
- help="Directory containing .tsv or .txt article files",
150
- )
151
- parser.add_argument(
152
- "--output-dir",
153
- type=Path,
154
- default=Path(__file__).parent / "data",
155
- help="Directory to write train.jsonl and eval.jsonl",
156
- )
157
- parser.add_argument(
158
- "--max-articles",
159
- type=int,
160
- default=200,
161
- help="Maximum number of articles to process",
162
- )
163
- parser.add_argument(
164
- "--split-ratio",
165
- type=float,
166
- default=0.8,
167
- help="Fraction of examples to use for training (rest goes to eval)",
168
- )
169
- args = parser.parse_args()
170
-
171
- if not args.input_dir.exists():
172
- logger.error(f"Input directory not found: {args.input_dir}")
173
- logger.error("Place LIAR .tsv files in data/datasets/liar/ or pass --input-dir")
174
- sys.exit(1)
175
-
176
- logger.info(f"Loading articles from {args.input_dir}")
177
- articles = load_articles(args.input_dir, args.max_articles)
178
- if not articles:
179
- logger.error("No articles loaded. Exiting.")
180
- sys.exit(1)
181
- logger.info(f"Loaded {len(articles)} articles")
182
-
183
- logger.info("Initialising SpacyExtractor (en_core_web_trf) — loading model once...")
184
- extractor = SpacyExtractor(model_name="en_core_web_trf")
185
-
186
- # Fail fast if model can't load
187
- try:
188
- _ = extractor.nlp
189
- except Exception as e:
190
- print(f"FATAL: Cannot load spaCy model: {e}")
191
- print("Run: python -m spacy download en_core_web_trf")
192
- sys.exit(1)
193
-
194
- examples: list[dict] = []
195
- skipped = 0
196
-
197
- for i, article in enumerate(articles, start=1):
198
- try:
199
- facts = extractor.extract(article)
200
- except Exception as e:
201
- logger.warning(f"Article {i}: extraction error — {e}")
202
- skipped += 1
203
- continue
204
-
205
- if not facts:
206
- skipped += 1
207
- continue
208
-
209
- examples.append(build_example(article, facts))
210
-
211
- if i % 20 == 0:
212
- logger.info(f"Progress: {i}/{len(articles)} — {len(examples)} usable so far")
213
-
214
- logger.info(
215
- f"Finished: {len(examples)} usable examples, {skipped} skipped (no temporal facts)"
216
- )
217
-
218
- split = int(len(examples) * args.split_ratio)
219
- train_examples = examples[:split]
220
- eval_examples = examples[split:]
221
-
222
- write_jsonl(train_examples, args.output_dir / "train.jsonl")
223
- write_jsonl(eval_examples, args.output_dir / "eval.jsonl")
224
-
225
- logger.info(f"Dataset ready: {len(train_examples)} train / {len(eval_examples)} eval")
226
-
227
-
228
- if __name__ == "__main__":
229
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
training/register_ollama.ps1 DELETED
@@ -1,22 +0,0 @@
1
- # Register the fine-tuned fakenews-ner model in Ollama using the local Modelfile.
2
- # Run from the repo root or the training/ directory.
3
-
4
- Set-Location $PSScriptRoot
5
-
6
- if (-not (Get-Command ollama -ErrorAction SilentlyContinue)) {
7
- Write-Error "ERROR: 'ollama' not found. Install Ollama from https://ollama.com and try again."
8
- exit 1
9
- }
10
-
11
- if (-not (Test-Path "output\fakenews-ner.gguf")) {
12
- Write-Error "ERROR: output\fakenews-ner.gguf not found."
13
- Write-Host "Run export_gguf.py first to generate the GGUF file."
14
- exit 1
15
- }
16
-
17
- Write-Host "Registering model 'fakenews-ner' in Ollama..."
18
- ollama create fakenews-ner -f Modelfile
19
-
20
- Write-Host ""
21
- Write-Host "Done. Test the model with:"
22
- Write-Host " ollama run fakenews-ner 'Extract temporal facts from: Obama was president from 2009 to 2017.'"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
training/register_ollama.sh DELETED
@@ -1,23 +0,0 @@
1
- #!/bin/bash
2
- # Register the fine-tuned fakenews-ner model in Ollama using the local Modelfile.
3
- # Run from the repo root or the training/ directory — the script locates itself.
4
-
5
- cd "$(dirname "$0")"
6
-
7
- if ! command -v ollama &>/dev/null; then
8
- echo "ERROR: 'ollama' not found. Install Ollama from https://ollama.com and try again."
9
- exit 1
10
- fi
11
-
12
- if [ ! -f "output/fakenews-ner.gguf" ]; then
13
- echo "ERROR: output/fakenews-ner.gguf not found."
14
- echo "Run export_gguf.py first to generate the GGUF file."
15
- exit 1
16
- fi
17
-
18
- echo "Registering model 'fakenews-ner' in Ollama..."
19
- ollama create fakenews-ner -f Modelfile
20
-
21
- echo ""
22
- echo "Done. Test the model with:"
23
- echo " ollama run fakenews-ner 'Extract temporal facts from: Obama was president from 2009 to 2017.'"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
training/requirements.txt DELETED
@@ -1,8 +0,0 @@
1
- unsloth[colab-new]
2
- torch>=2.1.0
3
- peft
4
- bitsandbytes
5
- transformers
6
- trl
7
- datasets
8
- accelerate
 
 
 
 
 
 
 
 
 
training/train_qlora.py DELETED
@@ -1,165 +0,0 @@
1
- """
2
- QLoRA fine-tuning of Mistral-7B for temporal fact extraction (fakenews-ner).
3
-
4
- Requires unsloth and a CUDA-capable GPU. Designed for RTX 4060 Laptop (8 GB VRAM)
5
- using 4-bit quantization. Adapter weights are saved to --output-dir and can be
6
- merged + exported with export_gguf.py.
7
-
8
- Usage:
9
- python training/train_qlora.py
10
- python training/train_qlora.py --epochs 5 --lr 1e-4
11
- python training/train_qlora.py --base-model unsloth/llama-3-8b-Instruct-bnb-4bit
12
- """
13
-
14
- from __future__ import annotations
15
-
16
- import argparse
17
- import logging
18
- from pathlib import Path
19
-
20
- logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
21
- logger = logging.getLogger(__name__)
22
-
23
- DEFAULT_BASE_MODEL = "unsloth/mistral-7b-instruct-v0.3-bnb-4bit"
24
- DEFAULT_DATA_DIR = Path(__file__).parent / "data"
25
- DEFAULT_OUTPUT_DIR = Path(__file__).parent / "output" / "fakenews-ner-lora"
26
-
27
-
28
- def parse_args() -> argparse.Namespace:
29
- parser = argparse.ArgumentParser(
30
- description="QLoRA fine-tuning for fakenews-ner on RTX 4060 (8 GB VRAM)"
31
- )
32
- parser.add_argument(
33
- "--base-model",
34
- default=DEFAULT_BASE_MODEL,
35
- help="Hugging Face model ID or local path (must be 4-bit bnb quantized for 8 GB VRAM)",
36
- )
37
- parser.add_argument(
38
- "--data-dir",
39
- type=Path,
40
- default=DEFAULT_DATA_DIR,
41
- help="Directory containing train.jsonl and eval.jsonl",
42
- )
43
- parser.add_argument(
44
- "--output-dir",
45
- type=Path,
46
- default=DEFAULT_OUTPUT_DIR,
47
- help="Directory to save LoRA adapter weights",
48
- )
49
- parser.add_argument("--epochs", type=int, default=3, help="Number of training epochs")
50
- parser.add_argument("--lr", type=float, default=2e-4, help="Learning rate")
51
- return parser.parse_args()
52
-
53
-
54
- def load_dataset(data_dir: Path):
55
- """Load train/eval JSONL files as Hugging Face datasets."""
56
- from datasets import load_dataset as hf_load_dataset
57
-
58
- train_path = data_dir / "train.jsonl"
59
- eval_path = data_dir / "eval.jsonl"
60
-
61
- if not train_path.exists():
62
- raise FileNotFoundError(
63
- f"Training data not found at {train_path}. "
64
- "Run prepare_dataset.py first."
65
- )
66
-
67
- data_files = {"train": str(train_path)}
68
- if eval_path.exists():
69
- data_files["eval"] = str(eval_path)
70
-
71
- dataset = hf_load_dataset("json", data_files=data_files)
72
- logger.info(
73
- f"Dataset loaded — train: {len(dataset['train'])} examples"
74
- + (f", eval: {len(dataset['eval'])} examples" if "eval" in dataset else "")
75
- )
76
- return dataset
77
-
78
-
79
- def format_prompt(example: dict) -> dict:
80
- """Convert instruction/input/output dict to a single formatted text field.
81
-
82
- Uses the Mistral instruct template: [INST] system + user [/INST] assistant
83
- """
84
- prompt = (
85
- f"[INST] {example['instruction']}\n\n{example['input']} [/INST] "
86
- f"{example['output']}"
87
- )
88
- return {"text": prompt}
89
-
90
-
91
- def main() -> None:
92
- args = parse_args()
93
- args.output_dir.mkdir(parents=True, exist_ok=True)
94
-
95
- logger.info(f"Base model : {args.base_model}")
96
- logger.info(f"Data dir : {args.data_dir}")
97
- logger.info(f"Output dir : {args.output_dir}")
98
- logger.info(f"Epochs : {args.epochs} LR: {args.lr}")
99
-
100
- # Lazy imports — unsloth rewrites CUDA kernels at import time
101
- from unsloth import FastLanguageModel
102
- from trl import SFTTrainer
103
- from transformers import TrainingArguments
104
-
105
- # Load 4-bit quantized base model
106
- model, tokenizer = FastLanguageModel.from_pretrained(
107
- model_name=args.base_model,
108
- max_seq_length=2048,
109
- dtype=None, # auto-detect: float16 on Ampere+
110
- load_in_4bit=True,
111
- )
112
- logger.info("Base model loaded")
113
-
114
- # Apply LoRA adapters
115
- model = FastLanguageModel.get_peft_model(
116
- model,
117
- r=16,
118
- lora_alpha=32,
119
- target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
120
- lora_dropout=0.05,
121
- bias="none",
122
- use_gradient_checkpointing="unsloth",
123
- random_state=42,
124
- )
125
- logger.info("LoRA adapters applied")
126
-
127
- dataset = load_dataset(args.data_dir)
128
- dataset = dataset.map(format_prompt)
129
-
130
- training_args = TrainingArguments(
131
- output_dir=str(args.output_dir),
132
- num_train_epochs=args.epochs,
133
- per_device_train_batch_size=1,
134
- gradient_accumulation_steps=4,
135
- warmup_steps=10,
136
- learning_rate=args.lr,
137
- fp16=True,
138
- logging_steps=10,
139
- save_strategy="epoch",
140
- eval_strategy="epoch" if "eval" in dataset else "no",
141
- load_best_model_at_end=("eval" in dataset),
142
- report_to="none",
143
- )
144
-
145
- trainer = SFTTrainer(
146
- model=model,
147
- tokenizer=tokenizer,
148
- train_dataset=dataset["train"],
149
- eval_dataset=dataset.get("eval"),
150
- dataset_text_field="text",
151
- max_seq_length=2048,
152
- args=training_args,
153
- )
154
-
155
- logger.info("Starting training...")
156
- trainer.train()
157
-
158
- model.save_pretrained(str(args.output_dir))
159
- tokenizer.save_pretrained(str(args.output_dir))
160
- logger.info(f"Adapter weights saved to {args.output_dir}")
161
- logger.info("Next step: run export_gguf.py to merge and export to GGUF")
162
-
163
-
164
- if __name__ == "__main__":
165
- main()