Spaces:
Sleeping
Sleeping
Add neo4j and cross article verification
Browse files- .claude/settings.local.json +8 -0
- .env.example +7 -0
- backend/config.py +7 -0
- backend/main.py +6 -0
- backend/pipeline/graph/base_store.py +32 -0
- backend/pipeline/graph/factory.py +28 -0
- backend/pipeline/graph/neo4j_store.py +262 -0
- backend/pipeline/graph/store.py +16 -6
- backend/pipeline/verification/cross_article.py +159 -0
- backend/routers/analyze.py +51 -1
- backend/routers/articles.py +50 -0
- backend/routers/batch.py +185 -0
- backend/routers/upload.py +42 -0
- frontend/src/App.tsx +44 -14
- frontend/src/api/client.ts +38 -1
- frontend/src/components/AnalyzeTab.tsx +59 -2
- frontend/src/components/ArticleHistory.tsx +213 -0
- frontend/src/components/ArticleInput.tsx +62 -66
- frontend/src/components/BatchTab.tsx +506 -0
- frontend/src/types/api.ts +55 -0
- requirements.txt +11 -11
- training/Modelfile +0 -32
- training/README.md +0 -120
- training/data/.gitkeep +0 -0
- training/data/eval.jsonl +0 -0
- training/data/train.jsonl +0 -0
- training/export_gguf.py +0 -96
- training/prepare_dataset.py +0 -229
- training/register_ollama.ps1 +0 -22
- training/register_ollama.sh +0 -23
- training/requirements.txt +0 -8
- training/train_qlora.py +0 -165
.claude/settings.local.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"permissions": {
|
| 3 |
+
"allow": [
|
| 4 |
+
"Read(//mnt/c/Users/pante/Desktop/**)",
|
| 5 |
+
"Bash(pip install *)"
|
| 6 |
+
]
|
| 7 |
+
}
|
| 8 |
+
}
|
.env.example
CHANGED
|
@@ -26,3 +26,10 @@ CORS_ORIGINS=http://localhost:3000
|
|
| 26 |
|
| 27 |
# NewsAPI key — only needed for the demo notebook, not the main pipeline
|
| 28 |
# NEWSAPI_KEY=your_key_here
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
|
| 27 |
# NewsAPI key — only needed for the demo notebook, not the main pipeline
|
| 28 |
# NEWSAPI_KEY=your_key_here
|
| 29 |
+
|
| 30 |
+
# Neo4j (optional — for persistent cross-article analysis)
|
| 31 |
+
# Start with: docker run -p 7687:7687 -e NEO4J_AUTH=neo4j/password neo4j:community
|
| 32 |
+
NEO4J_URI=bolt://localhost:7687
|
| 33 |
+
NEO4J_USER=neo4j
|
| 34 |
+
NEO4J_PASSWORD=password
|
| 35 |
+
NEO4J_ENABLED=false
|
backend/config.py
CHANGED
|
@@ -66,6 +66,13 @@ TCS_THRESHOLDS = {
|
|
| 66 |
}
|
| 67 |
|
| 68 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
# FastAPI settings
|
| 70 |
API_TITLE = "TCS - Temporal Coherence Score API"
|
| 71 |
API_VERSION = "0.1.0"
|
|
|
|
| 66 |
}
|
| 67 |
|
| 68 |
|
| 69 |
+
# Neo4j — optional persistent store for cross-article analysis
|
| 70 |
+
NEO4J_URI = os.getenv("NEO4J_URI", "bolt://localhost:7687")
|
| 71 |
+
NEO4J_USER = os.getenv("NEO4J_USER", "neo4j")
|
| 72 |
+
NEO4J_PASSWORD = os.getenv("NEO4J_PASSWORD", "password")
|
| 73 |
+
NEO4J_ENABLED = os.getenv("NEO4J_ENABLED", "false").lower() == "true"
|
| 74 |
+
|
| 75 |
+
|
| 76 |
# FastAPI settings
|
| 77 |
API_TITLE = "TCS - Temporal Coherence Score API"
|
| 78 |
API_VERSION = "0.1.0"
|
backend/main.py
CHANGED
|
@@ -9,6 +9,9 @@ from fastapi.middleware.cors import CORSMiddleware
|
|
| 9 |
|
| 10 |
from backend.config import API_TITLE, API_VERSION, CORS_ORIGINS
|
| 11 |
from backend.routers import analyze, compare, health
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
logging.basicConfig(
|
| 14 |
level=logging.INFO,
|
|
@@ -38,6 +41,9 @@ app.add_middleware(
|
|
| 38 |
app.include_router(health.router)
|
| 39 |
app.include_router(analyze.router)
|
| 40 |
app.include_router(compare.router)
|
|
|
|
|
|
|
|
|
|
| 41 |
|
| 42 |
|
| 43 |
@app.on_event("startup")
|
|
|
|
| 9 |
|
| 10 |
from backend.config import API_TITLE, API_VERSION, CORS_ORIGINS
|
| 11 |
from backend.routers import analyze, compare, health
|
| 12 |
+
from backend.routers.articles import router as articles_router
|
| 13 |
+
from backend.routers.batch import router as batch_router
|
| 14 |
+
from backend.routers.upload import router as upload_router
|
| 15 |
|
| 16 |
logging.basicConfig(
|
| 17 |
level=logging.INFO,
|
|
|
|
| 41 |
app.include_router(health.router)
|
| 42 |
app.include_router(analyze.router)
|
| 43 |
app.include_router(compare.router)
|
| 44 |
+
app.include_router(articles_router)
|
| 45 |
+
app.include_router(batch_router)
|
| 46 |
+
app.include_router(upload_router)
|
| 47 |
|
| 48 |
|
| 49 |
@app.on_event("startup")
|
backend/pipeline/graph/base_store.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Abstract interface for Temporal Knowledge Graph storage backends."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from abc import ABC, abstractmethod
|
| 6 |
+
|
| 7 |
+
from backend.pipeline.graph.models import TemporalFact
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class AbstractTKGStore(ABC):
|
| 11 |
+
"""Interface for Temporal Knowledge Graph storage backends."""
|
| 12 |
+
|
| 13 |
+
@abstractmethod
|
| 14 |
+
def add_fact(self, fact: TemporalFact, article_id: str | None = None) -> None: ...
|
| 15 |
+
|
| 16 |
+
@abstractmethod
|
| 17 |
+
def add_facts(self, facts: list[TemporalFact], article_id: str | None = None) -> None: ...
|
| 18 |
+
|
| 19 |
+
@abstractmethod
|
| 20 |
+
def get_all_facts(self, article_id: str | None = None) -> list[TemporalFact]: ...
|
| 21 |
+
|
| 22 |
+
@abstractmethod
|
| 23 |
+
def get_facts_for_entity(self, entity_name: str) -> list[TemporalFact]: ...
|
| 24 |
+
|
| 25 |
+
@abstractmethod
|
| 26 |
+
def get_articles(self) -> list[dict]: ...
|
| 27 |
+
|
| 28 |
+
@abstractmethod
|
| 29 |
+
def delete_article(self, article_id: str) -> bool: ...
|
| 30 |
+
|
| 31 |
+
@abstractmethod
|
| 32 |
+
def summary(self) -> dict: ...
|
backend/pipeline/graph/factory.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Factory for creating TKG storage backends."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import logging
|
| 6 |
+
|
| 7 |
+
from backend.pipeline.graph.base_store import AbstractTKGStore
|
| 8 |
+
|
| 9 |
+
logger = logging.getLogger(__name__)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def create_persistent_store() -> AbstractTKGStore | None:
|
| 13 |
+
"""Return a Neo4jTKGStore if NEO4J_ENABLED=true, otherwise None."""
|
| 14 |
+
from backend.config import NEO4J_ENABLED, NEO4J_PASSWORD, NEO4J_URI, NEO4J_USER
|
| 15 |
+
|
| 16 |
+
if not NEO4J_ENABLED:
|
| 17 |
+
logger.debug("NEO4J_ENABLED=false — persistent store disabled")
|
| 18 |
+
return None
|
| 19 |
+
|
| 20 |
+
from backend.pipeline.graph.neo4j_store import Neo4jTKGStore
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
store = Neo4jTKGStore(NEO4J_URI, NEO4J_USER, NEO4J_PASSWORD)
|
| 24 |
+
logger.info(f"Persistent Neo4j store created at {NEO4J_URI}")
|
| 25 |
+
return store
|
| 26 |
+
except Exception as exc:
|
| 27 |
+
logger.error(f"Failed to connect to Neo4j at {NEO4J_URI}: {exc}")
|
| 28 |
+
return None
|
backend/pipeline/graph/neo4j_store.py
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Persistent TKG storage using Neo4j Community Edition."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import logging
|
| 6 |
+
from datetime import datetime
|
| 7 |
+
from typing import TYPE_CHECKING
|
| 8 |
+
|
| 9 |
+
from backend.pipeline.graph.base_store import AbstractTKGStore
|
| 10 |
+
from backend.pipeline.graph.models import (
|
| 11 |
+
Entity, EntityType, RelationType, TemporalExpression, TemporalFact,
|
| 12 |
+
)
|
| 13 |
+
from backend.pipeline.graph.store import _entity_id
|
| 14 |
+
|
| 15 |
+
if TYPE_CHECKING:
|
| 16 |
+
from neo4j import Driver
|
| 17 |
+
|
| 18 |
+
logger = logging.getLogger(__name__)
|
| 19 |
+
|
| 20 |
+
# Fields returned by every fact-fetching query — defined once to avoid repetition
|
| 21 |
+
_FACT_RETURN = """
|
| 22 |
+
s.entity_id AS s_id, s.text AS s_text, s.entity_type AS s_type, s.wikidata_id AS s_wikidata,
|
| 23 |
+
o.entity_id AS o_id, o.text AS o_text, o.entity_type AS o_type, o.wikidata_id AS o_wikidata,
|
| 24 |
+
r.relation AS relation,
|
| 25 |
+
r.time_start_iso AS time_start_iso, r.time_start_str AS time_start_str,
|
| 26 |
+
r.time_end_iso AS time_end_iso, r.time_end_str AS time_end_str,
|
| 27 |
+
r.time_point_iso AS time_point_iso, r.time_point_str AS time_point_str,
|
| 28 |
+
r.source_sentence_idx AS sentence_idx,
|
| 29 |
+
r.confidence AS confidence,
|
| 30 |
+
r.extractor AS extractor
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class Neo4jTKGStore(AbstractTKGStore):
|
| 35 |
+
"""Persistent TKG storage using Neo4j."""
|
| 36 |
+
|
| 37 |
+
def __init__(self, uri: str, user: str, password: str) -> None:
|
| 38 |
+
from neo4j import GraphDatabase
|
| 39 |
+
self._driver: Driver = GraphDatabase.driver(uri, auth=(user, password))
|
| 40 |
+
self._ensure_indexes()
|
| 41 |
+
logger.info(f"Neo4jTKGStore connected to {uri}")
|
| 42 |
+
|
| 43 |
+
def _ensure_indexes(self) -> None:
|
| 44 |
+
"""Create indexes on first run so lookups on entity_id and article_id are fast."""
|
| 45 |
+
with self._driver.session() as session:
|
| 46 |
+
session.run("CREATE INDEX IF NOT EXISTS FOR (e:Entity) ON (e.entity_id)")
|
| 47 |
+
session.run("CREATE INDEX IF NOT EXISTS FOR (a:Article) ON (a.article_id)")
|
| 48 |
+
|
| 49 |
+
def close(self) -> None:
|
| 50 |
+
self._driver.close()
|
| 51 |
+
|
| 52 |
+
# ── Write ──
|
| 53 |
+
|
| 54 |
+
def add_fact(self, fact: TemporalFact, article_id: str | None = None) -> None:
|
| 55 |
+
"""Store a fact as two Entity nodes + a TEMPORAL_RELATION edge."""
|
| 56 |
+
with self._driver.session() as session:
|
| 57 |
+
session.execute_write(self._create_fact_tx, fact, article_id)
|
| 58 |
+
|
| 59 |
+
@staticmethod
|
| 60 |
+
def _create_fact_tx(tx, fact: TemporalFact, article_id: str | None) -> None:
|
| 61 |
+
def to_iso(expr: TemporalExpression | None) -> str | None:
|
| 62 |
+
if expr and expr.normalized_date:
|
| 63 |
+
return expr.normalized_date.isoformat()
|
| 64 |
+
return None
|
| 65 |
+
|
| 66 |
+
def to_str(expr: TemporalExpression | None) -> str | None:
|
| 67 |
+
return expr.date_string if expr else None
|
| 68 |
+
|
| 69 |
+
params = {
|
| 70 |
+
"subj_id": _entity_id(fact.subject),
|
| 71 |
+
"subj_text": fact.subject.text,
|
| 72 |
+
"subj_type": fact.subject.entity_type.value,
|
| 73 |
+
"subj_wikidata": fact.subject.wikidata_id,
|
| 74 |
+
"obj_id": _entity_id(fact.object),
|
| 75 |
+
"obj_text": fact.object.text,
|
| 76 |
+
"obj_type": fact.object.entity_type.value,
|
| 77 |
+
"obj_wikidata": fact.object.wikidata_id,
|
| 78 |
+
"relation": fact.predicate.value,
|
| 79 |
+
"time_start_iso": to_iso(fact.time_start),
|
| 80 |
+
"time_start_str": to_str(fact.time_start),
|
| 81 |
+
"time_end_iso": to_iso(fact.time_end),
|
| 82 |
+
"time_end_str": to_str(fact.time_end),
|
| 83 |
+
"time_point_iso": to_iso(fact.time_point),
|
| 84 |
+
"time_point_str": to_str(fact.time_point),
|
| 85 |
+
"sentence_idx": fact.source_sentence_idx,
|
| 86 |
+
"confidence": fact.extraction_confidence,
|
| 87 |
+
"extractor": fact.extractor,
|
| 88 |
+
"article_id": article_id,
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
# MERGE entity nodes so the same entity is never duplicated across articles
|
| 92 |
+
tx.run("""
|
| 93 |
+
MERGE (s:Entity {entity_id: $subj_id})
|
| 94 |
+
ON CREATE SET s.text = $subj_text, s.entity_type = $subj_type,
|
| 95 |
+
s.wikidata_id = $subj_wikidata
|
| 96 |
+
MERGE (o:Entity {entity_id: $obj_id})
|
| 97 |
+
ON CREATE SET o.text = $obj_text, o.entity_type = $obj_type,
|
| 98 |
+
o.wikidata_id = $obj_wikidata
|
| 99 |
+
CREATE (s)-[:TEMPORAL_RELATION {
|
| 100 |
+
relation: $relation,
|
| 101 |
+
time_start_iso: $time_start_iso,
|
| 102 |
+
time_start_str: $time_start_str,
|
| 103 |
+
time_end_iso: $time_end_iso,
|
| 104 |
+
time_end_str: $time_end_str,
|
| 105 |
+
time_point_iso: $time_point_iso,
|
| 106 |
+
time_point_str: $time_point_str,
|
| 107 |
+
source_sentence_idx: $sentence_idx,
|
| 108 |
+
confidence: $confidence,
|
| 109 |
+
extractor: $extractor,
|
| 110 |
+
article_id: $article_id
|
| 111 |
+
}]->(o)
|
| 112 |
+
""", **params)
|
| 113 |
+
|
| 114 |
+
if article_id:
|
| 115 |
+
tx.run("MERGE (a:Article {article_id: $article_id})", article_id=article_id)
|
| 116 |
+
|
| 117 |
+
def add_facts(self, facts: list[TemporalFact], article_id: str | None = None) -> None:
|
| 118 |
+
for fact in facts:
|
| 119 |
+
self.add_fact(fact, article_id)
|
| 120 |
+
logger.info(
|
| 121 |
+
f"Neo4jTKGStore: {len(facts)} facts written"
|
| 122 |
+
+ (f" for article {article_id}" if article_id else "")
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
# ── Read ──
|
| 126 |
+
|
| 127 |
+
def get_all_facts(self, article_id: str | None = None) -> list[TemporalFact]:
|
| 128 |
+
"""Retrieve facts, optionally filtered by article_id."""
|
| 129 |
+
if article_id:
|
| 130 |
+
query = f"""
|
| 131 |
+
MATCH (s:Entity)-[r:TEMPORAL_RELATION {{article_id: $article_id}}]->(o:Entity)
|
| 132 |
+
RETURN {_FACT_RETURN}
|
| 133 |
+
"""
|
| 134 |
+
params: dict = {"article_id": article_id}
|
| 135 |
+
else:
|
| 136 |
+
query = f"""
|
| 137 |
+
MATCH (s:Entity)-[r:TEMPORAL_RELATION]->(o:Entity)
|
| 138 |
+
RETURN {_FACT_RETURN}
|
| 139 |
+
"""
|
| 140 |
+
params = {}
|
| 141 |
+
|
| 142 |
+
with self._driver.session() as session:
|
| 143 |
+
result = session.run(query, **params)
|
| 144 |
+
return [_record_to_fact(r) for r in result]
|
| 145 |
+
|
| 146 |
+
def get_facts_for_entity(self, entity_name: str) -> list[TemporalFact]:
|
| 147 |
+
"""All facts across ALL articles where the entity appears as subject or object."""
|
| 148 |
+
entity_id = entity_name.lower().strip()
|
| 149 |
+
query = f"""
|
| 150 |
+
MATCH (s:Entity)-[r:TEMPORAL_RELATION]->(o:Entity)
|
| 151 |
+
WHERE s.entity_id = $entity_id OR o.entity_id = $entity_id
|
| 152 |
+
RETURN {_FACT_RETURN}
|
| 153 |
+
"""
|
| 154 |
+
with self._driver.session() as session:
|
| 155 |
+
result = session.run(query, entity_id=entity_id)
|
| 156 |
+
return [_record_to_fact(r) for r in result]
|
| 157 |
+
|
| 158 |
+
def get_articles(self) -> list[dict]:
|
| 159 |
+
"""List all analyzed articles with their fact counts."""
|
| 160 |
+
with self._driver.session() as session:
|
| 161 |
+
result = session.run("""
|
| 162 |
+
MATCH (a:Article)
|
| 163 |
+
OPTIONAL MATCH ()-[r:TEMPORAL_RELATION {article_id: a.article_id}]->()
|
| 164 |
+
RETURN a.article_id AS article_id,
|
| 165 |
+
a.title AS title,
|
| 166 |
+
a.source AS source,
|
| 167 |
+
a.analyzed_at AS analyzed_at,
|
| 168 |
+
count(r) AS fact_count
|
| 169 |
+
ORDER BY a.analyzed_at DESC
|
| 170 |
+
""")
|
| 171 |
+
return [dict(record) for record in result]
|
| 172 |
+
|
| 173 |
+
def delete_article(self, article_id: str) -> bool:
|
| 174 |
+
"""Delete an article node and all TEMPORAL_RELATION edges that belong to it."""
|
| 175 |
+
with self._driver.session() as session:
|
| 176 |
+
check = session.run(
|
| 177 |
+
"MATCH (a:Article {article_id: $id}) RETURN count(a) > 0 AS found",
|
| 178 |
+
id=article_id,
|
| 179 |
+
).single()
|
| 180 |
+
if not (check and check["found"]):
|
| 181 |
+
return False
|
| 182 |
+
session.run(
|
| 183 |
+
"MATCH ()-[r:TEMPORAL_RELATION {article_id: $id}]->() DELETE r",
|
| 184 |
+
id=article_id,
|
| 185 |
+
)
|
| 186 |
+
session.run(
|
| 187 |
+
"MATCH (a:Article {article_id: $id}) DELETE a",
|
| 188 |
+
id=article_id,
|
| 189 |
+
)
|
| 190 |
+
return True
|
| 191 |
+
|
| 192 |
+
def summary(self) -> dict:
|
| 193 |
+
with self._driver.session() as session:
|
| 194 |
+
result = session.run("""
|
| 195 |
+
MATCH (e:Entity) WITH count(e) AS nodes
|
| 196 |
+
MATCH ()-[r:TEMPORAL_RELATION]->() WITH nodes, count(r) AS edges
|
| 197 |
+
OPTIONAL MATCH (a:Article) WITH nodes, edges, count(a) AS articles
|
| 198 |
+
RETURN nodes, edges, articles
|
| 199 |
+
""").single()
|
| 200 |
+
if result is None:
|
| 201 |
+
return {"nodes": 0, "edges": 0, "articles": 0}
|
| 202 |
+
return {
|
| 203 |
+
"nodes": result["nodes"],
|
| 204 |
+
"edges": result["edges"],
|
| 205 |
+
"articles": result["articles"],
|
| 206 |
+
}
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
# ── Helpers ──
|
| 210 |
+
|
| 211 |
+
def _record_to_fact(record) -> TemporalFact:
|
| 212 |
+
"""Reconstruct a TemporalFact from a Neo4j query record."""
|
| 213 |
+
|
| 214 |
+
def parse_entity_type(val: str) -> EntityType:
|
| 215 |
+
try:
|
| 216 |
+
return EntityType(val)
|
| 217 |
+
except ValueError:
|
| 218 |
+
return EntityType.OTHER
|
| 219 |
+
|
| 220 |
+
def parse_relation_type(val: str) -> RelationType:
|
| 221 |
+
try:
|
| 222 |
+
return RelationType(val)
|
| 223 |
+
except ValueError:
|
| 224 |
+
return RelationType.GENERIC
|
| 225 |
+
|
| 226 |
+
def parse_temporal_expr(
|
| 227 |
+
iso: str | None, date_str: str | None
|
| 228 |
+
) -> TemporalExpression | None:
|
| 229 |
+
if iso is None:
|
| 230 |
+
return None
|
| 231 |
+
try:
|
| 232 |
+
dt: datetime | None = datetime.fromisoformat(iso)
|
| 233 |
+
except (ValueError, TypeError):
|
| 234 |
+
dt = None
|
| 235 |
+
return TemporalExpression(raw_text=date_str or "", normalized_date=dt, date_string=date_str)
|
| 236 |
+
|
| 237 |
+
subject = Entity(
|
| 238 |
+
text=record["s_text"],
|
| 239 |
+
entity_type=parse_entity_type(record["s_type"]),
|
| 240 |
+
start_char=0,
|
| 241 |
+
end_char=0,
|
| 242 |
+
wikidata_id=record.get("s_wikidata"),
|
| 243 |
+
)
|
| 244 |
+
obj = Entity(
|
| 245 |
+
text=record["o_text"],
|
| 246 |
+
entity_type=parse_entity_type(record["o_type"]),
|
| 247 |
+
start_char=0,
|
| 248 |
+
end_char=0,
|
| 249 |
+
wikidata_id=record.get("o_wikidata"),
|
| 250 |
+
)
|
| 251 |
+
|
| 252 |
+
return TemporalFact(
|
| 253 |
+
subject=subject,
|
| 254 |
+
predicate=parse_relation_type(record["relation"]),
|
| 255 |
+
object=obj,
|
| 256 |
+
time_start=parse_temporal_expr(record.get("time_start_iso"), record.get("time_start_str")),
|
| 257 |
+
time_end=parse_temporal_expr(record.get("time_end_iso"), record.get("time_end_str")),
|
| 258 |
+
time_point=parse_temporal_expr(record.get("time_point_iso"), record.get("time_point_str")),
|
| 259 |
+
source_sentence_idx=record.get("sentence_idx") or 0,
|
| 260 |
+
extraction_confidence=record.get("confidence") or 1.0,
|
| 261 |
+
extractor=record.get("extractor") or "unknown",
|
| 262 |
+
)
|
backend/pipeline/graph/store.py
CHANGED
|
@@ -8,6 +8,7 @@ from typing import Optional
|
|
| 8 |
|
| 9 |
import networkx as nx
|
| 10 |
|
|
|
|
| 11 |
from backend.pipeline.graph.models import (
|
| 12 |
Entity, EntityType, RelationType, TemporalExpression, TemporalFact,
|
| 13 |
)
|
|
@@ -15,7 +16,7 @@ from backend.pipeline.graph.models import (
|
|
| 15 |
logger = logging.getLogger(__name__)
|
| 16 |
|
| 17 |
|
| 18 |
-
class TemporalKnowledgeGraph:
|
| 19 |
"""
|
| 20 |
Graf temporal conform Cai et al. (2024): G = (E, R, T, F).
|
| 21 |
Noduri = entitati, Muchii = relatii cu metadata temporala.
|
|
@@ -26,8 +27,8 @@ class TemporalKnowledgeGraph:
|
|
| 26 |
self._facts: list[TemporalFact] = []
|
| 27 |
|
| 28 |
# ── Adaugare ──
|
| 29 |
-
def add_fact(self, fact: TemporalFact) -> None:
|
| 30 |
-
"""Adauga un TemporalFact ca nod+muchie in graf."""
|
| 31 |
subj_id = _entity_id(fact.subject)
|
| 32 |
obj_id = _entity_id(fact.object)
|
| 33 |
|
|
@@ -40,9 +41,9 @@ class TemporalKnowledgeGraph:
|
|
| 40 |
self._graph.add_edge(subj_id, obj_id, **edge_attrs)
|
| 41 |
self._facts.append(fact)
|
| 42 |
|
| 43 |
-
def add_facts(self, facts: list[TemporalFact]) -> None:
|
| 44 |
for fact in facts:
|
| 45 |
-
self.add_fact(fact)
|
| 46 |
logger.info(f"TKG: {len(facts)} fapte adaugate. Noduri: {self.node_count}, Muchii: {self.edge_count}")
|
| 47 |
|
| 48 |
# ── Interogare noduri ──
|
|
@@ -53,7 +54,8 @@ class TemporalKnowledgeGraph:
|
|
| 53 |
return dict(self._graph.nodes.get(entity_id, {}))
|
| 54 |
|
| 55 |
# ── Interogare fapte ──
|
| 56 |
-
def get_all_facts(self) -> list[TemporalFact]:
|
|
|
|
| 57 |
return list(self._facts)
|
| 58 |
|
| 59 |
def get_facts_for_entity(self, entity_name: str) -> list[TemporalFact]:
|
|
@@ -61,6 +63,14 @@ class TemporalKnowledgeGraph:
|
|
| 61 |
entity_id = entity_name.lower().strip()
|
| 62 |
return [f for f in self._facts if _entity_id(f.subject) == entity_id or _entity_id(f.object) == entity_id]
|
| 63 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
def get_edges_in_interval(self, t_start: datetime, t_end: datetime) -> list[dict]:
|
| 65 |
"""Muchiile active in intervalul [t_start, t_end]."""
|
| 66 |
result = []
|
|
|
|
| 8 |
|
| 9 |
import networkx as nx
|
| 10 |
|
| 11 |
+
from backend.pipeline.graph.base_store import AbstractTKGStore
|
| 12 |
from backend.pipeline.graph.models import (
|
| 13 |
Entity, EntityType, RelationType, TemporalExpression, TemporalFact,
|
| 14 |
)
|
|
|
|
| 16 |
logger = logging.getLogger(__name__)
|
| 17 |
|
| 18 |
|
| 19 |
+
class TemporalKnowledgeGraph(AbstractTKGStore):
|
| 20 |
"""
|
| 21 |
Graf temporal conform Cai et al. (2024): G = (E, R, T, F).
|
| 22 |
Noduri = entitati, Muchii = relatii cu metadata temporala.
|
|
|
|
| 27 |
self._facts: list[TemporalFact] = []
|
| 28 |
|
| 29 |
# ── Adaugare ──
|
| 30 |
+
def add_fact(self, fact: TemporalFact, article_id: str | None = None) -> None:
|
| 31 |
+
"""Adauga un TemporalFact ca nod+muchie in graf. article_id ignorat (in-memory)."""
|
| 32 |
subj_id = _entity_id(fact.subject)
|
| 33 |
obj_id = _entity_id(fact.object)
|
| 34 |
|
|
|
|
| 41 |
self._graph.add_edge(subj_id, obj_id, **edge_attrs)
|
| 42 |
self._facts.append(fact)
|
| 43 |
|
| 44 |
+
def add_facts(self, facts: list[TemporalFact], article_id: str | None = None) -> None:
|
| 45 |
for fact in facts:
|
| 46 |
+
self.add_fact(fact, article_id)
|
| 47 |
logger.info(f"TKG: {len(facts)} fapte adaugate. Noduri: {self.node_count}, Muchii: {self.edge_count}")
|
| 48 |
|
| 49 |
# ── Interogare noduri ──
|
|
|
|
| 54 |
return dict(self._graph.nodes.get(entity_id, {}))
|
| 55 |
|
| 56 |
# ── Interogare fapte ──
|
| 57 |
+
def get_all_facts(self, article_id: str | None = None) -> list[TemporalFact]:
|
| 58 |
+
# article_id ignored — in-memory store holds one article at a time
|
| 59 |
return list(self._facts)
|
| 60 |
|
| 61 |
def get_facts_for_entity(self, entity_name: str) -> list[TemporalFact]:
|
|
|
|
| 63 |
entity_id = entity_name.lower().strip()
|
| 64 |
return [f for f in self._facts if _entity_id(f.subject) == entity_id or _entity_id(f.object) == entity_id]
|
| 65 |
|
| 66 |
+
def get_articles(self) -> list[dict]:
|
| 67 |
+
# in-memory store has no persistence; articles are not tracked
|
| 68 |
+
return []
|
| 69 |
+
|
| 70 |
+
def delete_article(self, article_id: str) -> bool:
|
| 71 |
+
# in-memory store has no persistence
|
| 72 |
+
return False
|
| 73 |
+
|
| 74 |
def get_edges_in_interval(self, t_start: datetime, t_end: datetime) -> list[dict]:
|
| 75 |
"""Muchiile active in intervalul [t_start, t_end]."""
|
| 76 |
result = []
|
backend/pipeline/verification/cross_article.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""C3c — Cross-article inconsistency detection using the persistent TKG store."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import logging
|
| 6 |
+
from datetime import datetime
|
| 7 |
+
from difflib import SequenceMatcher
|
| 8 |
+
from typing import Optional
|
| 9 |
+
|
| 10 |
+
from backend.pipeline.graph.base_store import AbstractTKGStore
|
| 11 |
+
from backend.pipeline.graph.models import (
|
| 12 |
+
Inconsistency, InconsistencyType, Severity, TemporalFact,
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
logger = logging.getLogger(__name__)
|
| 16 |
+
|
| 17 |
+
SIMILARITY_THRESHOLD = 0.80
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class CrossArticleVerifier:
|
| 21 |
+
"""Detects inconsistencies between facts in the current article and
|
| 22 |
+
previously analyzed articles stored in Neo4j."""
|
| 23 |
+
|
| 24 |
+
def __init__(self, persistent_store: AbstractTKGStore) -> None:
|
| 25 |
+
self.store = persistent_store
|
| 26 |
+
|
| 27 |
+
def verify(
|
| 28 |
+
self,
|
| 29 |
+
current_facts: list[TemporalFact],
|
| 30 |
+
current_article_id: str,
|
| 31 |
+
) -> list[Inconsistency]:
|
| 32 |
+
"""Compare current article facts against all stored facts for the same entities.
|
| 33 |
+
|
| 34 |
+
Call this BEFORE persisting the current article so stored results belong
|
| 35 |
+
exclusively to prior articles.
|
| 36 |
+
"""
|
| 37 |
+
inconsistencies: list[Inconsistency] = []
|
| 38 |
+
|
| 39 |
+
for fact in current_facts:
|
| 40 |
+
entity_name = fact.subject.text.lower().strip()
|
| 41 |
+
stored_facts = self.store.get_facts_for_entity(entity_name)
|
| 42 |
+
|
| 43 |
+
# safety guard: skip any fact already tagged to this article
|
| 44 |
+
other_facts = [
|
| 45 |
+
f for f in stored_facts
|
| 46 |
+
if getattr(f, "article_id", None) != current_article_id
|
| 47 |
+
]
|
| 48 |
+
|
| 49 |
+
for stored in other_facts:
|
| 50 |
+
# check 1: same relation + similar object + conflicting dates
|
| 51 |
+
if (
|
| 52 |
+
fact.predicate == stored.predicate
|
| 53 |
+
and _objects_similar(fact.object.text, stored.object.text)
|
| 54 |
+
):
|
| 55 |
+
conflict = _check_date_conflict(fact, stored)
|
| 56 |
+
if conflict:
|
| 57 |
+
inconsistencies.append(conflict)
|
| 58 |
+
|
| 59 |
+
# check 2: same entity holds different positions at overlapping times
|
| 60 |
+
if (
|
| 61 |
+
fact.predicate.value == "holds_position"
|
| 62 |
+
and stored.predicate.value == "holds_position"
|
| 63 |
+
and _overlapping_time(fact, stored)
|
| 64 |
+
and not _objects_similar(fact.object.text, stored.object.text)
|
| 65 |
+
):
|
| 66 |
+
inconsistencies.append(
|
| 67 |
+
Inconsistency(
|
| 68 |
+
inconsistency_type=InconsistencyType.DATE_MISMATCH,
|
| 69 |
+
severity=Severity.HIGH,
|
| 70 |
+
description=(
|
| 71 |
+
f"Cross-article conflict: '{fact.subject.text}' holds "
|
| 72 |
+
f"different positions at overlapping times: "
|
| 73 |
+
f"'{fact.object.text}' vs '{stored.object.text}'"
|
| 74 |
+
),
|
| 75 |
+
facts_involved=[fact, stored],
|
| 76 |
+
sentence_indices=[fact.source_sentence_idx],
|
| 77 |
+
verified_by="cross_article",
|
| 78 |
+
evidence=f"Article {current_article_id} vs stored facts",
|
| 79 |
+
)
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
logger.info(
|
| 83 |
+
f"Cross-article verification: {len(inconsistencies)} conflicts found "
|
| 84 |
+
f"for article {current_article_id}"
|
| 85 |
+
)
|
| 86 |
+
return inconsistencies
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _objects_similar(a: str, b: str) -> bool:
|
| 90 |
+
"""True when strings are equal (case-insensitive) or fuzzy-match above threshold."""
|
| 91 |
+
a_norm, b_norm = a.lower().strip(), b.lower().strip()
|
| 92 |
+
if a_norm == b_norm:
|
| 93 |
+
return True
|
| 94 |
+
return SequenceMatcher(None, a_norm, b_norm).ratio() >= SIMILARITY_THRESHOLD
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _check_date_conflict(
|
| 98 |
+
fact_a: TemporalFact, fact_b: TemporalFact
|
| 99 |
+
) -> Optional[Inconsistency]:
|
| 100 |
+
"""Return an Inconsistency when the same claim appears with dates more than 1 year apart."""
|
| 101 |
+
time_a = _best_date(fact_a)
|
| 102 |
+
time_b = _best_date(fact_b)
|
| 103 |
+
if time_a is None or time_b is None:
|
| 104 |
+
return None
|
| 105 |
+
|
| 106 |
+
if abs((time_a - time_b).days) <= 365:
|
| 107 |
+
return None
|
| 108 |
+
|
| 109 |
+
return Inconsistency(
|
| 110 |
+
inconsistency_type=InconsistencyType.DATE_MISMATCH,
|
| 111 |
+
severity=Severity.MEDIUM,
|
| 112 |
+
description=(
|
| 113 |
+
f"Cross-article date conflict: '{fact_a.subject.text} "
|
| 114 |
+
f"{fact_a.predicate.value} {fact_a.object.text}' — "
|
| 115 |
+
f"{time_a.year} (current article) vs {time_b.year} (stored)"
|
| 116 |
+
),
|
| 117 |
+
facts_involved=[fact_a, fact_b],
|
| 118 |
+
sentence_indices=[fact_a.source_sentence_idx],
|
| 119 |
+
verified_by="cross_article",
|
| 120 |
+
evidence=f"{time_a.date()} (current) vs {time_b.date()} (stored)",
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _overlapping_time(fact_a: TemporalFact, fact_b: TemporalFact) -> bool:
|
| 125 |
+
"""True if the temporal ranges of the two facts overlap."""
|
| 126 |
+
start_a = _range_start(fact_a)
|
| 127 |
+
start_b = _range_start(fact_b)
|
| 128 |
+
if start_a is None or start_b is None:
|
| 129 |
+
return False
|
| 130 |
+
|
| 131 |
+
end_a = _range_end(fact_a) or datetime.max
|
| 132 |
+
end_b = _range_end(fact_b) or datetime.max
|
| 133 |
+
|
| 134 |
+
return start_a <= end_b and start_b <= end_a
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def _best_date(fact: TemporalFact) -> Optional[datetime]:
|
| 138 |
+
"""Most representative single datetime: time_point > time_start > time_end."""
|
| 139 |
+
for attr in ("time_point", "time_start", "time_end"):
|
| 140 |
+
expr = getattr(fact, attr)
|
| 141 |
+
if expr and expr.normalized_date:
|
| 142 |
+
return expr.normalized_date
|
| 143 |
+
return None
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def _range_start(fact: TemporalFact) -> Optional[datetime]:
|
| 147 |
+
for attr in ("time_start", "time_point"):
|
| 148 |
+
expr = getattr(fact, attr)
|
| 149 |
+
if expr and expr.normalized_date:
|
| 150 |
+
return expr.normalized_date
|
| 151 |
+
return None
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def _range_end(fact: TemporalFact) -> Optional[datetime]:
|
| 155 |
+
for attr in ("time_end", "time_point"):
|
| 156 |
+
expr = getattr(fact, attr)
|
| 157 |
+
if expr and expr.normalized_date:
|
| 158 |
+
return expr.normalized_date
|
| 159 |
+
return None
|
backend/routers/analyze.py
CHANGED
|
@@ -3,13 +3,14 @@
|
|
| 3 |
from __future__ import annotations
|
| 4 |
|
| 5 |
import logging
|
|
|
|
| 6 |
from datetime import datetime
|
| 7 |
from typing import Optional
|
| 8 |
|
| 9 |
from fastapi import APIRouter, HTTPException
|
| 10 |
from pydantic import BaseModel, Field
|
| 11 |
|
| 12 |
-
from backend.pipeline.graph.models import Article
|
| 13 |
from backend.pipeline.orchestrator import PipelineOrchestrator
|
| 14 |
from backend.routers.dependencies import get_orchestrator, explainer
|
| 15 |
|
|
@@ -26,6 +27,7 @@ class AnalyzeRequest(BaseModel):
|
|
| 26 |
source: str = Field(default="", description="Sursa articolului")
|
| 27 |
pipeline: str = Field(default="spacy", description="Pipeline: 'spacy' sau 'llm'")
|
| 28 |
model: Optional[str] = Field(default=None, description="Model specific: en_core_web_trf, llama3, mistral, etc. None = default per pipeline")
|
|
|
|
| 29 |
|
| 30 |
|
| 31 |
class InconsistencyResponse(BaseModel):
|
|
@@ -65,11 +67,26 @@ class AnalyzeResponse(BaseModel):
|
|
| 65 |
pipeline: str
|
| 66 |
model: str = Field(default="", description="Model folosit")
|
| 67 |
processing_time_ms: float
|
|
|
|
|
|
|
| 68 |
|
| 69 |
|
| 70 |
_explainer = explainer
|
| 71 |
|
| 72 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
# Endpoint
|
| 74 |
@router.post("/analyze", response_model=AnalyzeResponse)
|
| 75 |
async def analyze_article(req: AnalyzeRequest) -> AnalyzeResponse:
|
|
@@ -108,6 +125,37 @@ async def analyze_article(req: AnalyzeRequest) -> AnalyzeResponse:
|
|
| 108 |
logger.error(f"/analyze: eroare pipeline — {e}", exc_info=True)
|
| 109 |
raise HTTPException(status_code=500, detail="Eroare interna la procesare.")
|
| 110 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
# Genereaza explicatii structurate
|
| 112 |
explanation = _explainer.explain_structured(result)
|
| 113 |
|
|
@@ -124,4 +172,6 @@ async def analyze_article(req: AnalyzeRequest) -> AnalyzeResponse:
|
|
| 124 |
pipeline=result.pipeline_variant,
|
| 125 |
model=req.model or AVAILABLE_MODELS[req.pipeline]["default"],
|
| 126 |
processing_time_ms=result.processing_time_ms,
|
|
|
|
|
|
|
| 127 |
)
|
|
|
|
| 3 |
from __future__ import annotations
|
| 4 |
|
| 5 |
import logging
|
| 6 |
+
import uuid
|
| 7 |
from datetime import datetime
|
| 8 |
from typing import Optional
|
| 9 |
|
| 10 |
from fastapi import APIRouter, HTTPException
|
| 11 |
from pydantic import BaseModel, Field
|
| 12 |
|
| 13 |
+
from backend.pipeline.graph.models import Article, Inconsistency
|
| 14 |
from backend.pipeline.orchestrator import PipelineOrchestrator
|
| 15 |
from backend.routers.dependencies import get_orchestrator, explainer
|
| 16 |
|
|
|
|
| 27 |
source: str = Field(default="", description="Sursa articolului")
|
| 28 |
pipeline: str = Field(default="spacy", description="Pipeline: 'spacy' sau 'llm'")
|
| 29 |
model: Optional[str] = Field(default=None, description="Model specific: en_core_web_trf, llama3, mistral, etc. None = default per pipeline")
|
| 30 |
+
persist: bool = Field(default=False, description="Save facts to Neo4j for cross-article analysis")
|
| 31 |
|
| 32 |
|
| 33 |
class InconsistencyResponse(BaseModel):
|
|
|
|
| 67 |
pipeline: str
|
| 68 |
model: str = Field(default="", description="Model folosit")
|
| 69 |
processing_time_ms: float
|
| 70 |
+
article_id: Optional[str] = Field(default=None, description="Neo4j article ID (set when persist=True)")
|
| 71 |
+
cross_article_inconsistencies: list[InconsistencyResponse] = Field(default_factory=list)
|
| 72 |
|
| 73 |
|
| 74 |
_explainer = explainer
|
| 75 |
|
| 76 |
|
| 77 |
+
def _to_inconsistency_response(inc: Inconsistency) -> InconsistencyResponse:
|
| 78 |
+
"""Serialize an Inconsistency dataclass to the API response model."""
|
| 79 |
+
return InconsistencyResponse(
|
| 80 |
+
type=inc.inconsistency_type.value,
|
| 81 |
+
severity=inc.severity.value,
|
| 82 |
+
severity_label=inc.severity.value.title(),
|
| 83 |
+
description=inc.description,
|
| 84 |
+
evidence=inc.evidence,
|
| 85 |
+
verified_by=inc.verified_by,
|
| 86 |
+
sentence_indices=inc.sentence_indices,
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
# Endpoint
|
| 91 |
@router.post("/analyze", response_model=AnalyzeResponse)
|
| 92 |
async def analyze_article(req: AnalyzeRequest) -> AnalyzeResponse:
|
|
|
|
| 125 |
logger.error(f"/analyze: eroare pipeline — {e}", exc_info=True)
|
| 126 |
raise HTTPException(status_code=500, detail="Eroare interna la procesare.")
|
| 127 |
|
| 128 |
+
# optional: persist to Neo4j and run cross-article verification
|
| 129 |
+
article_id: Optional[str] = None
|
| 130 |
+
cross_article_inconsistencies: list[InconsistencyResponse] = []
|
| 131 |
+
|
| 132 |
+
if req.persist:
|
| 133 |
+
from backend.config import NEO4J_ENABLED
|
| 134 |
+
if NEO4J_ENABLED:
|
| 135 |
+
from backend.pipeline.graph.factory import create_persistent_store
|
| 136 |
+
from backend.pipeline.verification.cross_article import CrossArticleVerifier
|
| 137 |
+
|
| 138 |
+
article_id = str(uuid.uuid4())
|
| 139 |
+
store = create_persistent_store()
|
| 140 |
+
if store is not None:
|
| 141 |
+
try:
|
| 142 |
+
verifier = CrossArticleVerifier(store)
|
| 143 |
+
cross_incs = verifier.verify(result.facts, article_id)
|
| 144 |
+
store.add_facts(result.facts, article_id=article_id)
|
| 145 |
+
cross_article_inconsistencies = [
|
| 146 |
+
_to_inconsistency_response(c) for c in cross_incs
|
| 147 |
+
]
|
| 148 |
+
logger.info(
|
| 149 |
+
f"/analyze: persisted {len(result.facts)} facts as {article_id}, "
|
| 150 |
+
f"{len(cross_incs)} cross-article conflicts"
|
| 151 |
+
)
|
| 152 |
+
except Exception as e:
|
| 153 |
+
logger.error(f"/analyze: persistence error — {e}", exc_info=True)
|
| 154 |
+
finally:
|
| 155 |
+
store.close()
|
| 156 |
+
else:
|
| 157 |
+
logger.warning("/analyze: persist=True but NEO4J_ENABLED=false — skipping")
|
| 158 |
+
|
| 159 |
# Genereaza explicatii structurate
|
| 160 |
explanation = _explainer.explain_structured(result)
|
| 161 |
|
|
|
|
| 172 |
pipeline=result.pipeline_variant,
|
| 173 |
model=req.model or AVAILABLE_MODELS[req.pipeline]["default"],
|
| 174 |
processing_time_ms=result.processing_time_ms,
|
| 175 |
+
article_id=article_id,
|
| 176 |
+
cross_article_inconsistencies=cross_article_inconsistencies,
|
| 177 |
)
|
backend/routers/articles.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Router — GET /articles, DELETE /articles/{article_id}: article management via Neo4j."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import logging
|
| 6 |
+
|
| 7 |
+
from fastapi import APIRouter, HTTPException
|
| 8 |
+
|
| 9 |
+
from backend.pipeline.graph.factory import create_persistent_store
|
| 10 |
+
|
| 11 |
+
logger = logging.getLogger(__name__)
|
| 12 |
+
router = APIRouter(tags=["articles"])
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@router.get("/articles")
|
| 16 |
+
async def list_articles():
|
| 17 |
+
"""List all previously analyzed articles stored in Neo4j."""
|
| 18 |
+
store = create_persistent_store()
|
| 19 |
+
if store is None:
|
| 20 |
+
return {"articles": [], "neo4j_enabled": False}
|
| 21 |
+
try:
|
| 22 |
+
articles = store.get_articles()
|
| 23 |
+
return {"articles": articles, "neo4j_enabled": True}
|
| 24 |
+
except Exception as e:
|
| 25 |
+
logger.error(f"GET /articles error: {e}", exc_info=True)
|
| 26 |
+
raise HTTPException(status_code=500, detail="Failed to retrieve articles from Neo4j.")
|
| 27 |
+
finally:
|
| 28 |
+
store.close()
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@router.delete("/articles/{article_id}")
|
| 32 |
+
async def delete_article(article_id: str):
|
| 33 |
+
"""Delete an article and all its facts from Neo4j."""
|
| 34 |
+
store = create_persistent_store()
|
| 35 |
+
if store is None:
|
| 36 |
+
raise HTTPException(status_code=503, detail="Neo4j not enabled.")
|
| 37 |
+
try:
|
| 38 |
+
deleted = store.delete_article(article_id)
|
| 39 |
+
if not deleted:
|
| 40 |
+
raise HTTPException(
|
| 41 |
+
status_code=404, detail=f"Article '{article_id}' not found."
|
| 42 |
+
)
|
| 43 |
+
return {"deleted": True, "article_id": article_id}
|
| 44 |
+
except HTTPException:
|
| 45 |
+
raise
|
| 46 |
+
except Exception as e:
|
| 47 |
+
logger.error(f"DELETE /articles/{article_id} error: {e}", exc_info=True)
|
| 48 |
+
raise HTTPException(status_code=500, detail="Failed to delete article from Neo4j.")
|
| 49 |
+
finally:
|
| 50 |
+
store.close()
|
backend/routers/batch.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Router — POST /analyze-batch: analyze multiple articles with optional Neo4j persistence."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import logging
|
| 6 |
+
import uuid
|
| 7 |
+
from datetime import datetime
|
| 8 |
+
from typing import Optional
|
| 9 |
+
|
| 10 |
+
from fastapi import APIRouter, HTTPException
|
| 11 |
+
from pydantic import BaseModel, Field
|
| 12 |
+
|
| 13 |
+
from backend.pipeline.graph.models import Article
|
| 14 |
+
from backend.routers.analyze import InconsistencyResponse, _to_inconsistency_response
|
| 15 |
+
from backend.routers.dependencies import explainer, get_orchestrator
|
| 16 |
+
|
| 17 |
+
logger = logging.getLogger(__name__)
|
| 18 |
+
router = APIRouter(tags=["batch"])
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class BatchArticle(BaseModel):
|
| 22 |
+
text: str = Field(..., min_length=20)
|
| 23 |
+
title: str = Field(default="")
|
| 24 |
+
publication_date: Optional[str] = None
|
| 25 |
+
source: str = Field(default="")
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class BatchRequest(BaseModel):
|
| 29 |
+
articles: list[BatchArticle] = Field(..., min_length=1, max_length=50)
|
| 30 |
+
pipeline: str = Field(default="spacy")
|
| 31 |
+
model: Optional[str] = None
|
| 32 |
+
persist: bool = Field(
|
| 33 |
+
default=True,
|
| 34 |
+
description="Save to Neo4j for cross-article analysis",
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class BatchArticleResult(BaseModel):
|
| 39 |
+
article_id: str
|
| 40 |
+
title: str
|
| 41 |
+
score: float
|
| 42 |
+
label: str
|
| 43 |
+
summary: str
|
| 44 |
+
n_claims: int
|
| 45 |
+
n_inconsistencies: int
|
| 46 |
+
n_cross_article_inconsistencies: int
|
| 47 |
+
cross_article_conflicts: list[InconsistencyResponse]
|
| 48 |
+
processing_time_ms: float
|
| 49 |
+
error: Optional[str] = None
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class BatchResponse(BaseModel):
|
| 53 |
+
results: list[BatchArticleResult]
|
| 54 |
+
total_articles: int
|
| 55 |
+
total_cross_article_conflicts: int
|
| 56 |
+
avg_score: float
|
| 57 |
+
neo4j_enabled: bool
|
| 58 |
+
persisted: bool
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@router.post("/analyze-batch", response_model=BatchResponse)
|
| 62 |
+
async def analyze_batch(req: BatchRequest) -> BatchResponse:
|
| 63 |
+
"""Analyze multiple articles. Persist to Neo4j and run cross-article verification when enabled."""
|
| 64 |
+
from backend.config import AVAILABLE_MODELS, NEO4J_ENABLED
|
| 65 |
+
|
| 66 |
+
if req.pipeline not in ("spacy", "llm"):
|
| 67 |
+
raise HTTPException(
|
| 68 |
+
status_code=400,
|
| 69 |
+
detail=f"Unknown pipeline '{req.pipeline}'. Options: 'spacy', 'llm'.",
|
| 70 |
+
)
|
| 71 |
+
if req.model:
|
| 72 |
+
available = AVAILABLE_MODELS.get(req.pipeline, {}).get("models", [])
|
| 73 |
+
if req.model not in available:
|
| 74 |
+
raise HTTPException(
|
| 75 |
+
status_code=400,
|
| 76 |
+
detail=f"Model '{req.model}' not available for pipeline '{req.pipeline}'. Options: {available}",
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
persist = req.persist and NEO4J_ENABLED
|
| 80 |
+
|
| 81 |
+
# open one shared store connection for the whole batch
|
| 82 |
+
store = None
|
| 83 |
+
if persist:
|
| 84 |
+
from backend.pipeline.graph.factory import create_persistent_store
|
| 85 |
+
store = create_persistent_store()
|
| 86 |
+
if store is None:
|
| 87 |
+
persist = False
|
| 88 |
+
|
| 89 |
+
results: list[BatchArticleResult] = []
|
| 90 |
+
total_cross_conflicts = 0
|
| 91 |
+
|
| 92 |
+
try:
|
| 93 |
+
for batch_article in req.articles:
|
| 94 |
+
article_id = str(uuid.uuid4())
|
| 95 |
+
|
| 96 |
+
pub_date: Optional[datetime] = None
|
| 97 |
+
if batch_article.publication_date:
|
| 98 |
+
try:
|
| 99 |
+
pub_date = datetime.strptime(batch_article.publication_date, "%Y-%m-%d")
|
| 100 |
+
except ValueError:
|
| 101 |
+
pass
|
| 102 |
+
|
| 103 |
+
article = Article(
|
| 104 |
+
text=batch_article.text,
|
| 105 |
+
title=batch_article.title,
|
| 106 |
+
publication_date=pub_date,
|
| 107 |
+
source=batch_article.source,
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
try:
|
| 111 |
+
orchestrator = get_orchestrator(req.pipeline, req.model)
|
| 112 |
+
result = orchestrator.run(article)
|
| 113 |
+
except Exception as e:
|
| 114 |
+
logger.error(
|
| 115 |
+
f"/analyze-batch: pipeline error on '{batch_article.title}': {e}",
|
| 116 |
+
exc_info=True,
|
| 117 |
+
)
|
| 118 |
+
results.append(
|
| 119 |
+
BatchArticleResult(
|
| 120 |
+
article_id=article_id,
|
| 121 |
+
title=batch_article.title,
|
| 122 |
+
score=0.0,
|
| 123 |
+
label="Error",
|
| 124 |
+
summary="Pipeline error — see server logs.",
|
| 125 |
+
n_claims=0,
|
| 126 |
+
n_inconsistencies=0,
|
| 127 |
+
n_cross_article_inconsistencies=0,
|
| 128 |
+
cross_article_conflicts=[],
|
| 129 |
+
processing_time_ms=0.0,
|
| 130 |
+
error=str(e)[:300],
|
| 131 |
+
)
|
| 132 |
+
)
|
| 133 |
+
continue
|
| 134 |
+
|
| 135 |
+
explanation = explainer.explain_structured(result)
|
| 136 |
+
|
| 137 |
+
cross_conflicts: list[InconsistencyResponse] = []
|
| 138 |
+
if store is not None:
|
| 139 |
+
from backend.pipeline.verification.cross_article import CrossArticleVerifier
|
| 140 |
+
|
| 141 |
+
try:
|
| 142 |
+
# verify BEFORE persisting so current article is not in the store yet
|
| 143 |
+
verifier = CrossArticleVerifier(store)
|
| 144 |
+
raw_conflicts = verifier.verify(result.facts, article_id)
|
| 145 |
+
store.add_facts(result.facts, article_id=article_id)
|
| 146 |
+
cross_conflicts = [_to_inconsistency_response(c) for c in raw_conflicts]
|
| 147 |
+
total_cross_conflicts += len(cross_conflicts)
|
| 148 |
+
except Exception as e:
|
| 149 |
+
logger.error(
|
| 150 |
+
f"/analyze-batch: persistence error for {article_id}: {e}",
|
| 151 |
+
exc_info=True,
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
results.append(
|
| 155 |
+
BatchArticleResult(
|
| 156 |
+
article_id=article_id,
|
| 157 |
+
title=batch_article.title,
|
| 158 |
+
score=result.score,
|
| 159 |
+
label=result.label,
|
| 160 |
+
summary=explanation["summary"],
|
| 161 |
+
n_claims=result.n_temporal_claims,
|
| 162 |
+
n_inconsistencies=result.n_inconsistencies,
|
| 163 |
+
n_cross_article_inconsistencies=len(cross_conflicts),
|
| 164 |
+
cross_article_conflicts=cross_conflicts,
|
| 165 |
+
processing_time_ms=result.processing_time_ms,
|
| 166 |
+
)
|
| 167 |
+
)
|
| 168 |
+
finally:
|
| 169 |
+
if store is not None:
|
| 170 |
+
store.close()
|
| 171 |
+
|
| 172 |
+
successful = [r for r in results if r.error is None]
|
| 173 |
+
avg_score = (
|
| 174 |
+
round(sum(r.score for r in successful) / len(successful), 4)
|
| 175 |
+
if successful else 0.0
|
| 176 |
+
)
|
| 177 |
+
|
| 178 |
+
return BatchResponse(
|
| 179 |
+
results=results,
|
| 180 |
+
total_articles=len(results),
|
| 181 |
+
total_cross_article_conflicts=total_cross_conflicts,
|
| 182 |
+
avg_score=avg_score,
|
| 183 |
+
neo4j_enabled=NEO4J_ENABLED,
|
| 184 |
+
persisted=persist,
|
| 185 |
+
)
|
backend/routers/upload.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, UploadFile, File, HTTPException
|
| 2 |
+
|
| 3 |
+
router = APIRouter(tags=["upload"])
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
@router.post("/upload")
|
| 7 |
+
async def upload_file(file: UploadFile = File(...)) -> dict:
|
| 8 |
+
"""Extract text from an uploaded file (.txt, .pdf, .docx)."""
|
| 9 |
+
filename = file.filename or ""
|
| 10 |
+
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
| 11 |
+
content = await file.read()
|
| 12 |
+
|
| 13 |
+
if ext == "txt":
|
| 14 |
+
text = content.decode("utf-8", errors="replace")
|
| 15 |
+
|
| 16 |
+
elif ext == "pdf":
|
| 17 |
+
from PyPDF2 import PdfReader
|
| 18 |
+
import io
|
| 19 |
+
reader = PdfReader(io.BytesIO(content))
|
| 20 |
+
text = "\n\n".join(page.extract_text() or "" for page in reader.pages)
|
| 21 |
+
|
| 22 |
+
elif ext in ("docx", "doc"):
|
| 23 |
+
from docx import Document
|
| 24 |
+
import io
|
| 25 |
+
doc = Document(io.BytesIO(content))
|
| 26 |
+
text = "\n\n".join(p.text for p in doc.paragraphs if p.text.strip())
|
| 27 |
+
|
| 28 |
+
else:
|
| 29 |
+
raise HTTPException(
|
| 30 |
+
status_code=400,
|
| 31 |
+
detail=f"Unsupported file type: .{ext}. Use .txt, .pdf, or .docx",
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
if not text.strip():
|
| 35 |
+
raise HTTPException(status_code=400, detail="No text could be extracted from the file")
|
| 36 |
+
|
| 37 |
+
return {
|
| 38 |
+
"text": text.strip(),
|
| 39 |
+
"filename": filename,
|
| 40 |
+
"file_type": ext,
|
| 41 |
+
"char_count": len(text.strip()),
|
| 42 |
+
}
|
frontend/src/App.tsx
CHANGED
|
@@ -3,13 +3,19 @@ import {
|
|
| 3 |
AppBar,
|
| 4 |
Box,
|
| 5 |
Container,
|
|
|
|
| 6 |
Tab,
|
| 7 |
Tabs,
|
| 8 |
Toolbar,
|
|
|
|
| 9 |
Typography,
|
| 10 |
} from "@mui/material";
|
| 11 |
-
import {
|
|
|
|
|
|
|
| 12 |
import AnalyzeTab from "./components/AnalyzeTab";
|
|
|
|
|
|
|
| 13 |
import CompareTab from "./components/CompareTab";
|
| 14 |
|
| 15 |
type HealthStatus = "checking" | "ok" | "error";
|
|
@@ -17,6 +23,8 @@ type HealthStatus = "checking" | "ok" | "error";
|
|
| 17 |
function App(): React.ReactElement {
|
| 18 |
const [activeTab, setActiveTab] = useState<number>(0);
|
| 19 |
const [healthStatus, setHealthStatus] = useState<HealthStatus>("checking");
|
|
|
|
|
|
|
| 20 |
|
| 21 |
useEffect(() => {
|
| 22 |
checkHealth()
|
|
@@ -24,6 +32,12 @@ function App(): React.ReactElement {
|
|
| 24 |
.catch(() => setHealthStatus("error"));
|
| 25 |
}, []);
|
| 26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
const healthColor: string =
|
| 28 |
healthStatus === "ok" ? "#22c55e" : healthStatus === "error" ? "#ef4444" : "#94a3b8";
|
| 29 |
|
|
@@ -61,19 +75,31 @@ function App(): React.ReactElement {
|
|
| 61 |
</Typography>
|
| 62 |
</Box>
|
| 63 |
|
| 64 |
-
<Box sx={{ display: "flex", alignItems: "center", gap:
|
| 65 |
-
<Box
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
</Box>
|
| 78 |
</Toolbar>
|
| 79 |
</AppBar>
|
|
@@ -92,6 +118,7 @@ function App(): React.ReactElement {
|
|
| 92 |
>
|
| 93 |
<Tab label="Analyze" />
|
| 94 |
<Tab label="Compare" />
|
|
|
|
| 95 |
</Tabs>
|
| 96 |
</Container>
|
| 97 |
</Box>
|
|
@@ -100,6 +127,7 @@ function App(): React.ReactElement {
|
|
| 100 |
<Container maxWidth="lg" sx={{ py: 3 }}>
|
| 101 |
{activeTab === 0 && <AnalyzeTab />}
|
| 102 |
{activeTab === 1 && <CompareTab />}
|
|
|
|
| 103 |
</Container>
|
| 104 |
</Box>
|
| 105 |
|
|
@@ -115,6 +143,8 @@ function App(): React.ReactElement {
|
|
| 115 |
Bachelor Thesis — UTCN 2026 — Marius Pantea
|
| 116 |
</Typography>
|
| 117 |
</Box>
|
|
|
|
|
|
|
| 118 |
</Box>
|
| 119 |
);
|
| 120 |
}
|
|
|
|
| 3 |
AppBar,
|
| 4 |
Box,
|
| 5 |
Container,
|
| 6 |
+
IconButton,
|
| 7 |
Tab,
|
| 8 |
Tabs,
|
| 9 |
Toolbar,
|
| 10 |
+
Tooltip,
|
| 11 |
Typography,
|
| 12 |
} from "@mui/material";
|
| 13 |
+
import { History } from "@mui/icons-material";
|
| 14 |
+
import { checkHealth, getModels } from "./api/client";
|
| 15 |
+
import type { ModelsResponse } from "./types";
|
| 16 |
import AnalyzeTab from "./components/AnalyzeTab";
|
| 17 |
+
import ArticleHistory from "./components/ArticleHistory";
|
| 18 |
+
import BatchTab from "./components/BatchTab";
|
| 19 |
import CompareTab from "./components/CompareTab";
|
| 20 |
|
| 21 |
type HealthStatus = "checking" | "ok" | "error";
|
|
|
|
| 23 |
function App(): React.ReactElement {
|
| 24 |
const [activeTab, setActiveTab] = useState<number>(0);
|
| 25 |
const [healthStatus, setHealthStatus] = useState<HealthStatus>("checking");
|
| 26 |
+
const [availableModels, setAvailableModels] = useState<ModelsResponse | undefined>(undefined);
|
| 27 |
+
const [historyOpen, setHistoryOpen] = useState<boolean>(false);
|
| 28 |
|
| 29 |
useEffect(() => {
|
| 30 |
checkHealth()
|
|
|
|
| 32 |
.catch(() => setHealthStatus("error"));
|
| 33 |
}, []);
|
| 34 |
|
| 35 |
+
useEffect(() => {
|
| 36 |
+
getModels()
|
| 37 |
+
.then(setAvailableModels)
|
| 38 |
+
.catch(() => undefined);
|
| 39 |
+
}, []);
|
| 40 |
+
|
| 41 |
const healthColor: string =
|
| 42 |
healthStatus === "ok" ? "#22c55e" : healthStatus === "error" ? "#ef4444" : "#94a3b8";
|
| 43 |
|
|
|
|
| 75 |
</Typography>
|
| 76 |
</Box>
|
| 77 |
|
| 78 |
+
<Box sx={{ display: "flex", alignItems: "center", gap: 1.5 }}>
|
| 79 |
+
<Box sx={{ display: "flex", alignItems: "center", gap: 0.75 }}>
|
| 80 |
+
<Box
|
| 81 |
+
sx={{
|
| 82 |
+
width: 8,
|
| 83 |
+
height: 8,
|
| 84 |
+
borderRadius: "50%",
|
| 85 |
+
bgcolor: healthColor,
|
| 86 |
+
flexShrink: 0,
|
| 87 |
+
}}
|
| 88 |
+
/>
|
| 89 |
+
<Typography variant="caption" color="text.secondary">
|
| 90 |
+
{healthLabel}
|
| 91 |
+
</Typography>
|
| 92 |
+
</Box>
|
| 93 |
+
|
| 94 |
+
<Tooltip title="Article history (Neo4j)">
|
| 95 |
+
<IconButton
|
| 96 |
+
size="small"
|
| 97 |
+
onClick={() => setHistoryOpen(true)}
|
| 98 |
+
aria-label="Open article history"
|
| 99 |
+
>
|
| 100 |
+
<History fontSize="small" />
|
| 101 |
+
</IconButton>
|
| 102 |
+
</Tooltip>
|
| 103 |
</Box>
|
| 104 |
</Toolbar>
|
| 105 |
</AppBar>
|
|
|
|
| 118 |
>
|
| 119 |
<Tab label="Analyze" />
|
| 120 |
<Tab label="Compare" />
|
| 121 |
+
<Tab label="Batch" />
|
| 122 |
</Tabs>
|
| 123 |
</Container>
|
| 124 |
</Box>
|
|
|
|
| 127 |
<Container maxWidth="lg" sx={{ py: 3 }}>
|
| 128 |
{activeTab === 0 && <AnalyzeTab />}
|
| 129 |
{activeTab === 1 && <CompareTab />}
|
| 130 |
+
{activeTab === 2 && <BatchTab availableModels={availableModels} />}
|
| 131 |
</Container>
|
| 132 |
</Box>
|
| 133 |
|
|
|
|
| 143 |
Bachelor Thesis — UTCN 2026 — Marius Pantea
|
| 144 |
</Typography>
|
| 145 |
</Box>
|
| 146 |
+
|
| 147 |
+
<ArticleHistory open={historyOpen} onClose={() => setHistoryOpen(false)} />
|
| 148 |
</Box>
|
| 149 |
);
|
| 150 |
}
|
frontend/src/api/client.ts
CHANGED
|
@@ -1,7 +1,13 @@
|
|
| 1 |
/* client.ts — Axios instance and typed API functions */
|
| 2 |
|
| 3 |
import axios from "axios";
|
| 4 |
-
import type {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
|
| 6 |
// axios instance
|
| 7 |
const apiClient = axios.create({
|
|
@@ -33,4 +39,35 @@ export async function getModels(): Promise<ModelsResponse> {
|
|
| 33 |
return data;
|
| 34 |
}
|
| 35 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
export default apiClient;
|
|
|
|
| 1 |
/* client.ts — Axios instance and typed API functions */
|
| 2 |
|
| 3 |
import axios from "axios";
|
| 4 |
+
import type {
|
| 5 |
+
AnalyzeRequest, AnalyzeResponse,
|
| 6 |
+
ArticlesResponse,
|
| 7 |
+
BatchRequest, BatchResponse,
|
| 8 |
+
CompareRequest, CompareResponse,
|
| 9 |
+
HealthResponse, ModelsResponse,
|
| 10 |
+
} from "../types";
|
| 11 |
|
| 12 |
// axios instance
|
| 13 |
const apiClient = axios.create({
|
|
|
|
| 39 |
return data;
|
| 40 |
}
|
| 41 |
|
| 42 |
+
export async function analyzeBatch(req: BatchRequest): Promise<BatchResponse> {
|
| 43 |
+
const { data } = await apiClient.post<BatchResponse>("/analyze-batch", req);
|
| 44 |
+
return data;
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
export async function getArticles(): Promise<ArticlesResponse> {
|
| 48 |
+
const { data } = await apiClient.get<ArticlesResponse>("/articles");
|
| 49 |
+
return data;
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
export async function deleteArticle(articleId: string): Promise<void> {
|
| 53 |
+
await apiClient.delete(`/articles/${encodeURIComponent(articleId)}`);
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
export interface UploadResponse {
|
| 57 |
+
text: string;
|
| 58 |
+
filename: string;
|
| 59 |
+
file_type: string;
|
| 60 |
+
char_count: number;
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
export async function uploadFile(file: File): Promise<UploadResponse> {
|
| 64 |
+
const formData = new FormData();
|
| 65 |
+
formData.append("file", file);
|
| 66 |
+
const { data } = await apiClient.post<UploadResponse>("/upload", formData, {
|
| 67 |
+
headers: { "Content-Type": "multipart/form-data" },
|
| 68 |
+
timeout: 30_000,
|
| 69 |
+
});
|
| 70 |
+
return data;
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
export default apiClient;
|
frontend/src/components/AnalyzeTab.tsx
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
import React, { useEffect, useState } from "react";
|
| 2 |
-
import { Alert, Stack } from "@mui/material";
|
| 3 |
import axios from "axios";
|
| 4 |
import type { AnalyzeRequest, AnalyzeResponse, ModelsResponse } from "../types";
|
| 5 |
import { analyzeArticle, getModels } from "../api/client";
|
|
@@ -16,6 +16,7 @@ function AnalyzeTab(): React.ReactElement {
|
|
| 16 |
const [isLoading, setIsLoading] = useState<boolean>(false);
|
| 17 |
const [error, setError] = useState<string | null>(null);
|
| 18 |
const [availableModels, setAvailableModels] = useState<ModelsResponse | undefined>(undefined);
|
|
|
|
| 19 |
|
| 20 |
useEffect(() => {
|
| 21 |
getModels()
|
|
@@ -28,7 +29,7 @@ function AnalyzeTab(): React.ReactElement {
|
|
| 28 |
setError(null);
|
| 29 |
setArticleText(request.text);
|
| 30 |
try {
|
| 31 |
-
const result = await analyzeArticle(request);
|
| 32 |
setAnalyzeResult(result);
|
| 33 |
} catch (err: unknown) {
|
| 34 |
if (axios.isAxiosError(err)) {
|
|
@@ -44,6 +45,8 @@ function AnalyzeTab(): React.ReactElement {
|
|
| 44 |
}
|
| 45 |
};
|
| 46 |
|
|
|
|
|
|
|
| 47 |
return (
|
| 48 |
<Stack spacing={3}>
|
| 49 |
<ArticleInput
|
|
@@ -53,6 +56,22 @@ function AnalyzeTab(): React.ReactElement {
|
|
| 53 |
availableModels={availableModels}
|
| 54 |
/>
|
| 55 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
{error && (
|
| 57 |
<Alert severity="error" onClose={() => setError(null)}>
|
| 58 |
{error}
|
|
@@ -72,6 +91,44 @@ function AnalyzeTab(): React.ReactElement {
|
|
| 72 |
model={analyzeResult.model}
|
| 73 |
processingTimeMs={analyzeResult.processing_time_ms}
|
| 74 |
/>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
<TextHighlight
|
| 76 |
text={articleText}
|
| 77 |
annotations={analyzeResult.fact_annotations}
|
|
|
|
| 1 |
import React, { useEffect, useState } from "react";
|
| 2 |
+
import { Alert, Box, Checkbox, Chip, FormControlLabel, Stack, Typography } from "@mui/material";
|
| 3 |
import axios from "axios";
|
| 4 |
import type { AnalyzeRequest, AnalyzeResponse, ModelsResponse } from "../types";
|
| 5 |
import { analyzeArticle, getModels } from "../api/client";
|
|
|
|
| 16 |
const [isLoading, setIsLoading] = useState<boolean>(false);
|
| 17 |
const [error, setError] = useState<string | null>(null);
|
| 18 |
const [availableModels, setAvailableModels] = useState<ModelsResponse | undefined>(undefined);
|
| 19 |
+
const [persist, setPersist] = useState<boolean>(false);
|
| 20 |
|
| 21 |
useEffect(() => {
|
| 22 |
getModels()
|
|
|
|
| 29 |
setError(null);
|
| 30 |
setArticleText(request.text);
|
| 31 |
try {
|
| 32 |
+
const result = await analyzeArticle({ ...request, persist });
|
| 33 |
setAnalyzeResult(result);
|
| 34 |
} catch (err: unknown) {
|
| 35 |
if (axios.isAxiosError(err)) {
|
|
|
|
| 45 |
}
|
| 46 |
};
|
| 47 |
|
| 48 |
+
const crossArticleConflicts = analyzeResult?.cross_article_inconsistencies ?? [];
|
| 49 |
+
|
| 50 |
return (
|
| 51 |
<Stack spacing={3}>
|
| 52 |
<ArticleInput
|
|
|
|
| 56 |
availableModels={availableModels}
|
| 57 |
/>
|
| 58 |
|
| 59 |
+
<FormControlLabel
|
| 60 |
+
sx={{ mt: -1 }}
|
| 61 |
+
control={
|
| 62 |
+
<Checkbox
|
| 63 |
+
size="small"
|
| 64 |
+
checked={persist}
|
| 65 |
+
onChange={(e) => setPersist(e.target.checked)}
|
| 66 |
+
/>
|
| 67 |
+
}
|
| 68 |
+
label={
|
| 69 |
+
<Typography variant="body2">
|
| 70 |
+
Save to Neo4j for cross-article analysis
|
| 71 |
+
</Typography>
|
| 72 |
+
}
|
| 73 |
+
/>
|
| 74 |
+
|
| 75 |
{error && (
|
| 76 |
<Alert severity="error" onClose={() => setError(null)}>
|
| 77 |
{error}
|
|
|
|
| 91 |
model={analyzeResult.model}
|
| 92 |
processingTimeMs={analyzeResult.processing_time_ms}
|
| 93 |
/>
|
| 94 |
+
|
| 95 |
+
{/* Cross-article conflicts banner */}
|
| 96 |
+
{crossArticleConflicts.length > 0 && (
|
| 97 |
+
<Alert
|
| 98 |
+
severity="warning"
|
| 99 |
+
action={
|
| 100 |
+
<Chip
|
| 101 |
+
label={`${crossArticleConflicts.length} conflict${crossArticleConflicts.length !== 1 ? "s" : ""}`}
|
| 102 |
+
size="small"
|
| 103 |
+
color="warning"
|
| 104 |
+
/>
|
| 105 |
+
}
|
| 106 |
+
>
|
| 107 |
+
<Box>
|
| 108 |
+
<Typography variant="body2" sx={{ fontWeight: "medium", mb: 0.5 }}>
|
| 109 |
+
Cross-article conflicts detected
|
| 110 |
+
</Typography>
|
| 111 |
+
{crossArticleConflicts.map((c, i) => (
|
| 112 |
+
<Typography key={i} variant="caption" sx={{ display: "block" }}>
|
| 113 |
+
• {c.description}
|
| 114 |
+
</Typography>
|
| 115 |
+
))}
|
| 116 |
+
{analyzeResult.article_id && (
|
| 117 |
+
<Typography variant="caption" color="text.secondary" sx={{ mt: 0.5, display: "block" }}>
|
| 118 |
+
Saved as {analyzeResult.article_id}
|
| 119 |
+
</Typography>
|
| 120 |
+
)}
|
| 121 |
+
</Box>
|
| 122 |
+
</Alert>
|
| 123 |
+
)}
|
| 124 |
+
|
| 125 |
+
{/* Saved without conflicts */}
|
| 126 |
+
{analyzeResult.article_id && crossArticleConflicts.length === 0 && (
|
| 127 |
+
<Alert severity="success" sx={{ py: 0.5 }}>
|
| 128 |
+
Saved to Neo4j — no cross-article conflicts found.
|
| 129 |
+
</Alert>
|
| 130 |
+
)}
|
| 131 |
+
|
| 132 |
<TextHighlight
|
| 133 |
text={articleText}
|
| 134 |
annotations={analyzeResult.fact_annotations}
|
frontend/src/components/ArticleHistory.tsx
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import React, { useCallback, useEffect, useState } from "react";
|
| 2 |
+
import {
|
| 3 |
+
Alert,
|
| 4 |
+
Box,
|
| 5 |
+
CircularProgress,
|
| 6 |
+
Divider,
|
| 7 |
+
Drawer,
|
| 8 |
+
IconButton,
|
| 9 |
+
List,
|
| 10 |
+
ListItem,
|
| 11 |
+
ListItemSecondaryAction,
|
| 12 |
+
ListItemText,
|
| 13 |
+
Tooltip,
|
| 14 |
+
Typography,
|
| 15 |
+
} from "@mui/material";
|
| 16 |
+
import { Close, Delete } from "@mui/icons-material";
|
| 17 |
+
import type { StoredArticle } from "../types";
|
| 18 |
+
import { deleteArticle, getArticles } from "../api/client";
|
| 19 |
+
|
| 20 |
+
interface ArticleHistoryProps {
|
| 21 |
+
open: boolean;
|
| 22 |
+
onClose: () => void;
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
function ArticleHistory({ open, onClose }: ArticleHistoryProps): React.ReactElement {
|
| 26 |
+
const [articles, setArticles] = useState<StoredArticle[]>([]);
|
| 27 |
+
const [neo4jEnabled, setNeo4jEnabled] = useState<boolean>(true);
|
| 28 |
+
const [isLoading, setIsLoading] = useState<boolean>(false);
|
| 29 |
+
const [deletingId, setDeletingId] = useState<string | null>(null);
|
| 30 |
+
const [error, setError] = useState<string | null>(null);
|
| 31 |
+
|
| 32 |
+
const fetchArticles = useCallback(async (): Promise<void> => {
|
| 33 |
+
setIsLoading(true);
|
| 34 |
+
setError(null);
|
| 35 |
+
try {
|
| 36 |
+
const response = await getArticles();
|
| 37 |
+
setNeo4jEnabled(response.neo4j_enabled);
|
| 38 |
+
setArticles(response.articles);
|
| 39 |
+
} catch {
|
| 40 |
+
setError("Failed to load article history.");
|
| 41 |
+
} finally {
|
| 42 |
+
setIsLoading(false);
|
| 43 |
+
}
|
| 44 |
+
}, []);
|
| 45 |
+
|
| 46 |
+
useEffect(() => {
|
| 47 |
+
if (open) {
|
| 48 |
+
void fetchArticles();
|
| 49 |
+
}
|
| 50 |
+
}, [open, fetchArticles]);
|
| 51 |
+
|
| 52 |
+
const handleDelete = async (articleId: string): Promise<void> => {
|
| 53 |
+
setDeletingId(articleId);
|
| 54 |
+
setError(null);
|
| 55 |
+
try {
|
| 56 |
+
await deleteArticle(articleId);
|
| 57 |
+
setArticles((prev) => prev.filter((a) => a.article_id !== articleId));
|
| 58 |
+
} catch {
|
| 59 |
+
setError(`Failed to delete article ${articleId}.`);
|
| 60 |
+
} finally {
|
| 61 |
+
setDeletingId(null);
|
| 62 |
+
}
|
| 63 |
+
};
|
| 64 |
+
|
| 65 |
+
const formatDate = (raw: string | null): string => {
|
| 66 |
+
if (!raw) return "Unknown date";
|
| 67 |
+
try {
|
| 68 |
+
return new Date(raw).toLocaleDateString(undefined, {
|
| 69 |
+
year: "numeric",
|
| 70 |
+
month: "short",
|
| 71 |
+
day: "numeric",
|
| 72 |
+
});
|
| 73 |
+
} catch {
|
| 74 |
+
return raw;
|
| 75 |
+
}
|
| 76 |
+
};
|
| 77 |
+
|
| 78 |
+
return (
|
| 79 |
+
<Drawer
|
| 80 |
+
anchor="right"
|
| 81 |
+
open={open}
|
| 82 |
+
onClose={onClose}
|
| 83 |
+
slotProps={{ paper: { sx: { width: 350, display: "flex", flexDirection: "column" } } }}
|
| 84 |
+
>
|
| 85 |
+
{/* Header */}
|
| 86 |
+
<Box
|
| 87 |
+
sx={{
|
| 88 |
+
px: 2,
|
| 89 |
+
py: 1.5,
|
| 90 |
+
display: "flex",
|
| 91 |
+
alignItems: "center",
|
| 92 |
+
justifyContent: "space-between",
|
| 93 |
+
borderBottom: "1px solid",
|
| 94 |
+
borderColor: "divider",
|
| 95 |
+
flexShrink: 0,
|
| 96 |
+
}}
|
| 97 |
+
>
|
| 98 |
+
<Typography variant="subtitle1" sx={{ fontWeight: "medium" }}>
|
| 99 |
+
Article History
|
| 100 |
+
</Typography>
|
| 101 |
+
<IconButton size="small" onClick={onClose} aria-label="Close history">
|
| 102 |
+
<Close fontSize="small" />
|
| 103 |
+
</IconButton>
|
| 104 |
+
</Box>
|
| 105 |
+
|
| 106 |
+
{/* Body */}
|
| 107 |
+
<Box sx={{ flex: 1, overflowY: "auto", px: 0 }}>
|
| 108 |
+
{!neo4jEnabled && (
|
| 109 |
+
<Box sx={{ p: 2 }}>
|
| 110 |
+
<Alert severity="info">
|
| 111 |
+
Neo4j not configured. Enable in <code>.env</code> to save articles.
|
| 112 |
+
</Alert>
|
| 113 |
+
</Box>
|
| 114 |
+
)}
|
| 115 |
+
|
| 116 |
+
{neo4jEnabled && isLoading && (
|
| 117 |
+
<Box sx={{ display: "flex", justifyContent: "center", pt: 4 }}>
|
| 118 |
+
<CircularProgress size={28} />
|
| 119 |
+
</Box>
|
| 120 |
+
)}
|
| 121 |
+
|
| 122 |
+
{neo4jEnabled && error && (
|
| 123 |
+
<Box sx={{ p: 2 }}>
|
| 124 |
+
<Alert severity="error" onClose={() => setError(null)}>
|
| 125 |
+
{error}
|
| 126 |
+
</Alert>
|
| 127 |
+
</Box>
|
| 128 |
+
)}
|
| 129 |
+
|
| 130 |
+
{neo4jEnabled && !isLoading && articles.length === 0 && !error && (
|
| 131 |
+
<Box sx={{ p: 2, textAlign: "center" }}>
|
| 132 |
+
<Typography variant="body2" color="text.secondary">
|
| 133 |
+
No articles saved yet. Use{" "}
|
| 134 |
+
<strong>Save to Neo4j</strong> when analyzing to store results.
|
| 135 |
+
</Typography>
|
| 136 |
+
</Box>
|
| 137 |
+
)}
|
| 138 |
+
|
| 139 |
+
{neo4jEnabled && !isLoading && articles.length > 0 && (
|
| 140 |
+
<List dense disablePadding>
|
| 141 |
+
{articles.map((article, idx) => (
|
| 142 |
+
<React.Fragment key={article.article_id}>
|
| 143 |
+
<ListItem
|
| 144 |
+
alignItems="flex-start"
|
| 145 |
+
sx={{ pr: 6 }}
|
| 146 |
+
>
|
| 147 |
+
<ListItemText
|
| 148 |
+
primary={
|
| 149 |
+
<Typography variant="body2" noWrap sx={{ fontWeight: "medium" }}>
|
| 150 |
+
{article.title ?? <em>Untitled</em>}
|
| 151 |
+
</Typography>
|
| 152 |
+
}
|
| 153 |
+
secondary={
|
| 154 |
+
<Box component="span" sx={{ display: "flex", flexDirection: "column", gap: 0.25 }}>
|
| 155 |
+
{article.source && (
|
| 156 |
+
<Typography variant="caption" color="text.secondary" noWrap>
|
| 157 |
+
{article.source}
|
| 158 |
+
</Typography>
|
| 159 |
+
)}
|
| 160 |
+
<Typography variant="caption" color="text.secondary">
|
| 161 |
+
{formatDate(article.analyzed_at)} · {article.fact_count} facts
|
| 162 |
+
</Typography>
|
| 163 |
+
</Box>
|
| 164 |
+
}
|
| 165 |
+
/>
|
| 166 |
+
<ListItemSecondaryAction>
|
| 167 |
+
<Tooltip title="Delete article">
|
| 168 |
+
<span>
|
| 169 |
+
<IconButton
|
| 170 |
+
size="small"
|
| 171 |
+
edge="end"
|
| 172 |
+
onClick={() => void handleDelete(article.article_id)}
|
| 173 |
+
disabled={deletingId === article.article_id}
|
| 174 |
+
aria-label="Delete article"
|
| 175 |
+
>
|
| 176 |
+
{deletingId === article.article_id ? (
|
| 177 |
+
<CircularProgress size={16} />
|
| 178 |
+
) : (
|
| 179 |
+
<Delete fontSize="small" />
|
| 180 |
+
)}
|
| 181 |
+
</IconButton>
|
| 182 |
+
</span>
|
| 183 |
+
</Tooltip>
|
| 184 |
+
</ListItemSecondaryAction>
|
| 185 |
+
</ListItem>
|
| 186 |
+
{idx < articles.length - 1 && <Divider component="li" />}
|
| 187 |
+
</React.Fragment>
|
| 188 |
+
))}
|
| 189 |
+
</List>
|
| 190 |
+
)}
|
| 191 |
+
</Box>
|
| 192 |
+
|
| 193 |
+
{/* Footer */}
|
| 194 |
+
{neo4jEnabled && articles.length > 0 && (
|
| 195 |
+
<Box
|
| 196 |
+
sx={{
|
| 197 |
+
px: 2,
|
| 198 |
+
py: 1,
|
| 199 |
+
borderTop: "1px solid",
|
| 200 |
+
borderColor: "divider",
|
| 201 |
+
flexShrink: 0,
|
| 202 |
+
}}
|
| 203 |
+
>
|
| 204 |
+
<Typography variant="caption" color="text.secondary">
|
| 205 |
+
{articles.length} article{articles.length !== 1 ? "s" : ""} stored
|
| 206 |
+
</Typography>
|
| 207 |
+
</Box>
|
| 208 |
+
)}
|
| 209 |
+
</Drawer>
|
| 210 |
+
);
|
| 211 |
+
}
|
| 212 |
+
|
| 213 |
+
export default ArticleHistory;
|
frontend/src/components/ArticleInput.tsx
CHANGED
|
@@ -5,6 +5,7 @@ import {
|
|
| 5 |
AccordionSummary,
|
| 6 |
Box,
|
| 7 |
Button,
|
|
|
|
| 8 |
CircularProgress,
|
| 9 |
FormControl,
|
| 10 |
FormControlLabel,
|
|
@@ -24,6 +25,7 @@ import type { SelectChangeEvent } from "@mui/material";
|
|
| 24 |
import { Clear, ExpandMore, PlayArrow, Upload } from "@mui/icons-material";
|
| 25 |
import type { AnalyzeRequest, ModelsResponse } from "../types";
|
| 26 |
import { getModelDescription, getModelLabel } from "../utils/modelLabels";
|
|
|
|
| 27 |
|
| 28 |
// Fields that ArticleInput owns (text, metadata — no pipeline/model)
|
| 29 |
type ArticleFields = Omit<AnalyzeRequest, "pipeline" | "model">;
|
|
@@ -47,9 +49,11 @@ function ArticleInput({
|
|
| 47 |
hideSubmitButton = false,
|
| 48 |
onRequestChange,
|
| 49 |
}: ArticleInputProps): React.ReactElement {
|
| 50 |
-
const [inputMode, setInputMode] = useState<"text" | "file"
|
| 51 |
const [textValue, setTextValue] = useState<string>("");
|
| 52 |
-
const [
|
|
|
|
|
|
|
| 53 |
const [title, setTitle] = useState<string>("");
|
| 54 |
const [publicationDate, setPublicationDate] = useState<string>("");
|
| 55 |
const [source, setSource] = useState<string>("");
|
|
@@ -83,41 +87,37 @@ function ArticleInput({
|
|
| 83 |
newMode: string | null,
|
| 84 |
): void => {
|
| 85 |
if (newMode !== null) {
|
| 86 |
-
setInputMode(newMode as "text" | "file"
|
| 87 |
}
|
| 88 |
};
|
| 89 |
|
| 90 |
-
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>): void => {
|
| 91 |
const file = e.target.files?.[0];
|
| 92 |
if (!file) return;
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
}
|
| 111 |
-
};
|
| 112 |
-
reader.readAsText(file);
|
| 113 |
};
|
| 114 |
|
| 115 |
const handleClearFile = (): void => {
|
| 116 |
-
|
|
|
|
| 117 |
setTextValue("");
|
| 118 |
-
if (fileInputRef.current)
|
| 119 |
-
fileInputRef.current.value = "";
|
| 120 |
-
}
|
| 121 |
};
|
| 122 |
|
| 123 |
const handleSubmit = (): void => {
|
|
@@ -143,7 +143,6 @@ function ArticleInput({
|
|
| 143 |
>
|
| 144 |
<ToggleButton value="text">Paste Text</ToggleButton>
|
| 145 |
<ToggleButton value="file">Upload File</ToggleButton>
|
| 146 |
-
<ToggleButton value="url">URL</ToggleButton>
|
| 147 |
</ToggleButtonGroup>
|
| 148 |
|
| 149 |
{inputMode === "text" && (
|
|
@@ -157,13 +156,20 @@ function ArticleInput({
|
|
| 157 |
onChange={(e) => setTextValue(e.target.value)}
|
| 158 |
sx={{ "& .MuiInputBase-input": { fontFamily: "IBM Plex Mono, monospace" } }}
|
| 159 |
/>
|
| 160 |
-
<
|
| 161 |
-
variant="caption"
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
</Box>
|
| 168 |
)}
|
| 169 |
|
|
@@ -171,7 +177,7 @@ function ArticleInput({
|
|
| 171 |
<Box sx={{ display: "flex", flexDirection: "column", gap: 1.5 }}>
|
| 172 |
<input
|
| 173 |
type="file"
|
| 174 |
-
accept=".txt,.
|
| 175 |
ref={fileInputRef}
|
| 176 |
onChange={handleFileChange}
|
| 177 |
style={{ display: "none" }}
|
|
@@ -179,44 +185,34 @@ function ArticleInput({
|
|
| 179 |
<Box sx={{ display: "flex", alignItems: "center", gap: 1.5 }}>
|
| 180 |
<Button
|
| 181 |
variant="outlined"
|
| 182 |
-
startIcon={<Upload />}
|
| 183 |
-
onClick={() => fileInputRef.current?.click()}
|
|
|
|
| 184 |
>
|
| 185 |
-
Choose File
|
| 186 |
</Button>
|
| 187 |
-
{
|
| 188 |
-
<
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
</
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
onClick={handleClearFile}
|
| 197 |
-
>
|
| 198 |
-
Clear
|
| 199 |
-
</Button>
|
| 200 |
-
</>
|
| 201 |
)}
|
| 202 |
</Box>
|
| 203 |
-
{
|
| 204 |
-
<Typography variant="caption" color="
|
| 205 |
-
{
|
| 206 |
</Typography>
|
| 207 |
)}
|
|
|
|
|
|
|
|
|
|
| 208 |
</Box>
|
| 209 |
)}
|
| 210 |
|
| 211 |
-
{inputMode === "url" && (
|
| 212 |
-
<TextField
|
| 213 |
-
fullWidth
|
| 214 |
-
disabled
|
| 215 |
-
placeholder="https://..."
|
| 216 |
-
helperText="Coming soon — requires RSS/API integration"
|
| 217 |
-
/>
|
| 218 |
-
)}
|
| 219 |
-
|
| 220 |
<Accordion
|
| 221 |
disableGutters
|
| 222 |
elevation={0}
|
|
|
|
| 5 |
AccordionSummary,
|
| 6 |
Box,
|
| 7 |
Button,
|
| 8 |
+
Chip,
|
| 9 |
CircularProgress,
|
| 10 |
FormControl,
|
| 11 |
FormControlLabel,
|
|
|
|
| 25 |
import { Clear, ExpandMore, PlayArrow, Upload } from "@mui/icons-material";
|
| 26 |
import type { AnalyzeRequest, ModelsResponse } from "../types";
|
| 27 |
import { getModelDescription, getModelLabel } from "../utils/modelLabels";
|
| 28 |
+
import { uploadFile } from "../api/client";
|
| 29 |
|
| 30 |
// Fields that ArticleInput owns (text, metadata — no pipeline/model)
|
| 31 |
type ArticleFields = Omit<AnalyzeRequest, "pipeline" | "model">;
|
|
|
|
| 49 |
hideSubmitButton = false,
|
| 50 |
onRequestChange,
|
| 51 |
}: ArticleInputProps): React.ReactElement {
|
| 52 |
+
const [inputMode, setInputMode] = useState<"text" | "file">("text");
|
| 53 |
const [textValue, setTextValue] = useState<string>("");
|
| 54 |
+
const [isUploading, setIsUploading] = useState<boolean>(false);
|
| 55 |
+
const [uploadError, setUploadError] = useState<string | null>(null);
|
| 56 |
+
const [extractedInfo, setExtractedInfo] = useState<{ filename: string; char_count: number } | null>(null);
|
| 57 |
const [title, setTitle] = useState<string>("");
|
| 58 |
const [publicationDate, setPublicationDate] = useState<string>("");
|
| 59 |
const [source, setSource] = useState<string>("");
|
|
|
|
| 87 |
newMode: string | null,
|
| 88 |
): void => {
|
| 89 |
if (newMode !== null) {
|
| 90 |
+
setInputMode(newMode as "text" | "file");
|
| 91 |
}
|
| 92 |
};
|
| 93 |
|
| 94 |
+
const handleFileChange = async (e: React.ChangeEvent<HTMLInputElement>): Promise<void> => {
|
| 95 |
const file = e.target.files?.[0];
|
| 96 |
if (!file) return;
|
| 97 |
+
setUploadError(null);
|
| 98 |
+
setExtractedInfo(null);
|
| 99 |
+
setIsUploading(true);
|
| 100 |
+
try {
|
| 101 |
+
const result = await uploadFile(file);
|
| 102 |
+
setTextValue(result.text);
|
| 103 |
+
setExtractedInfo({ filename: result.filename, char_count: result.char_count });
|
| 104 |
+
setInputMode("text");
|
| 105 |
+
} catch (err: unknown) {
|
| 106 |
+
const msg =
|
| 107 |
+
(err as { response?: { data?: { detail?: string } } })?.response?.data?.detail ??
|
| 108 |
+
"Upload failed";
|
| 109 |
+
setUploadError(msg);
|
| 110 |
+
} finally {
|
| 111 |
+
setIsUploading(false);
|
| 112 |
+
if (fileInputRef.current) fileInputRef.current.value = "";
|
| 113 |
+
}
|
|
|
|
|
|
|
|
|
|
| 114 |
};
|
| 115 |
|
| 116 |
const handleClearFile = (): void => {
|
| 117 |
+
setExtractedInfo(null);
|
| 118 |
+
setUploadError(null);
|
| 119 |
setTextValue("");
|
| 120 |
+
if (fileInputRef.current) fileInputRef.current.value = "";
|
|
|
|
|
|
|
| 121 |
};
|
| 122 |
|
| 123 |
const handleSubmit = (): void => {
|
|
|
|
| 143 |
>
|
| 144 |
<ToggleButton value="text">Paste Text</ToggleButton>
|
| 145 |
<ToggleButton value="file">Upload File</ToggleButton>
|
|
|
|
| 146 |
</ToggleButtonGroup>
|
| 147 |
|
| 148 |
{inputMode === "text" && (
|
|
|
|
| 156 |
onChange={(e) => setTextValue(e.target.value)}
|
| 157 |
sx={{ "& .MuiInputBase-input": { fontFamily: "IBM Plex Mono, monospace" } }}
|
| 158 |
/>
|
| 159 |
+
<Box sx={{ mt: 0.5, display: "flex", alignItems: "center", gap: 1, flexWrap: "wrap" }}>
|
| 160 |
+
<Typography variant="caption" color="text.secondary">
|
| 161 |
+
{textValue.length} characters
|
| 162 |
+
</Typography>
|
| 163 |
+
{extractedInfo && (
|
| 164 |
+
<Chip
|
| 165 |
+
size="small"
|
| 166 |
+
label={`Extracted from: ${extractedInfo.filename} (${extractedInfo.char_count} chars)`}
|
| 167 |
+
onDelete={handleClearFile}
|
| 168 |
+
color="info"
|
| 169 |
+
variant="outlined"
|
| 170 |
+
/>
|
| 171 |
+
)}
|
| 172 |
+
</Box>
|
| 173 |
</Box>
|
| 174 |
)}
|
| 175 |
|
|
|
|
| 177 |
<Box sx={{ display: "flex", flexDirection: "column", gap: 1.5 }}>
|
| 178 |
<input
|
| 179 |
type="file"
|
| 180 |
+
accept=".txt,.pdf,.docx,.doc"
|
| 181 |
ref={fileInputRef}
|
| 182 |
onChange={handleFileChange}
|
| 183 |
style={{ display: "none" }}
|
|
|
|
| 185 |
<Box sx={{ display: "flex", alignItems: "center", gap: 1.5 }}>
|
| 186 |
<Button
|
| 187 |
variant="outlined"
|
| 188 |
+
startIcon={isUploading ? <CircularProgress size={18} color="inherit" /> : <Upload />}
|
| 189 |
+
onClick={() => !isUploading && fileInputRef.current?.click()}
|
| 190 |
+
disabled={isUploading}
|
| 191 |
>
|
| 192 |
+
{isUploading ? "Extracting..." : "Choose File"}
|
| 193 |
</Button>
|
| 194 |
+
{!isUploading && extractedInfo && (
|
| 195 |
+
<Button
|
| 196 |
+
size="small"
|
| 197 |
+
color="inherit"
|
| 198 |
+
startIcon={<Clear />}
|
| 199 |
+
onClick={handleClearFile}
|
| 200 |
+
>
|
| 201 |
+
Clear
|
| 202 |
+
</Button>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
)}
|
| 204 |
</Box>
|
| 205 |
+
{uploadError && (
|
| 206 |
+
<Typography variant="caption" color="error">
|
| 207 |
+
{uploadError}
|
| 208 |
</Typography>
|
| 209 |
)}
|
| 210 |
+
<Typography variant="caption" color="text.secondary">
|
| 211 |
+
Supported formats: .txt, .pdf, .docx
|
| 212 |
+
</Typography>
|
| 213 |
</Box>
|
| 214 |
)}
|
| 215 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
<Accordion
|
| 217 |
disableGutters
|
| 218 |
elevation={0}
|
frontend/src/components/BatchTab.tsx
ADDED
|
@@ -0,0 +1,506 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import React, { useRef, useState } from "react";
|
| 2 |
+
import {
|
| 3 |
+
Alert,
|
| 4 |
+
Box,
|
| 5 |
+
Button,
|
| 6 |
+
Checkbox,
|
| 7 |
+
Chip,
|
| 8 |
+
CircularProgress,
|
| 9 |
+
Collapse,
|
| 10 |
+
FormControl,
|
| 11 |
+
FormControlLabel,
|
| 12 |
+
IconButton,
|
| 13 |
+
InputLabel,
|
| 14 |
+
LinearProgress,
|
| 15 |
+
MenuItem,
|
| 16 |
+
Paper,
|
| 17 |
+
Select,
|
| 18 |
+
Stack,
|
| 19 |
+
Table,
|
| 20 |
+
TableBody,
|
| 21 |
+
TableCell,
|
| 22 |
+
TableHead,
|
| 23 |
+
TableRow,
|
| 24 |
+
TableSortLabel,
|
| 25 |
+
TextField,
|
| 26 |
+
Typography,
|
| 27 |
+
} from "@mui/material";
|
| 28 |
+
import type { SelectChangeEvent } from "@mui/material";
|
| 29 |
+
import { Add, Delete, PlayArrow } from "@mui/icons-material";
|
| 30 |
+
import axios from "axios";
|
| 31 |
+
import type { BatchArticleResult, BatchResponse, ModelsResponse } from "../types";
|
| 32 |
+
import { analyzeBatch } from "../api/client";
|
| 33 |
+
import { getModelLabel } from "../utils/modelLabels";
|
| 34 |
+
|
| 35 |
+
interface BatchTabProps {
|
| 36 |
+
availableModels?: ModelsResponse;
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
interface BatchRow {
|
| 40 |
+
id: number;
|
| 41 |
+
text: string;
|
| 42 |
+
title: string;
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
type SortField = "score" | "n_claims" | "n_inconsistencies" | "n_cross_article_inconsistencies" | "processing_time_ms";
|
| 46 |
+
|
| 47 |
+
function scoreColor(score: number): "success" | "warning" | "default" | "error" {
|
| 48 |
+
if (score >= 0.8) return "success";
|
| 49 |
+
if (score >= 0.5) return "warning";
|
| 50 |
+
if (score >= 0.2) return "default";
|
| 51 |
+
return "error";
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
const MAX_ROWS = 20;
|
| 55 |
+
let _rowIdCounter = 0;
|
| 56 |
+
function nextId(): number {
|
| 57 |
+
return ++_rowIdCounter;
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
function BatchTab({ availableModels }: BatchTabProps): React.ReactElement {
|
| 61 |
+
const [rows, setRows] = useState<BatchRow[]>([
|
| 62 |
+
{ id: nextId(), text: "", title: "" },
|
| 63 |
+
{ id: nextId(), text: "", title: "" },
|
| 64 |
+
]);
|
| 65 |
+
const [pipeline, setPipeline] = useState<"spacy" | "llm">("spacy");
|
| 66 |
+
const [model, setModel] = useState<string>("");
|
| 67 |
+
const [persist, setPersist] = useState<boolean>(true);
|
| 68 |
+
const [isLoading, setIsLoading] = useState<boolean>(false);
|
| 69 |
+
const [result, setResult] = useState<BatchResponse | null>(null);
|
| 70 |
+
const [error, setError] = useState<string | null>(null);
|
| 71 |
+
const [sortField, setSortField] = useState<SortField>("score");
|
| 72 |
+
const [sortDir, setSortDir] = useState<"asc" | "desc">("asc");
|
| 73 |
+
const [expandedId, setExpandedId] = useState<string | null>(null);
|
| 74 |
+
|
| 75 |
+
const pipelineMeta = availableModels
|
| 76 |
+
? pipeline === "llm"
|
| 77 |
+
? availableModels.llm
|
| 78 |
+
: availableModels.spacy
|
| 79 |
+
: null;
|
| 80 |
+
|
| 81 |
+
const addRow = (): void => {
|
| 82 |
+
if (rows.length >= MAX_ROWS) return;
|
| 83 |
+
setRows((prev) => [...prev, { id: nextId(), text: "", title: "" }]);
|
| 84 |
+
};
|
| 85 |
+
|
| 86 |
+
const removeRow = (id: number): void => {
|
| 87 |
+
if (rows.length <= 1) return;
|
| 88 |
+
setRows((prev) => prev.filter((r) => r.id !== id));
|
| 89 |
+
};
|
| 90 |
+
|
| 91 |
+
const updateRow = (id: number, field: "text" | "title", value: string): void => {
|
| 92 |
+
setRows((prev) =>
|
| 93 |
+
prev.map((r) => (r.id === id ? { ...r, [field]: value } : r))
|
| 94 |
+
);
|
| 95 |
+
};
|
| 96 |
+
|
| 97 |
+
const handlePipelineChange = (e: SelectChangeEvent<string>): void => {
|
| 98 |
+
const p = e.target.value as "spacy" | "llm";
|
| 99 |
+
setPipeline(p);
|
| 100 |
+
if (availableModels) {
|
| 101 |
+
setModel(p === "llm" ? availableModels.llm.default : availableModels.spacy.default);
|
| 102 |
+
}
|
| 103 |
+
};
|
| 104 |
+
|
| 105 |
+
const handleSort = (field: SortField): void => {
|
| 106 |
+
if (sortField === field) {
|
| 107 |
+
setSortDir((d) => (d === "asc" ? "desc" : "asc"));
|
| 108 |
+
} else {
|
| 109 |
+
setSortField(field);
|
| 110 |
+
setSortDir("asc");
|
| 111 |
+
}
|
| 112 |
+
};
|
| 113 |
+
|
| 114 |
+
const validRows = rows.filter((r) => r.text.trim().length >= 20);
|
| 115 |
+
|
| 116 |
+
const handleSubmit = async (): Promise<void> => {
|
| 117 |
+
if (validRows.length === 0) return;
|
| 118 |
+
setIsLoading(true);
|
| 119 |
+
setError(null);
|
| 120 |
+
setResult(null);
|
| 121 |
+
setExpandedId(null);
|
| 122 |
+
|
| 123 |
+
try {
|
| 124 |
+
const response = await analyzeBatch({
|
| 125 |
+
articles: validRows.map((r) => ({
|
| 126 |
+
text: r.text.trim(),
|
| 127 |
+
title: r.title.trim() || undefined,
|
| 128 |
+
})),
|
| 129 |
+
pipeline,
|
| 130 |
+
model: model || undefined,
|
| 131 |
+
persist,
|
| 132 |
+
});
|
| 133 |
+
setResult(response);
|
| 134 |
+
} catch (err: unknown) {
|
| 135 |
+
if (axios.isAxiosError(err)) {
|
| 136 |
+
const detail = err.response?.data?.detail as string | undefined;
|
| 137 |
+
setError(detail ?? err.message);
|
| 138 |
+
} else if (err instanceof Error) {
|
| 139 |
+
setError(err.message);
|
| 140 |
+
} else {
|
| 141 |
+
setError("An unexpected error occurred");
|
| 142 |
+
}
|
| 143 |
+
} finally {
|
| 144 |
+
setIsLoading(false);
|
| 145 |
+
}
|
| 146 |
+
};
|
| 147 |
+
|
| 148 |
+
const sortedResults: BatchArticleResult[] = result
|
| 149 |
+
? [...result.results].sort((a, b) => {
|
| 150 |
+
const aVal = a[sortField] as number;
|
| 151 |
+
const bVal = b[sortField] as number;
|
| 152 |
+
return sortDir === "asc" ? aVal - bVal : bVal - aVal;
|
| 153 |
+
})
|
| 154 |
+
: [];
|
| 155 |
+
|
| 156 |
+
return (
|
| 157 |
+
<Stack spacing={3}>
|
| 158 |
+
{/* Input section */}
|
| 159 |
+
<Paper variant="outlined" sx={{ p: 2.5 }}>
|
| 160 |
+
<Typography variant="subtitle2" sx={{ mb: 2 }}>
|
| 161 |
+
Articles
|
| 162 |
+
</Typography>
|
| 163 |
+
|
| 164 |
+
<Stack spacing={2}>
|
| 165 |
+
{rows.map((row, idx) => (
|
| 166 |
+
<Box key={row.id} sx={{ display: "flex", gap: 1.5, alignItems: "flex-start" }}>
|
| 167 |
+
<Typography
|
| 168 |
+
variant="caption"
|
| 169 |
+
color="text.secondary"
|
| 170 |
+
sx={{ pt: 1.5, minWidth: 20, textAlign: "right" }}
|
| 171 |
+
>
|
| 172 |
+
{idx + 1}
|
| 173 |
+
</Typography>
|
| 174 |
+
<Box sx={{ flex: 1, display: "flex", flexDirection: "column", gap: 1 }}>
|
| 175 |
+
<TextField
|
| 176 |
+
size="small"
|
| 177 |
+
placeholder="Article title (optional)"
|
| 178 |
+
value={row.title}
|
| 179 |
+
onChange={(e) => updateRow(row.id, "title", e.target.value)}
|
| 180 |
+
fullWidth
|
| 181 |
+
/>
|
| 182 |
+
<TextField
|
| 183 |
+
multiline
|
| 184 |
+
minRows={4}
|
| 185 |
+
fullWidth
|
| 186 |
+
placeholder="Paste article text here... (min 20 characters)"
|
| 187 |
+
value={row.text}
|
| 188 |
+
onChange={(e) => updateRow(row.id, "text", e.target.value)}
|
| 189 |
+
error={row.text.length > 0 && row.text.trim().length < 20}
|
| 190 |
+
helperText={row.text.length > 0 ? `${row.text.length} characters` : undefined}
|
| 191 |
+
sx={{ "& .MuiInputBase-input": { fontFamily: "IBM Plex Mono, monospace", fontSize: 13 } }}
|
| 192 |
+
/>
|
| 193 |
+
</Box>
|
| 194 |
+
<IconButton
|
| 195 |
+
size="small"
|
| 196 |
+
onClick={() => removeRow(row.id)}
|
| 197 |
+
disabled={rows.length <= 1}
|
| 198 |
+
sx={{ mt: 0.5 }}
|
| 199 |
+
>
|
| 200 |
+
<Delete fontSize="small" />
|
| 201 |
+
</IconButton>
|
| 202 |
+
</Box>
|
| 203 |
+
))}
|
| 204 |
+
|
| 205 |
+
<Box>
|
| 206 |
+
<Button
|
| 207 |
+
size="small"
|
| 208 |
+
startIcon={<Add />}
|
| 209 |
+
onClick={addRow}
|
| 210 |
+
disabled={rows.length >= MAX_ROWS}
|
| 211 |
+
>
|
| 212 |
+
Add Article {rows.length >= MAX_ROWS ? `(max ${MAX_ROWS})` : ""}
|
| 213 |
+
</Button>
|
| 214 |
+
</Box>
|
| 215 |
+
</Stack>
|
| 216 |
+
|
| 217 |
+
<Box
|
| 218 |
+
sx={{
|
| 219 |
+
mt: 2.5,
|
| 220 |
+
pt: 2,
|
| 221 |
+
borderTop: "1px solid",
|
| 222 |
+
borderColor: "divider",
|
| 223 |
+
display: "flex",
|
| 224 |
+
gap: 2,
|
| 225 |
+
flexWrap: "wrap",
|
| 226 |
+
alignItems: "center",
|
| 227 |
+
}}
|
| 228 |
+
>
|
| 229 |
+
<FormControl size="small" sx={{ minWidth: 130 }}>
|
| 230 |
+
<InputLabel>Pipeline</InputLabel>
|
| 231 |
+
<Select value={pipeline} label="Pipeline" onChange={handlePipelineChange}>
|
| 232 |
+
<MenuItem value="spacy">spaCy</MenuItem>
|
| 233 |
+
<MenuItem value="llm">LLM</MenuItem>
|
| 234 |
+
</Select>
|
| 235 |
+
</FormControl>
|
| 236 |
+
|
| 237 |
+
{pipelineMeta && (
|
| 238 |
+
<FormControl size="small" sx={{ minWidth: 180 }}>
|
| 239 |
+
<InputLabel>Model</InputLabel>
|
| 240 |
+
<Select
|
| 241 |
+
value={model || pipelineMeta.default}
|
| 242 |
+
label="Model"
|
| 243 |
+
onChange={(e: SelectChangeEvent<string>) => setModel(e.target.value)}
|
| 244 |
+
>
|
| 245 |
+
{pipelineMeta.models.map((m) => (
|
| 246 |
+
<MenuItem key={m} value={m}>
|
| 247 |
+
{getModelLabel(m)}
|
| 248 |
+
</MenuItem>
|
| 249 |
+
))}
|
| 250 |
+
</Select>
|
| 251 |
+
</FormControl>
|
| 252 |
+
)}
|
| 253 |
+
|
| 254 |
+
<FormControlLabel
|
| 255 |
+
control={
|
| 256 |
+
<Checkbox
|
| 257 |
+
size="small"
|
| 258 |
+
checked={persist}
|
| 259 |
+
onChange={(e) => setPersist(e.target.checked)}
|
| 260 |
+
/>
|
| 261 |
+
}
|
| 262 |
+
label={
|
| 263 |
+
<Typography variant="body2">Save to Neo4j for cross-article analysis</Typography>
|
| 264 |
+
}
|
| 265 |
+
/>
|
| 266 |
+
|
| 267 |
+
<Box sx={{ ml: "auto" }}>
|
| 268 |
+
<Button
|
| 269 |
+
variant="contained"
|
| 270 |
+
startIcon={
|
| 271 |
+
isLoading ? (
|
| 272 |
+
<CircularProgress size={18} color="inherit" />
|
| 273 |
+
) : (
|
| 274 |
+
<PlayArrow />
|
| 275 |
+
)
|
| 276 |
+
}
|
| 277 |
+
disabled={validRows.length === 0 || isLoading}
|
| 278 |
+
onClick={handleSubmit}
|
| 279 |
+
>
|
| 280 |
+
{isLoading ? `Analyzing ${validRows.length} articles…` : "Analyze Batch"}
|
| 281 |
+
</Button>
|
| 282 |
+
</Box>
|
| 283 |
+
</Box>
|
| 284 |
+
</Paper>
|
| 285 |
+
|
| 286 |
+
{/* Loading indicator */}
|
| 287 |
+
{isLoading && (
|
| 288 |
+
<Box>
|
| 289 |
+
<LinearProgress />
|
| 290 |
+
<Typography variant="caption" color="text.secondary" sx={{ mt: 0.5, display: "block" }} component="p">
|
| 291 |
+
Analyzing {validRows.length} article{validRows.length !== 1 ? "s" : ""}…
|
| 292 |
+
</Typography>
|
| 293 |
+
</Box>
|
| 294 |
+
)}
|
| 295 |
+
|
| 296 |
+
{error && (
|
| 297 |
+
<Alert severity="error" onClose={() => setError(null)}>
|
| 298 |
+
{error}
|
| 299 |
+
</Alert>
|
| 300 |
+
)}
|
| 301 |
+
|
| 302 |
+
{/* Results section */}
|
| 303 |
+
{result && (
|
| 304 |
+
<Paper variant="outlined" sx={{ p: 2.5 }}>
|
| 305 |
+
{/* Aggregate stats */}
|
| 306 |
+
<Box sx={{ display: "flex", gap: 3, flexWrap: "wrap", mb: 2.5 }}>
|
| 307 |
+
<StatBox label="Articles" value={result.total_articles} />
|
| 308 |
+
<StatBox
|
| 309 |
+
label="Mean TCS"
|
| 310 |
+
value={result.avg_score.toFixed(3)}
|
| 311 |
+
color={
|
| 312 |
+
result.avg_score >= 0.8
|
| 313 |
+
? "success.main"
|
| 314 |
+
: result.avg_score >= 0.5
|
| 315 |
+
? "warning.main"
|
| 316 |
+
: "error.main"
|
| 317 |
+
}
|
| 318 |
+
/>
|
| 319 |
+
<StatBox
|
| 320 |
+
label="Total claims"
|
| 321 |
+
value={result.results.reduce((s, r) => s + r.n_claims, 0)}
|
| 322 |
+
/>
|
| 323 |
+
<StatBox
|
| 324 |
+
label="Inconsistencies"
|
| 325 |
+
value={result.results.reduce((s, r) => s + r.n_inconsistencies, 0)}
|
| 326 |
+
/>
|
| 327 |
+
<StatBox label="Cross-article conflicts" value={result.total_cross_article_conflicts} />
|
| 328 |
+
{result.persisted && (
|
| 329 |
+
<Chip label="Saved to Neo4j" size="small" color="success" variant="outlined" />
|
| 330 |
+
)}
|
| 331 |
+
</Box>
|
| 332 |
+
|
| 333 |
+
{/* Results table */}
|
| 334 |
+
<Table size="small">
|
| 335 |
+
<TableHead>
|
| 336 |
+
<TableRow>
|
| 337 |
+
<TableCell sx={{ width: 36 }}>#</TableCell>
|
| 338 |
+
<TableCell>Title</TableCell>
|
| 339 |
+
<SortHeaderCell field="score" active={sortField} dir={sortDir} onSort={handleSort}>
|
| 340 |
+
TCS Score
|
| 341 |
+
</SortHeaderCell>
|
| 342 |
+
<SortHeaderCell field="n_claims" active={sortField} dir={sortDir} onSort={handleSort}>
|
| 343 |
+
Claims
|
| 344 |
+
</SortHeaderCell>
|
| 345 |
+
<SortHeaderCell field="n_inconsistencies" active={sortField} dir={sortDir} onSort={handleSort}>
|
| 346 |
+
Inconsistencies
|
| 347 |
+
</SortHeaderCell>
|
| 348 |
+
<SortHeaderCell
|
| 349 |
+
field="n_cross_article_inconsistencies"
|
| 350 |
+
active={sortField}
|
| 351 |
+
dir={sortDir}
|
| 352 |
+
onSort={handleSort}
|
| 353 |
+
>
|
| 354 |
+
Cross-Article
|
| 355 |
+
</SortHeaderCell>
|
| 356 |
+
<SortHeaderCell
|
| 357 |
+
field="processing_time_ms"
|
| 358 |
+
active={sortField}
|
| 359 |
+
dir={sortDir}
|
| 360 |
+
onSort={handleSort}
|
| 361 |
+
>
|
| 362 |
+
Time
|
| 363 |
+
</SortHeaderCell>
|
| 364 |
+
</TableRow>
|
| 365 |
+
</TableHead>
|
| 366 |
+
<TableBody>
|
| 367 |
+
{sortedResults.map((row, idx) => (
|
| 368 |
+
<React.Fragment key={row.article_id}>
|
| 369 |
+
<TableRow
|
| 370 |
+
hover
|
| 371 |
+
sx={{ cursor: "pointer" }}
|
| 372 |
+
onClick={() =>
|
| 373 |
+
setExpandedId((prev) =>
|
| 374 |
+
prev === row.article_id ? null : row.article_id
|
| 375 |
+
)
|
| 376 |
+
}
|
| 377 |
+
>
|
| 378 |
+
<TableCell sx={{ color: "text.secondary" }}>{idx + 1}</TableCell>
|
| 379 |
+
<TableCell sx={{ maxWidth: 220 }}>
|
| 380 |
+
<Typography variant="body2" noWrap title={row.title || row.article_id}>
|
| 381 |
+
{row.title || <em style={{ opacity: 0.6 }}>Untitled</em>}
|
| 382 |
+
</Typography>
|
| 383 |
+
{row.error && (
|
| 384 |
+
<Typography variant="caption" color="error">
|
| 385 |
+
Error
|
| 386 |
+
</Typography>
|
| 387 |
+
)}
|
| 388 |
+
</TableCell>
|
| 389 |
+
<TableCell>
|
| 390 |
+
<Chip
|
| 391 |
+
label={row.score.toFixed(3)}
|
| 392 |
+
size="small"
|
| 393 |
+
color={scoreColor(row.score)}
|
| 394 |
+
/>
|
| 395 |
+
</TableCell>
|
| 396 |
+
<TableCell>{row.n_claims}</TableCell>
|
| 397 |
+
<TableCell>{row.n_inconsistencies}</TableCell>
|
| 398 |
+
<TableCell>
|
| 399 |
+
{row.n_cross_article_inconsistencies > 0 ? (
|
| 400 |
+
<Chip
|
| 401 |
+
label={row.n_cross_article_inconsistencies}
|
| 402 |
+
size="small"
|
| 403 |
+
color="warning"
|
| 404 |
+
/>
|
| 405 |
+
) : (
|
| 406 |
+
<Typography variant="body2" color="text.secondary">
|
| 407 |
+
0
|
| 408 |
+
</Typography>
|
| 409 |
+
)}
|
| 410 |
+
</TableCell>
|
| 411 |
+
<TableCell>
|
| 412 |
+
<Typography variant="body2" color="text.secondary">
|
| 413 |
+
{Math.round(row.processing_time_ms)}ms
|
| 414 |
+
</Typography>
|
| 415 |
+
</TableCell>
|
| 416 |
+
</TableRow>
|
| 417 |
+
|
| 418 |
+
{/* Expandable detail row */}
|
| 419 |
+
<TableRow>
|
| 420 |
+
<TableCell colSpan={7} sx={{ p: 0, border: 0 }}>
|
| 421 |
+
<Collapse in={expandedId === row.article_id} unmountOnExit>
|
| 422 |
+
<Box sx={{ px: 3, py: 1.5, bgcolor: "action.hover" }}>
|
| 423 |
+
<Typography variant="body2" sx={{ mb: 1 }}>
|
| 424 |
+
{row.summary}
|
| 425 |
+
</Typography>
|
| 426 |
+
{row.cross_article_conflicts.length > 0 && (
|
| 427 |
+
<Stack spacing={0.5}>
|
| 428 |
+
<Typography variant="caption" color="text.secondary">
|
| 429 |
+
Cross-article conflicts:
|
| 430 |
+
</Typography>
|
| 431 |
+
{row.cross_article_conflicts.map((c, i) => (
|
| 432 |
+
<Typography key={i} variant="caption" color="warning.main">
|
| 433 |
+
• {c.description}
|
| 434 |
+
</Typography>
|
| 435 |
+
))}
|
| 436 |
+
</Stack>
|
| 437 |
+
)}
|
| 438 |
+
{row.error && (
|
| 439 |
+
<Typography variant="caption" color="error">
|
| 440 |
+
{row.error}
|
| 441 |
+
</Typography>
|
| 442 |
+
)}
|
| 443 |
+
</Box>
|
| 444 |
+
</Collapse>
|
| 445 |
+
</TableCell>
|
| 446 |
+
</TableRow>
|
| 447 |
+
</React.Fragment>
|
| 448 |
+
))}
|
| 449 |
+
</TableBody>
|
| 450 |
+
</Table>
|
| 451 |
+
</Paper>
|
| 452 |
+
)}
|
| 453 |
+
</Stack>
|
| 454 |
+
);
|
| 455 |
+
}
|
| 456 |
+
|
| 457 |
+
// Stat summary box used in aggregate row
|
| 458 |
+
function StatBox({
|
| 459 |
+
label,
|
| 460 |
+
value,
|
| 461 |
+
color,
|
| 462 |
+
}: {
|
| 463 |
+
label: string;
|
| 464 |
+
value: string | number;
|
| 465 |
+
color?: string;
|
| 466 |
+
}): React.ReactElement {
|
| 467 |
+
return (
|
| 468 |
+
<Box>
|
| 469 |
+
<Typography variant="caption" color="text.secondary" sx={{ display: "block" }}>
|
| 470 |
+
{label}
|
| 471 |
+
</Typography>
|
| 472 |
+
<Typography variant="h6" sx={{ color: color ?? "text.primary", lineHeight: 1.2 }}>
|
| 473 |
+
{value}
|
| 474 |
+
</Typography>
|
| 475 |
+
</Box>
|
| 476 |
+
);
|
| 477 |
+
}
|
| 478 |
+
|
| 479 |
+
// Sortable table header cell
|
| 480 |
+
function SortHeaderCell({
|
| 481 |
+
field,
|
| 482 |
+
active,
|
| 483 |
+
dir,
|
| 484 |
+
onSort,
|
| 485 |
+
children,
|
| 486 |
+
}: {
|
| 487 |
+
field: SortField;
|
| 488 |
+
active: SortField;
|
| 489 |
+
dir: "asc" | "desc";
|
| 490 |
+
onSort: (f: SortField) => void;
|
| 491 |
+
children: React.ReactNode;
|
| 492 |
+
}): React.ReactElement {
|
| 493 |
+
return (
|
| 494 |
+
<TableCell>
|
| 495 |
+
<TableSortLabel
|
| 496 |
+
active={active === field}
|
| 497 |
+
direction={active === field ? dir : "asc"}
|
| 498 |
+
onClick={() => onSort(field)}
|
| 499 |
+
>
|
| 500 |
+
{children}
|
| 501 |
+
</TableSortLabel>
|
| 502 |
+
</TableCell>
|
| 503 |
+
);
|
| 504 |
+
}
|
| 505 |
+
|
| 506 |
+
export default BatchTab;
|
frontend/src/types/api.ts
CHANGED
|
@@ -14,6 +14,7 @@ export interface AnalyzeRequest {
|
|
| 14 |
source?: string;
|
| 15 |
pipeline?: "spacy" | "llm";
|
| 16 |
model?: string;
|
|
|
|
| 17 |
}
|
| 18 |
|
| 19 |
export interface InconsistencyResponse {
|
|
@@ -65,6 +66,60 @@ export interface AnalyzeResponse {
|
|
| 65 |
pipeline: string;
|
| 66 |
model: string;
|
| 67 |
processing_time_ms: number;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
}
|
| 69 |
|
| 70 |
// compare schemas
|
|
|
|
| 14 |
source?: string;
|
| 15 |
pipeline?: "spacy" | "llm";
|
| 16 |
model?: string;
|
| 17 |
+
persist?: boolean;
|
| 18 |
}
|
| 19 |
|
| 20 |
export interface InconsistencyResponse {
|
|
|
|
| 66 |
pipeline: string;
|
| 67 |
model: string;
|
| 68 |
processing_time_ms: number;
|
| 69 |
+
article_id?: string | null;
|
| 70 |
+
cross_article_inconsistencies?: InconsistencyResponse[];
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
// batch schemas
|
| 74 |
+
export interface BatchArticle {
|
| 75 |
+
text: string;
|
| 76 |
+
title?: string;
|
| 77 |
+
publication_date?: string;
|
| 78 |
+
source?: string;
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
export interface BatchRequest {
|
| 82 |
+
articles: BatchArticle[];
|
| 83 |
+
pipeline: string;
|
| 84 |
+
model?: string;
|
| 85 |
+
persist: boolean;
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
export interface BatchArticleResult {
|
| 89 |
+
article_id: string;
|
| 90 |
+
title: string;
|
| 91 |
+
score: number;
|
| 92 |
+
label: string;
|
| 93 |
+
summary: string;
|
| 94 |
+
n_claims: number;
|
| 95 |
+
n_inconsistencies: number;
|
| 96 |
+
n_cross_article_inconsistencies: number;
|
| 97 |
+
cross_article_conflicts: InconsistencyResponse[];
|
| 98 |
+
processing_time_ms: number;
|
| 99 |
+
error?: string | null;
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
export interface BatchResponse {
|
| 103 |
+
results: BatchArticleResult[];
|
| 104 |
+
total_articles: number;
|
| 105 |
+
total_cross_article_conflicts: number;
|
| 106 |
+
avg_score: number;
|
| 107 |
+
neo4j_enabled: boolean;
|
| 108 |
+
persisted: boolean;
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
// article history schemas
|
| 112 |
+
export interface StoredArticle {
|
| 113 |
+
article_id: string;
|
| 114 |
+
title: string | null;
|
| 115 |
+
source: string | null;
|
| 116 |
+
analyzed_at: string | null;
|
| 117 |
+
fact_count: number;
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
export interface ArticlesResponse {
|
| 121 |
+
articles: StoredArticle[];
|
| 122 |
+
neo4j_enabled: boolean;
|
| 123 |
}
|
| 124 |
|
| 125 |
// compare schemas
|
requirements.txt
CHANGED
|
@@ -1,17 +1,20 @@
|
|
| 1 |
-
# Core
|
| 2 |
-
spacy
|
| 3 |
dateparser
|
| 4 |
-
networkx
|
| 5 |
-
|
| 6 |
-
SPARQLWrapper
|
| 7 |
|
| 8 |
# API
|
| 9 |
fastapi
|
| 10 |
uvicorn[standard]
|
| 11 |
-
pydantic
|
|
|
|
| 12 |
|
| 13 |
-
#
|
| 14 |
-
|
|
|
|
|
|
|
|
|
|
| 15 |
requests
|
| 16 |
python-dotenv
|
| 17 |
|
|
@@ -19,6 +22,3 @@ python-dotenv
|
|
| 19 |
pytest
|
| 20 |
pytest-asyncio
|
| 21 |
httpx
|
| 22 |
-
|
| 23 |
-
# Notebooks
|
| 24 |
-
jupyter
|
|
|
|
| 1 |
+
# Core pipeline
|
| 2 |
+
spacy
|
| 3 |
dateparser
|
| 4 |
+
networkx
|
| 5 |
+
neo4j
|
|
|
|
| 6 |
|
| 7 |
# API
|
| 8 |
fastapi
|
| 9 |
uvicorn[standard]
|
| 10 |
+
pydantic
|
| 11 |
+
python-multipart
|
| 12 |
|
| 13 |
+
# File upload extraction
|
| 14 |
+
python-docx
|
| 15 |
+
PyPDF2
|
| 16 |
+
|
| 17 |
+
# HTTP and env
|
| 18 |
requests
|
| 19 |
python-dotenv
|
| 20 |
|
|
|
|
| 22 |
pytest
|
| 23 |
pytest-asyncio
|
| 24 |
httpx
|
|
|
|
|
|
|
|
|
training/Modelfile
DELETED
|
@@ -1,32 +0,0 @@
|
|
| 1 |
-
FROM ./output/fakenews-ner.gguf
|
| 2 |
-
|
| 3 |
-
PARAMETER temperature 0.1
|
| 4 |
-
PARAMETER num_predict 4096
|
| 5 |
-
|
| 6 |
-
SYSTEM """You are a temporal fact extraction system. Given a news article, extract ALL temporal facts.
|
| 7 |
-
|
| 8 |
-
For each fact, return a JSON object with:
|
| 9 |
-
- "subject": the main entity (person, organization, location)
|
| 10 |
-
- "subject_type": one of PERSON, ORG, GPE, EVENT, NORP, PRODUCT, OTHER
|
| 11 |
-
- "predicate": the relationship (e.g. "holds_position", "member_of", "occurred_on", "caused", "preceded", "followed", "generic")
|
| 12 |
-
- "object": the related entity or context
|
| 13 |
-
- "object_type": same types as subject_type
|
| 14 |
-
- "time_expression": the temporal expression exactly as it appears in the text
|
| 15 |
-
- "time_start": start date if it's a range (ISO format YYYY-MM-DD or null)
|
| 16 |
-
- "time_end": end date if it's a range (ISO format YYYY-MM-DD or null)
|
| 17 |
-
- "time_point": specific date if it's a single point (ISO format or null)
|
| 18 |
-
- "source_sentence": the original sentence from the article that contains this fact
|
| 19 |
-
- "confidence": your confidence 0.0–1.0
|
| 20 |
-
|
| 21 |
-
Examples:
|
| 22 |
-
|
| 23 |
-
Input: "Barack Obama served as the 44th President of the United States from January 2009 to January 2017."
|
| 24 |
-
Output:
|
| 25 |
-
[{"subject": "Barack Obama", "subject_type": "PERSON", "predicate": "holds_position", "object": "44th President of the United States", "object_type": "ORG", "time_expression": "from January 2009 to January 2017", "time_start": "2009-01-20", "time_end": "2017-01-20", "time_point": null, "source_sentence": "Barack Obama served as the 44th President of the United States from January 2009 to January 2017.", "confidence": 0.95}]
|
| 26 |
-
|
| 27 |
-
Input: "The Affordable Care Act was signed into law on March 23, 2010 by President Obama."
|
| 28 |
-
Output:
|
| 29 |
-
[{"subject": "Affordable Care Act", "subject_type": "EVENT", "predicate": "occurred_on", "object": "signed into law", "object_type": "EVENT", "time_expression": "March 23, 2010", "time_start": null, "time_end": null, "time_point": "2010-03-23", "source_sentence": "The Affordable Care Act was signed into law on March 23, 2010 by President Obama.", "confidence": 0.92},
|
| 30 |
-
{"subject": "President Obama", "subject_type": "PERSON", "predicate": "generic", "object": "Affordable Care Act", "object_type": "EVENT", "time_expression": "March 23, 2010", "time_start": null, "time_end": null, "time_point": "2010-03-23", "source_sentence": "The Affordable Care Act was signed into law on March 23, 2010 by President Obama.", "confidence": 0.88}]
|
| 31 |
-
|
| 32 |
-
Return ONLY a JSON array. No explanations, no markdown fences."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
training/README.md
DELETED
|
@@ -1,120 +0,0 @@
|
|
| 1 |
-
# Fine-Tuning: fakenews-ner
|
| 2 |
-
|
| 3 |
-
QLoRA fine-tuning pipeline for the `fakenews-ner` model — a Mistral-7B variant
|
| 4 |
-
specialized for temporal fact extraction from political news articles.
|
| 5 |
-
|
| 6 |
-
The fine-tuned model is exported to GGUF and loaded into Ollama as `fakenews-ner`,
|
| 7 |
-
where it replaces zero-shot inference with task-specific extraction.
|
| 8 |
-
|
| 9 |
-
## Prerequisites
|
| 10 |
-
|
| 11 |
-
**Hardware:** CUDA-capable GPU with at least 8 GB VRAM (developed on RTX 4060 Laptop).
|
| 12 |
-
|
| 13 |
-
**Software:**
|
| 14 |
-
- Python 3.11
|
| 15 |
-
- CUDA 12.1+ with matching PyTorch
|
| 16 |
-
- Ollama (for the final registration step)
|
| 17 |
-
|
| 18 |
-
**Install training dependencies** (separate from the main backend requirements):
|
| 19 |
-
|
| 20 |
-
```bash
|
| 21 |
-
pip install -r training/requirements.txt
|
| 22 |
-
```
|
| 23 |
-
|
| 24 |
-
unsloth must be installed with the correct CUDA index. If the above fails, follow
|
| 25 |
-
the manual install at https://docs.unsloth.ai/get-started/installation.
|
| 26 |
-
|
| 27 |
-
## Steps
|
| 28 |
-
|
| 29 |
-
### 1 — Prepare dataset (~5 minutes)
|
| 30 |
-
|
| 31 |
-
Runs `SpacyExtractor(en_core_web_trf)` on each article to produce silver-label
|
| 32 |
-
temporal facts, then formats them as instruction/input/output triplets.
|
| 33 |
-
|
| 34 |
-
```bash
|
| 35 |
-
# Default: reads data/datasets/liar/, writes training/data/
|
| 36 |
-
python training/prepare_dataset.py
|
| 37 |
-
|
| 38 |
-
# Custom article directory (.txt files)
|
| 39 |
-
python training/prepare_dataset.py --input-dir path/to/articles/ --max-articles 300
|
| 40 |
-
```
|
| 41 |
-
|
| 42 |
-
**Input format:** either LIAR-style `.tsv` files (id, label, statement, ...) or
|
| 43 |
-
plain `.txt` files, one article per file.
|
| 44 |
-
|
| 45 |
-
**Output:** `training/data/train.jsonl` (80%) and `training/data/eval.jsonl` (20%).
|
| 46 |
-
|
| 47 |
-
Articles where spaCy extracts zero temporal facts are skipped — they provide no
|
| 48 |
-
training signal. Expect 100–150 usable examples from 200 articles.
|
| 49 |
-
|
| 50 |
-
### 2 — Train (~1–2 hours on RTX 4060)
|
| 51 |
-
|
| 52 |
-
```bash
|
| 53 |
-
python training/train_qlora.py
|
| 54 |
-
```
|
| 55 |
-
|
| 56 |
-
LoRA configuration: r=16, alpha=32, target modules q/k/v/o projections.
|
| 57 |
-
Batch size 1 with 4 gradient accumulation steps (effective batch = 4).
|
| 58 |
-
Adapter weights are saved to `training/output/fakenews-ner-lora/`.
|
| 59 |
-
|
| 60 |
-
Optional overrides:
|
| 61 |
-
|
| 62 |
-
```bash
|
| 63 |
-
python training/train_qlora.py --epochs 5 --lr 1e-4
|
| 64 |
-
python training/train_qlora.py --base-model unsloth/llama-3-8b-Instruct-bnb-4bit
|
| 65 |
-
```
|
| 66 |
-
|
| 67 |
-
### 3 — Export to GGUF (~10 minutes)
|
| 68 |
-
|
| 69 |
-
Merges the LoRA adapter into the base model and quantizes to Q4_K_M.
|
| 70 |
-
|
| 71 |
-
```bash
|
| 72 |
-
python training/export_gguf.py
|
| 73 |
-
```
|
| 74 |
-
|
| 75 |
-
Output: `training/output/fakenews-ner.gguf`
|
| 76 |
-
|
| 77 |
-
### 4 — Register in Ollama
|
| 78 |
-
|
| 79 |
-
```bash
|
| 80 |
-
# Linux / macOS / Git Bash
|
| 81 |
-
bash training/register_ollama.sh
|
| 82 |
-
|
| 83 |
-
# Windows PowerShell
|
| 84 |
-
.\training\register_ollama.ps1
|
| 85 |
-
```
|
| 86 |
-
|
| 87 |
-
This runs `ollama create fakenews-ner -f training/Modelfile`. The Modelfile points
|
| 88 |
-
to the `.gguf` file and embeds the system prompt with few-shot examples.
|
| 89 |
-
|
| 90 |
-
Verify the model is available:
|
| 91 |
-
|
| 92 |
-
```bash
|
| 93 |
-
ollama list
|
| 94 |
-
ollama run fakenews-ner "Extract temporal facts from: Obama was president from 2009 to 2017."
|
| 95 |
-
```
|
| 96 |
-
|
| 97 |
-
Once registered, set `LLM_MODELS=sciphi/triplex,fakenews-ner` and
|
| 98 |
-
`LLM_DEFAULT_MODEL=fakenews-ner` in `.env` to use it as the default LLM pipeline.
|
| 99 |
-
|
| 100 |
-
## File Overview
|
| 101 |
-
|
| 102 |
-
| File | Purpose |
|
| 103 |
-
|------|---------|
|
| 104 |
-
| `prepare_dataset.py` | Generate train/eval JSONL from articles using spaCy silver labels |
|
| 105 |
-
| `train_qlora.py` | QLoRA fine-tuning with unsloth + SFTTrainer |
|
| 106 |
-
| `export_gguf.py` | Merge adapter and export to GGUF Q4_K_M |
|
| 107 |
-
| `Modelfile` | Ollama model definition (FROM + SYSTEM prompt) |
|
| 108 |
-
| `register_ollama.sh` | Register GGUF in Ollama (Linux/macOS/Git Bash) |
|
| 109 |
-
| `register_ollama.ps1` | Register GGUF in Ollama (Windows PowerShell) |
|
| 110 |
-
| `data/` | Generated train.jsonl and eval.jsonl (git-ignored) |
|
| 111 |
-
| `output/` | LoRA adapter and GGUF file (git-ignored) |
|
| 112 |
-
|
| 113 |
-
## Expected VRAM Usage
|
| 114 |
-
|
| 115 |
-
| Stage | VRAM |
|
| 116 |
-
|-------|------|
|
| 117 |
-
| Prepare dataset (spaCy en_core_web_trf) | ~2 GB |
|
| 118 |
-
| Training (Mistral-7B 4-bit + LoRA) | ~7–7.5 GB |
|
| 119 |
-
| GGUF export (merge + quantize) | ~7 GB |
|
| 120 |
-
| Ollama inference (Q4_K_M) | ~5 GB |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
training/data/.gitkeep
DELETED
|
File without changes
|
training/data/eval.jsonl
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|
training/data/train.jsonl
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|
training/export_gguf.py
DELETED
|
@@ -1,96 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Merge LoRA adapter with the base model and export to GGUF Q4_K_M format.
|
| 3 |
-
|
| 4 |
-
The resulting .gguf file can be registered directly in Ollama via register_ollama.sh
|
| 5 |
-
or register_ollama.ps1.
|
| 6 |
-
|
| 7 |
-
Usage:
|
| 8 |
-
python training/export_gguf.py
|
| 9 |
-
python training/export_gguf.py --adapter-dir training/output/fakenews-ner-lora
|
| 10 |
-
python training/export_gguf.py --output training/output/fakenews-ner.gguf
|
| 11 |
-
"""
|
| 12 |
-
|
| 13 |
-
from __future__ import annotations
|
| 14 |
-
|
| 15 |
-
import argparse
|
| 16 |
-
import logging
|
| 17 |
-
from pathlib import Path
|
| 18 |
-
|
| 19 |
-
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
| 20 |
-
logger = logging.getLogger(__name__)
|
| 21 |
-
|
| 22 |
-
DEFAULT_ADAPTER_DIR = Path(__file__).parent / "output" / "fakenews-ner-lora"
|
| 23 |
-
DEFAULT_OUTPUT = Path(__file__).parent / "output" / "fakenews-ner.gguf"
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
def parse_args() -> argparse.Namespace:
|
| 27 |
-
parser = argparse.ArgumentParser(
|
| 28 |
-
description="Merge LoRA adapter and export to GGUF for Ollama"
|
| 29 |
-
)
|
| 30 |
-
parser.add_argument(
|
| 31 |
-
"--adapter-dir",
|
| 32 |
-
type=Path,
|
| 33 |
-
default=DEFAULT_ADAPTER_DIR,
|
| 34 |
-
help="Directory containing the saved LoRA adapter (output of train_qlora.py)",
|
| 35 |
-
)
|
| 36 |
-
parser.add_argument(
|
| 37 |
-
"--output",
|
| 38 |
-
type=Path,
|
| 39 |
-
default=DEFAULT_OUTPUT,
|
| 40 |
-
help="Path for the exported .gguf file",
|
| 41 |
-
)
|
| 42 |
-
parser.add_argument(
|
| 43 |
-
"--quantization",
|
| 44 |
-
default="q4_k_m",
|
| 45 |
-
choices=["q4_k_m", "q8_0", "f16"],
|
| 46 |
-
help="GGUF quantization type (q4_k_m recommended for 8 GB VRAM inference)",
|
| 47 |
-
)
|
| 48 |
-
return parser.parse_args()
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
def main() -> None:
|
| 52 |
-
args = parse_args()
|
| 53 |
-
|
| 54 |
-
if not args.adapter_dir.exists():
|
| 55 |
-
raise FileNotFoundError(
|
| 56 |
-
f"Adapter directory not found: {args.adapter_dir}\n"
|
| 57 |
-
"Run train_qlora.py first."
|
| 58 |
-
)
|
| 59 |
-
|
| 60 |
-
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 61 |
-
|
| 62 |
-
logger.info(f"Loading adapter from : {args.adapter_dir}")
|
| 63 |
-
logger.info(f"Output GGUF path : {args.output}")
|
| 64 |
-
logger.info(f"Quantization : {args.quantization}")
|
| 65 |
-
|
| 66 |
-
from unsloth import FastLanguageModel
|
| 67 |
-
|
| 68 |
-
# Reload the adapter (unsloth re-attaches it to the base model automatically)
|
| 69 |
-
model, tokenizer = FastLanguageModel.from_pretrained(
|
| 70 |
-
model_name=str(args.adapter_dir),
|
| 71 |
-
max_seq_length=2048,
|
| 72 |
-
dtype=None,
|
| 73 |
-
load_in_4bit=True,
|
| 74 |
-
)
|
| 75 |
-
|
| 76 |
-
logger.info("Merging adapter weights into base model...")
|
| 77 |
-
|
| 78 |
-
# save_pretrained_gguf merges, quantizes, and writes the .gguf in one step
|
| 79 |
-
model.save_pretrained_gguf(
|
| 80 |
-
str(args.output.with_suffix("")),
|
| 81 |
-
tokenizer,
|
| 82 |
-
quantization_method=args.quantization,
|
| 83 |
-
)
|
| 84 |
-
|
| 85 |
-
# unsloth appends the quantization suffix; rename to the clean target path
|
| 86 |
-
candidate = args.output.parent / f"{args.output.stem}-{args.quantization}.gguf"
|
| 87 |
-
if candidate.exists() and candidate != args.output:
|
| 88 |
-
candidate.rename(args.output)
|
| 89 |
-
logger.info(f"Renamed {candidate.name} -> {args.output.name}")
|
| 90 |
-
|
| 91 |
-
logger.info(f"Export complete: {args.output}")
|
| 92 |
-
logger.info("Next step: run register_ollama.sh (or .ps1) to load into Ollama")
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
if __name__ == "__main__":
|
| 96 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
training/prepare_dataset.py
DELETED
|
@@ -1,229 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Prepare instruction-tuning dataset for fakenews-ner fine-tuning.
|
| 3 |
-
|
| 4 |
-
Uses SpacyExtractor (en_core_web_trf) as a silver-label teacher to annotate
|
| 5 |
-
articles, then formats each article as an instruction/input/output triplet
|
| 6 |
-
matching the LLMExtractor prompt schema.
|
| 7 |
-
|
| 8 |
-
Usage:
|
| 9 |
-
python training/prepare_dataset.py
|
| 10 |
-
python training/prepare_dataset.py --input-dir data/datasets/liar --max-articles 500
|
| 11 |
-
python training/prepare_dataset.py --input-dir my_articles/ --split-ratio 0.85
|
| 12 |
-
"""
|
| 13 |
-
|
| 14 |
-
from __future__ import annotations
|
| 15 |
-
|
| 16 |
-
import argparse
|
| 17 |
-
import json
|
| 18 |
-
import logging
|
| 19 |
-
import sys
|
| 20 |
-
from pathlib import Path
|
| 21 |
-
from typing import Optional
|
| 22 |
-
|
| 23 |
-
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
| 24 |
-
logger = logging.getLogger(__name__)
|
| 25 |
-
|
| 26 |
-
# Repo root must be on sys.path so backend imports resolve
|
| 27 |
-
_REPO_ROOT = Path(__file__).parent.parent
|
| 28 |
-
if str(_REPO_ROOT) not in sys.path:
|
| 29 |
-
sys.path.insert(0, str(_REPO_ROOT))
|
| 30 |
-
|
| 31 |
-
from backend.pipeline.extraction.llm_extractor import SYSTEM_PROMPT, USER_PROMPT_TEMPLATE
|
| 32 |
-
from backend.pipeline.extraction.spacy_extractor import SpacyExtractor
|
| 33 |
-
from backend.pipeline.graph.models import Article, TemporalFact
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
def fact_to_dict(fact: TemporalFact) -> dict:
|
| 37 |
-
"""Convert a TemporalFact to the JSON schema expected by the LLM."""
|
| 38 |
-
time_expr = ""
|
| 39 |
-
if fact.time_point:
|
| 40 |
-
time_expr = fact.time_point.raw_text
|
| 41 |
-
elif fact.time_start:
|
| 42 |
-
time_expr = fact.time_start.raw_text
|
| 43 |
-
|
| 44 |
-
return {
|
| 45 |
-
"subject": fact.subject.text,
|
| 46 |
-
"subject_type": fact.subject.entity_type.value,
|
| 47 |
-
"predicate": fact.predicate.value,
|
| 48 |
-
"object": fact.object.text,
|
| 49 |
-
"object_type": fact.object.entity_type.value,
|
| 50 |
-
"time_expression": time_expr,
|
| 51 |
-
"time_start": fact.time_start.date_string if fact.time_start else None,
|
| 52 |
-
"time_end": fact.time_end.date_string if fact.time_end else None,
|
| 53 |
-
"time_point": fact.time_point.date_string if fact.time_point else None,
|
| 54 |
-
"source_sentence": fact.source_sentence,
|
| 55 |
-
"confidence": round(fact.extraction_confidence, 2),
|
| 56 |
-
}
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
def load_articles_tsv(tsv_path: Path, max_articles: int) -> list[Article]:
|
| 60 |
-
"""Load articles from a LIAR-style TSV file (no header row).
|
| 61 |
-
|
| 62 |
-
Column layout: id, label, statement, subject, speaker, job, state,
|
| 63 |
-
party, counts x5, context
|
| 64 |
-
"""
|
| 65 |
-
articles = []
|
| 66 |
-
with open(tsv_path, encoding="utf-8") as f:
|
| 67 |
-
for line in f:
|
| 68 |
-
if len(articles) >= max_articles:
|
| 69 |
-
break
|
| 70 |
-
parts = line.rstrip("\n").split("\t")
|
| 71 |
-
if len(parts) < 3:
|
| 72 |
-
continue
|
| 73 |
-
text = parts[2].strip()
|
| 74 |
-
if not text:
|
| 75 |
-
continue
|
| 76 |
-
speaker = parts[4].strip() if len(parts) > 4 else ""
|
| 77 |
-
articles.append(Article(text=text, title=speaker))
|
| 78 |
-
return articles
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
def load_articles_txt(txt_dir: Path, max_articles: int) -> list[Article]:
|
| 82 |
-
"""Load articles from a directory of plain-text .txt files."""
|
| 83 |
-
articles = []
|
| 84 |
-
for p in sorted(txt_dir.glob("*.txt")):
|
| 85 |
-
if len(articles) >= max_articles:
|
| 86 |
-
break
|
| 87 |
-
text = p.read_text(encoding="utf-8").strip()
|
| 88 |
-
if text:
|
| 89 |
-
articles.append(Article(text=text, title=p.stem))
|
| 90 |
-
return articles
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
def load_articles(input_dir: Path, max_articles: int) -> list[Article]:
|
| 94 |
-
"""Auto-detect TSV files or .txt files in the input directory."""
|
| 95 |
-
tsv_files = sorted(input_dir.glob("*.tsv")) + sorted(input_dir.glob("*.csv"))
|
| 96 |
-
if tsv_files:
|
| 97 |
-
logger.info(f"Found TSV file(s): {[f.name for f in tsv_files]}")
|
| 98 |
-
articles: list[Article] = []
|
| 99 |
-
for tsv in tsv_files:
|
| 100 |
-
articles.extend(load_articles_tsv(tsv, max_articles - len(articles)))
|
| 101 |
-
if len(articles) >= max_articles:
|
| 102 |
-
break
|
| 103 |
-
return articles
|
| 104 |
-
|
| 105 |
-
txt_files = list(input_dir.glob("*.txt"))
|
| 106 |
-
if txt_files:
|
| 107 |
-
logger.info(f"Found {len(txt_files)} .txt file(s)")
|
| 108 |
-
return load_articles_txt(input_dir, max_articles)
|
| 109 |
-
|
| 110 |
-
logger.warning(f"No .tsv or .txt files found in {input_dir}")
|
| 111 |
-
return []
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
def build_example(article: Article, facts: list[TemporalFact]) -> dict:
|
| 115 |
-
"""Format a single article + its silver facts as an instruction example."""
|
| 116 |
-
pub_date_str = (
|
| 117 |
-
article.publication_date.strftime("%Y-%m-%d")
|
| 118 |
-
if article.publication_date else "unknown"
|
| 119 |
-
)
|
| 120 |
-
user_input = USER_PROMPT_TEMPLATE.format(
|
| 121 |
-
title=article.title or "(untitled)",
|
| 122 |
-
pub_date=pub_date_str,
|
| 123 |
-
text=article.text[:4000],
|
| 124 |
-
)
|
| 125 |
-
output_json = json.dumps([fact_to_dict(f) for f in facts], ensure_ascii=False)
|
| 126 |
-
return {
|
| 127 |
-
"instruction": SYSTEM_PROMPT,
|
| 128 |
-
"input": user_input,
|
| 129 |
-
"output": output_json,
|
| 130 |
-
}
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
def write_jsonl(examples: list[dict], path: Path) -> None:
|
| 134 |
-
path.parent.mkdir(parents=True, exist_ok=True)
|
| 135 |
-
with open(path, "w", encoding="utf-8") as f:
|
| 136 |
-
for ex in examples:
|
| 137 |
-
f.write(json.dumps(ex, ensure_ascii=False) + "\n")
|
| 138 |
-
logger.info(f"Wrote {len(examples)} examples to {path}")
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
def main() -> None:
|
| 142 |
-
parser = argparse.ArgumentParser(
|
| 143 |
-
description="Prepare instruction-tuning data for fakenews-ner"
|
| 144 |
-
)
|
| 145 |
-
parser.add_argument(
|
| 146 |
-
"--input-dir",
|
| 147 |
-
type=Path,
|
| 148 |
-
default=_REPO_ROOT / "data" / "datasets" / "liar",
|
| 149 |
-
help="Directory containing .tsv or .txt article files",
|
| 150 |
-
)
|
| 151 |
-
parser.add_argument(
|
| 152 |
-
"--output-dir",
|
| 153 |
-
type=Path,
|
| 154 |
-
default=Path(__file__).parent / "data",
|
| 155 |
-
help="Directory to write train.jsonl and eval.jsonl",
|
| 156 |
-
)
|
| 157 |
-
parser.add_argument(
|
| 158 |
-
"--max-articles",
|
| 159 |
-
type=int,
|
| 160 |
-
default=200,
|
| 161 |
-
help="Maximum number of articles to process",
|
| 162 |
-
)
|
| 163 |
-
parser.add_argument(
|
| 164 |
-
"--split-ratio",
|
| 165 |
-
type=float,
|
| 166 |
-
default=0.8,
|
| 167 |
-
help="Fraction of examples to use for training (rest goes to eval)",
|
| 168 |
-
)
|
| 169 |
-
args = parser.parse_args()
|
| 170 |
-
|
| 171 |
-
if not args.input_dir.exists():
|
| 172 |
-
logger.error(f"Input directory not found: {args.input_dir}")
|
| 173 |
-
logger.error("Place LIAR .tsv files in data/datasets/liar/ or pass --input-dir")
|
| 174 |
-
sys.exit(1)
|
| 175 |
-
|
| 176 |
-
logger.info(f"Loading articles from {args.input_dir}")
|
| 177 |
-
articles = load_articles(args.input_dir, args.max_articles)
|
| 178 |
-
if not articles:
|
| 179 |
-
logger.error("No articles loaded. Exiting.")
|
| 180 |
-
sys.exit(1)
|
| 181 |
-
logger.info(f"Loaded {len(articles)} articles")
|
| 182 |
-
|
| 183 |
-
logger.info("Initialising SpacyExtractor (en_core_web_trf) — loading model once...")
|
| 184 |
-
extractor = SpacyExtractor(model_name="en_core_web_trf")
|
| 185 |
-
|
| 186 |
-
# Fail fast if model can't load
|
| 187 |
-
try:
|
| 188 |
-
_ = extractor.nlp
|
| 189 |
-
except Exception as e:
|
| 190 |
-
print(f"FATAL: Cannot load spaCy model: {e}")
|
| 191 |
-
print("Run: python -m spacy download en_core_web_trf")
|
| 192 |
-
sys.exit(1)
|
| 193 |
-
|
| 194 |
-
examples: list[dict] = []
|
| 195 |
-
skipped = 0
|
| 196 |
-
|
| 197 |
-
for i, article in enumerate(articles, start=1):
|
| 198 |
-
try:
|
| 199 |
-
facts = extractor.extract(article)
|
| 200 |
-
except Exception as e:
|
| 201 |
-
logger.warning(f"Article {i}: extraction error — {e}")
|
| 202 |
-
skipped += 1
|
| 203 |
-
continue
|
| 204 |
-
|
| 205 |
-
if not facts:
|
| 206 |
-
skipped += 1
|
| 207 |
-
continue
|
| 208 |
-
|
| 209 |
-
examples.append(build_example(article, facts))
|
| 210 |
-
|
| 211 |
-
if i % 20 == 0:
|
| 212 |
-
logger.info(f"Progress: {i}/{len(articles)} — {len(examples)} usable so far")
|
| 213 |
-
|
| 214 |
-
logger.info(
|
| 215 |
-
f"Finished: {len(examples)} usable examples, {skipped} skipped (no temporal facts)"
|
| 216 |
-
)
|
| 217 |
-
|
| 218 |
-
split = int(len(examples) * args.split_ratio)
|
| 219 |
-
train_examples = examples[:split]
|
| 220 |
-
eval_examples = examples[split:]
|
| 221 |
-
|
| 222 |
-
write_jsonl(train_examples, args.output_dir / "train.jsonl")
|
| 223 |
-
write_jsonl(eval_examples, args.output_dir / "eval.jsonl")
|
| 224 |
-
|
| 225 |
-
logger.info(f"Dataset ready: {len(train_examples)} train / {len(eval_examples)} eval")
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
if __name__ == "__main__":
|
| 229 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
training/register_ollama.ps1
DELETED
|
@@ -1,22 +0,0 @@
|
|
| 1 |
-
# Register the fine-tuned fakenews-ner model in Ollama using the local Modelfile.
|
| 2 |
-
# Run from the repo root or the training/ directory.
|
| 3 |
-
|
| 4 |
-
Set-Location $PSScriptRoot
|
| 5 |
-
|
| 6 |
-
if (-not (Get-Command ollama -ErrorAction SilentlyContinue)) {
|
| 7 |
-
Write-Error "ERROR: 'ollama' not found. Install Ollama from https://ollama.com and try again."
|
| 8 |
-
exit 1
|
| 9 |
-
}
|
| 10 |
-
|
| 11 |
-
if (-not (Test-Path "output\fakenews-ner.gguf")) {
|
| 12 |
-
Write-Error "ERROR: output\fakenews-ner.gguf not found."
|
| 13 |
-
Write-Host "Run export_gguf.py first to generate the GGUF file."
|
| 14 |
-
exit 1
|
| 15 |
-
}
|
| 16 |
-
|
| 17 |
-
Write-Host "Registering model 'fakenews-ner' in Ollama..."
|
| 18 |
-
ollama create fakenews-ner -f Modelfile
|
| 19 |
-
|
| 20 |
-
Write-Host ""
|
| 21 |
-
Write-Host "Done. Test the model with:"
|
| 22 |
-
Write-Host " ollama run fakenews-ner 'Extract temporal facts from: Obama was president from 2009 to 2017.'"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
training/register_ollama.sh
DELETED
|
@@ -1,23 +0,0 @@
|
|
| 1 |
-
#!/bin/bash
|
| 2 |
-
# Register the fine-tuned fakenews-ner model in Ollama using the local Modelfile.
|
| 3 |
-
# Run from the repo root or the training/ directory — the script locates itself.
|
| 4 |
-
|
| 5 |
-
cd "$(dirname "$0")"
|
| 6 |
-
|
| 7 |
-
if ! command -v ollama &>/dev/null; then
|
| 8 |
-
echo "ERROR: 'ollama' not found. Install Ollama from https://ollama.com and try again."
|
| 9 |
-
exit 1
|
| 10 |
-
fi
|
| 11 |
-
|
| 12 |
-
if [ ! -f "output/fakenews-ner.gguf" ]; then
|
| 13 |
-
echo "ERROR: output/fakenews-ner.gguf not found."
|
| 14 |
-
echo "Run export_gguf.py first to generate the GGUF file."
|
| 15 |
-
exit 1
|
| 16 |
-
fi
|
| 17 |
-
|
| 18 |
-
echo "Registering model 'fakenews-ner' in Ollama..."
|
| 19 |
-
ollama create fakenews-ner -f Modelfile
|
| 20 |
-
|
| 21 |
-
echo ""
|
| 22 |
-
echo "Done. Test the model with:"
|
| 23 |
-
echo " ollama run fakenews-ner 'Extract temporal facts from: Obama was president from 2009 to 2017.'"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
training/requirements.txt
DELETED
|
@@ -1,8 +0,0 @@
|
|
| 1 |
-
unsloth[colab-new]
|
| 2 |
-
torch>=2.1.0
|
| 3 |
-
peft
|
| 4 |
-
bitsandbytes
|
| 5 |
-
transformers
|
| 6 |
-
trl
|
| 7 |
-
datasets
|
| 8 |
-
accelerate
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
training/train_qlora.py
DELETED
|
@@ -1,165 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
QLoRA fine-tuning of Mistral-7B for temporal fact extraction (fakenews-ner).
|
| 3 |
-
|
| 4 |
-
Requires unsloth and a CUDA-capable GPU. Designed for RTX 4060 Laptop (8 GB VRAM)
|
| 5 |
-
using 4-bit quantization. Adapter weights are saved to --output-dir and can be
|
| 6 |
-
merged + exported with export_gguf.py.
|
| 7 |
-
|
| 8 |
-
Usage:
|
| 9 |
-
python training/train_qlora.py
|
| 10 |
-
python training/train_qlora.py --epochs 5 --lr 1e-4
|
| 11 |
-
python training/train_qlora.py --base-model unsloth/llama-3-8b-Instruct-bnb-4bit
|
| 12 |
-
"""
|
| 13 |
-
|
| 14 |
-
from __future__ import annotations
|
| 15 |
-
|
| 16 |
-
import argparse
|
| 17 |
-
import logging
|
| 18 |
-
from pathlib import Path
|
| 19 |
-
|
| 20 |
-
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
| 21 |
-
logger = logging.getLogger(__name__)
|
| 22 |
-
|
| 23 |
-
DEFAULT_BASE_MODEL = "unsloth/mistral-7b-instruct-v0.3-bnb-4bit"
|
| 24 |
-
DEFAULT_DATA_DIR = Path(__file__).parent / "data"
|
| 25 |
-
DEFAULT_OUTPUT_DIR = Path(__file__).parent / "output" / "fakenews-ner-lora"
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
def parse_args() -> argparse.Namespace:
|
| 29 |
-
parser = argparse.ArgumentParser(
|
| 30 |
-
description="QLoRA fine-tuning for fakenews-ner on RTX 4060 (8 GB VRAM)"
|
| 31 |
-
)
|
| 32 |
-
parser.add_argument(
|
| 33 |
-
"--base-model",
|
| 34 |
-
default=DEFAULT_BASE_MODEL,
|
| 35 |
-
help="Hugging Face model ID or local path (must be 4-bit bnb quantized for 8 GB VRAM)",
|
| 36 |
-
)
|
| 37 |
-
parser.add_argument(
|
| 38 |
-
"--data-dir",
|
| 39 |
-
type=Path,
|
| 40 |
-
default=DEFAULT_DATA_DIR,
|
| 41 |
-
help="Directory containing train.jsonl and eval.jsonl",
|
| 42 |
-
)
|
| 43 |
-
parser.add_argument(
|
| 44 |
-
"--output-dir",
|
| 45 |
-
type=Path,
|
| 46 |
-
default=DEFAULT_OUTPUT_DIR,
|
| 47 |
-
help="Directory to save LoRA adapter weights",
|
| 48 |
-
)
|
| 49 |
-
parser.add_argument("--epochs", type=int, default=3, help="Number of training epochs")
|
| 50 |
-
parser.add_argument("--lr", type=float, default=2e-4, help="Learning rate")
|
| 51 |
-
return parser.parse_args()
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
def load_dataset(data_dir: Path):
|
| 55 |
-
"""Load train/eval JSONL files as Hugging Face datasets."""
|
| 56 |
-
from datasets import load_dataset as hf_load_dataset
|
| 57 |
-
|
| 58 |
-
train_path = data_dir / "train.jsonl"
|
| 59 |
-
eval_path = data_dir / "eval.jsonl"
|
| 60 |
-
|
| 61 |
-
if not train_path.exists():
|
| 62 |
-
raise FileNotFoundError(
|
| 63 |
-
f"Training data not found at {train_path}. "
|
| 64 |
-
"Run prepare_dataset.py first."
|
| 65 |
-
)
|
| 66 |
-
|
| 67 |
-
data_files = {"train": str(train_path)}
|
| 68 |
-
if eval_path.exists():
|
| 69 |
-
data_files["eval"] = str(eval_path)
|
| 70 |
-
|
| 71 |
-
dataset = hf_load_dataset("json", data_files=data_files)
|
| 72 |
-
logger.info(
|
| 73 |
-
f"Dataset loaded — train: {len(dataset['train'])} examples"
|
| 74 |
-
+ (f", eval: {len(dataset['eval'])} examples" if "eval" in dataset else "")
|
| 75 |
-
)
|
| 76 |
-
return dataset
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
def format_prompt(example: dict) -> dict:
|
| 80 |
-
"""Convert instruction/input/output dict to a single formatted text field.
|
| 81 |
-
|
| 82 |
-
Uses the Mistral instruct template: [INST] system + user [/INST] assistant
|
| 83 |
-
"""
|
| 84 |
-
prompt = (
|
| 85 |
-
f"[INST] {example['instruction']}\n\n{example['input']} [/INST] "
|
| 86 |
-
f"{example['output']}"
|
| 87 |
-
)
|
| 88 |
-
return {"text": prompt}
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
def main() -> None:
|
| 92 |
-
args = parse_args()
|
| 93 |
-
args.output_dir.mkdir(parents=True, exist_ok=True)
|
| 94 |
-
|
| 95 |
-
logger.info(f"Base model : {args.base_model}")
|
| 96 |
-
logger.info(f"Data dir : {args.data_dir}")
|
| 97 |
-
logger.info(f"Output dir : {args.output_dir}")
|
| 98 |
-
logger.info(f"Epochs : {args.epochs} LR: {args.lr}")
|
| 99 |
-
|
| 100 |
-
# Lazy imports — unsloth rewrites CUDA kernels at import time
|
| 101 |
-
from unsloth import FastLanguageModel
|
| 102 |
-
from trl import SFTTrainer
|
| 103 |
-
from transformers import TrainingArguments
|
| 104 |
-
|
| 105 |
-
# Load 4-bit quantized base model
|
| 106 |
-
model, tokenizer = FastLanguageModel.from_pretrained(
|
| 107 |
-
model_name=args.base_model,
|
| 108 |
-
max_seq_length=2048,
|
| 109 |
-
dtype=None, # auto-detect: float16 on Ampere+
|
| 110 |
-
load_in_4bit=True,
|
| 111 |
-
)
|
| 112 |
-
logger.info("Base model loaded")
|
| 113 |
-
|
| 114 |
-
# Apply LoRA adapters
|
| 115 |
-
model = FastLanguageModel.get_peft_model(
|
| 116 |
-
model,
|
| 117 |
-
r=16,
|
| 118 |
-
lora_alpha=32,
|
| 119 |
-
target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
|
| 120 |
-
lora_dropout=0.05,
|
| 121 |
-
bias="none",
|
| 122 |
-
use_gradient_checkpointing="unsloth",
|
| 123 |
-
random_state=42,
|
| 124 |
-
)
|
| 125 |
-
logger.info("LoRA adapters applied")
|
| 126 |
-
|
| 127 |
-
dataset = load_dataset(args.data_dir)
|
| 128 |
-
dataset = dataset.map(format_prompt)
|
| 129 |
-
|
| 130 |
-
training_args = TrainingArguments(
|
| 131 |
-
output_dir=str(args.output_dir),
|
| 132 |
-
num_train_epochs=args.epochs,
|
| 133 |
-
per_device_train_batch_size=1,
|
| 134 |
-
gradient_accumulation_steps=4,
|
| 135 |
-
warmup_steps=10,
|
| 136 |
-
learning_rate=args.lr,
|
| 137 |
-
fp16=True,
|
| 138 |
-
logging_steps=10,
|
| 139 |
-
save_strategy="epoch",
|
| 140 |
-
eval_strategy="epoch" if "eval" in dataset else "no",
|
| 141 |
-
load_best_model_at_end=("eval" in dataset),
|
| 142 |
-
report_to="none",
|
| 143 |
-
)
|
| 144 |
-
|
| 145 |
-
trainer = SFTTrainer(
|
| 146 |
-
model=model,
|
| 147 |
-
tokenizer=tokenizer,
|
| 148 |
-
train_dataset=dataset["train"],
|
| 149 |
-
eval_dataset=dataset.get("eval"),
|
| 150 |
-
dataset_text_field="text",
|
| 151 |
-
max_seq_length=2048,
|
| 152 |
-
args=training_args,
|
| 153 |
-
)
|
| 154 |
-
|
| 155 |
-
logger.info("Starting training...")
|
| 156 |
-
trainer.train()
|
| 157 |
-
|
| 158 |
-
model.save_pretrained(str(args.output_dir))
|
| 159 |
-
tokenizer.save_pretrained(str(args.output_dir))
|
| 160 |
-
logger.info(f"Adapter weights saved to {args.output_dir}")
|
| 161 |
-
logger.info("Next step: run export_gguf.py to merge and export to GGUF")
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
if __name__ == "__main__":
|
| 165 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|