diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..bc71228df9bffc46ac7032542595a0543e30e6e1 --- /dev/null +++ b/.env.example @@ -0,0 +1,42 @@ +# ============================================================ +# JuriX Backend — Environment Variables +# Copy this file to .env and fill in your values. +# ============================================================ + +# ---- Application ---- +ENVIRONMENT=development +DEBUG=true + +# ---- Neon PostgreSQL (pgvector) ---- +# Get this from your Neon project dashboard → Connection string +# ⚠️ Add ?sslmode=require at the end for production +DATABASE_URL=postgresql+asyncpg://user:password@ep-xxx-yyy.eu-central-1.aws.neon.tech/neondb?sslmode=require + +# ---- Upstash Redis ---- +# Get this from your Upstash console → REST API or Redis URL +# Use rediss:// (with SSL) for Upstash +REDIS_URL=rediss://:your-upstash-password@your-endpoint.upstash.io:6379 + +# ---- Gemini API (Google AI Studio) ---- +GEMINI_API_KEY=your-gemini-api-key-here +GEMINI_MODEL=gemini-3-flash +GEMINI_EMBEDDING_MODEL=models/gemini-embedding-001 + +# ---- LlamaParse (LlamaCloud) ---- +LLAMA_CLOUD_API_KEY=llx-your-llama-parse-key-here + +# ---- Meilisearch ---- +# If self-hosted: +MEILISEARCH_URL=http://localhost:7700 +MEILISEARCH_KEY=your-meilisearch-master-key + +# ---- Security ---- +# Generate with: openssl rand -hex 32 +SECRET_KEY=generate-a-strong-random-key-here +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=30 + +# ---- CORS ---- +# Vercel frontend URL(s), comma-separated (no trailing slash) +# Example: https://jurix.vercel.app,https://www.jurix.cm +ALLOWED_ORIGINS=https://jurix.vercel.app diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..a05f4b5fc9deaeff55693f63aedf33682df2287e --- /dev/null +++ b/.gitignore @@ -0,0 +1,52 @@ +# Python +__pycache__/ +*.py[cod] +*.pyo +*.pyd +.Python +*.egg-info/ +dist/ +build/ + +# Virtual envs +.venv/ +venv/ +env/ + +# Environment files (never commit secrets) +.env +.env.local +.env.*.local + +# Testing +.pytest_cache/ +htmlcov/ +.coverage +coverage.xml + +# ML models (large binary files — store elsewhere or use git LFS) +models/fasttext/ +models/*.bin +models/*.pkl + +# Temporary data +data/ +debug_output.txt +article_content.txt +final_verification.txt +models.txt +verification_report.txt +nul + +# IDE +.vscode/ +.idea/ +.qodo/ +pyrefly.toml + +# Docker +*.dockerignore + +# OS +.DS_Store +Thumbs.db diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..cd21da76b25233dad37931787fcfc0d984075437 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,69 @@ +# JuriX Backend — Production Dockerfile +# Python 3.11 avec support ML (scikit-learn, spaCy, fastText) + +FROM python:3.11-slim + +LABEL maintainer="JuriX Team " +LABEL description="JuriX v2.1 Backend API" +LABEL version="2.1.0" + +# System dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + g++ \ + curl \ + git \ + wget \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy dependency files first (for Docker layer caching) +COPY pyproject.toml poetry.lock* ./ + +# Install Poetry and dependencies +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir poetry==1.7.1 + +RUN poetry config virtualenvs.create false +RUN poetry install --no-dev --no-interaction --no-ansi + +# Copy application code +COPY . . + +# Download spaCy models +RUN python -m spacy download fr_core_news_sm --quiet || echo "fr model download failed" && \ + python -m spacy download en_core_web_sm --quiet || echo "en model download failed" + +# Create required directories +RUN mkdir -p /app/models/spacy /app/models/fasttext /app/logs /app/data + +# Download fastText language identification model +RUN if [ ! -f /app/models/fasttext/lid.176.bin ]; then \ + echo "Downloading fastText model..." && \ + wget -q https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin \ + -O /app/models/fasttext/lid.176.bin || \ + echo "fastText model download failed - will skip language detection"; \ + fi + +# Non-root user for security +RUN useradd -m -u 1000 -s /bin/bash jurix && \ + chown -R jurix:jurix /app + +USER jurix + +# Runtime env vars +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH=/app \ + ML_MODELS_PATH=/app/models \ + LOG_LEVEL=INFO + +# Render provides PORT env var — use it directly +# Use 1 worker on Render Starter (512 MB RAM); scale up on paid plans +CMD uvicorn app.main:app \ + --host 0.0.0.0 \ + --port ${PORT:-8000} \ + --workers 1 \ + --log-level info diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c1e43de8612307e03a71b8c6ccfcddd542f9cfee --- /dev/null +++ b/README.md @@ -0,0 +1,659 @@ +# JuriX Backend v2.1 + +Backend API pour la plateforme juridique camerounaise JuriX. + +## 🚀 Quick Start + +### Prérequis + +- Python 3.11+ +- PostgreSQL 15+ avec pgvector +- Redis 7.2+ +- Meilisearch 1.5+ +- Ollama avec mistral:7b + +### Installation + +```bash +# Cloner le projet +cd backend + +# Installer les dépendances +pip install -r requirements.txt +# OU avec Poetry +poetry install + +# Télécharger les modèles NLP (requis pour détection de langue) +python scripts/download_models.py + +# Configurer les variables d'environnement +cp .env.example .env +# Éditer .env avec vos paramètres + +# Lancer les migrations +alembic upgrade head + +# Démarrer le serveur +uvicorn app.main:app --reload +``` + +L'API sera disponible sur `http://localhost:8000` + +Documentation interactive: `http://localhost:8000/docs` + +## 📦 Modules Implémentés + +### ✅ Module 1: Détection Automatique de Langue + +Service de détection automatique de langue pour documents juridiques. + +**Architecture:** Triple-méthode ensemble avec vote majoritaire +- **Méthodes:** langdetect + spaCy + fastText +- **Précision:** >99% sur documents juridiques +- **Performance:** <1s pour textes jusqu'à 5000 caractères +- **Langues:** Français + Anglais (extensible) + +#### Setup + +```bash +# 1. Télécharger les modèles NLP +python scripts/download_models.py + +# Ceci télécharge: +# - spaCy fr_core_news_sm (~60MB) +# - spaCy en_core_web_sm (~60MB) +# - fastText lid.176.bin (~130MB) +# Total: ~250MB + +# 2. Vérifier l'installation +python -c "from app.services.language_detector import LanguageDetector; d = LanguageDetector(); print(d.health_check())" +``` + +#### Usage Programmatique + +```python +from app.services.language_detector import LanguageDetector + +# Créer instance (charge les modèles) +detector = LanguageDetector() + +# Détecter langue +text = """ +Article 1. La présente loi régit les conditions de création +des sociétés commerciales au Cameroun conformément au droit OHADA. +""" + +result = detector.detect(text) + +print(result) +# { +# 'language': 'fr', +# 'confidence': 0.98, +# 'method_votes': { +# 'langdetect': 'fr', +# 'spacy': 'fr', +# 'fasttext': 'fr' +# }, +# 'consensus': True, +# 'processing_time_ms': 450, +# 'text_length': 156 +# } +``` + +#### Usage API + +```bash +# Détecter langue d'un texte +curl -X POST http://localhost:8000/api/v1/language/detect \ + -H "Content-Type: application/json" \ + -d '{ + "text": "Article 1. La présente loi régit les sociétés commerciales au Cameroun.", + "min_confidence": 0.80 + }' + +# Réponse: +# { +# "language": "fr", +# "confidence": 0.98, +# "method_votes": { +# "langdetect": "fr", +# "spacy": "fr", +# "fasttext": "fr" +# }, +# "consensus": true, +# "processing_time_ms": 450, +# "text_length": 75 +# } + +# Health check +curl http://localhost:8000/api/v1/language/health + +# Réponse: +# { +# "service": "LanguageDetector", +# "status": "healthy", +# "models": { +# "spacy_fr": "✅ OK", +# "spacy_en": "✅ OK", +# "fasttext": "✅ OK" +# } +# } +``` + +#### Tests + +```bash +# Tests unitaires (5 tests de base + edge cases) +pytest backend/tests/test_services/test_language_detector.py -v + +# Avec coverage +pytest backend/tests/test_services/test_language_detector.py -v \ + --cov=backend/app/services/language_detector \ + --cov-report=html + +# Génération tests Qodo (auto, ~25 tests) +qodo test generate backend/app/services/language_detector.py --max-tests 25 + +# Tous les tests +pytest backend/tests/test_services/test_language_detector.py -v +``` + +#### Troubleshooting + +**❌ Erreur: "No module named spacy"** +```bash +pip install spacy langdetect fasttext-wheel scikit-learn +``` + +**❌ Erreur: "Can't find model 'fr_core_news_sm'"** +```bash +python scripts/download_models.py +# OU manuellement: +python -m spacy download fr_core_news_sm +python -m spacy download en_core_web_sm +``` + +**❌ Erreur: "fastText model not found"** +```bash +# Télécharger manuellement +wget https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin \ + -O backend/models/fasttext/lid.176.bin +``` + +**❌ Performance lente (>2s)** +- Vérifier que ThreadPoolExecutor est utilisé (parallélisme) +- Réduire la longueur du texte (sampling automatique à 5000 chars) +- Utiliser singleton pattern (éviter rechargement modèles) + +**❌ Précision faible (<95%)** +- Vérifier longueur texte (minimum 100 chars recommandé) +- Textes bilingues donnent confiance plus basse (normal) +- Vérifier que les 3 modèles sont correctement chargés + +## 🧪 Tests + +### Lancer les tests + +```bash +# Tous les tests +pytest backend/tests/ -v + +# Tests spécifiques +pytest backend/tests/test_services/test_language_detector.py -v + +# Avec coverage +pytest backend/tests/ -v --cov=backend/app --cov-report=html + +# Tests de performance +pytest backend/tests/test_services/test_language_detector.py::TestLanguageDetectorPerformance -v +``` + +### Structure des tests + +``` +backend/tests/ +├── test_main.py # Tests de base (health check, root) +├── test_services/ +│ └── test_language_detector.py # Tests détection langue (32 tests) +├── test_api/ +│ └── (à venir) +├── test_tasks/ +│ └── (à venir) +├── integration/ +│ └── (à venir) +└── performance/ + └── (à venir) +``` + +## 📊 Code Quality + +### Linting & Formatting + +```bash +# Black (formatting) +black backend/app backend/tests + +# Ruff (linting) +ruff check backend/app backend/tests + +# MyPy (type checking) +mypy backend/app/services/language_detector.py + +# Pylint +pylint backend/app/services/language_detector.py +``` + +### Coverage Target + +- **Overall:** >80% +- **Services:** >85% +- **Critical paths:** >95% + +## 🐳 Docker + +```bash +# Build image +docker build -t jurix-backend:v2.1 -f docker/Dockerfile.prod . + +# Run avec docker-compose +docker-compose up -d + +# Vérifier logs +docker-compose logs -f api + +# Health check +curl http://localhost:8000/health +curl http://localhost:8000/api/v1/language/health +``` + +## 📁 Structure du Projet + +``` +backend/ +├── app/ +│ ├── __init__.py +│ ├── main.py # Point d'entrée FastAPI +│ ├── core/ +│ │ ├── config.py # Configuration (Settings) +│ │ └── dependencies.py # Dependency injection +│ ├── api/ +│ │ ├── routes/ +│ │ │ └── language.py # ✅ Endpoints détection langue +│ │ └── dependencies/ +│ ├── services/ +│ │ └── language_detector.py # ✅ Service détection langue +│ ├── models/ # ORM models (SQLAlchemy) +│ ├── tasks/ # Celery tasks +│ └── utils/ # Utilitaires +├── models/ # Modèles NLP +│ ├── spacy/ +│ │ ├── fr_core_news_sm/ +│ │ └── en_core_web_sm/ +│ └── fasttext/ +│ └── lid.176.bin +├── scripts/ +│ └── download_models.py # ✅ Script téléchargement modèles +├── tests/ +│ ├── test_main.py +│ └── test_services/ +│ └── test_language_detector.py # ✅ 32 tests +├── alembic/ # Migrations DB +│ ├── env.py +│ └── versions/ +│ └── 001_initial_schema.py +├── pyproject.toml # Dependencies (Poetry) +├── requirements.txt # Dependencies (pip) +└── README.md # This file +``` + +## 🔧 Configuration + +Variables d'environnement (.env): + +```bash +# App +APP_NAME="JuriX API" +VERSION="2.1.0" +DEBUG=True +ENVIRONMENT="development" + +# Database +DATABASE_URL="postgresql://jurix:password@localhost:5432/jurix_db" + +# Redis +REDIS_URL="redis://:password@localhost:6379/0" + +# Meilisearch +MEILISEARCH_URL="http://localhost:7700" +MEILISEARCH_KEY="master_key" + +# Ollama +OLLAMA_URL="http://localhost:11434" +OLLAMA_MODEL="mistral:7b" + +# Security +SECRET_KEY="your_secret_key_here" +ALGORITHM="HS256" +ACCESS_TOKEN_EXPIRE_MINUTES=30 + +# Qodo (Tests auto-générés) +QODO_API_KEY="your_qodo_api_key" + +# CORS +CORS_ORIGINS=["http://localhost:5173","http://localhost:3000"] +``` + +## 📝 API Documentation + +Documentation interactive disponible sur: +- **Swagger UI:** http://localhost:8000/docs +- **ReDoc:** http://localhost:8000/redoc + +### Endpoints Principaux + +#### Language Detection + +``` +POST /api/v1/language/detect +GET /api/v1/language/health +``` + +#### À venir + +``` +POST /api/v1/laws # Créer loi +GET /api/v1/laws # Lister lois +GET /api/v1/laws/{id} # Détail loi +POST /api/v1/search # Recherche +POST /api/v1/chatbot/ask # Question RAG +GET /api/v1/categories # Catégories +GET /health # Health check global +``` + +### ✅ Module 3: Extraction d'Articles + +Utilitaire d'extraction automatique d'articles pour documents juridiques camerounais. + +**Architecture:** Regex-based pattern matching avec détection automatique +- **Patterns supportés:** Article X, Art. X, Article premier/première +- **Numérotation hiérarchique:** Support 1.1, 1.2.3 (parent_id) +- **Précision:** >95% sur documents légaux camerounais +- **Performance:** <500ms pour 1000 articles +- **Validation:** Minimum 3 articles requis (configurable) + +#### Usage Programmatique + +```python +from app.utils.text_chunker import extract_articles + +# Texte juridique +law_text = """ +LOI N° 2023-001 DU 15 JANVIER 2023 + +Article 1. Objet de la loi +La présente loi régit les sociétés commerciales au Cameroun. + +Article 2. Champ d'application +Les dispositions s'appliquent à toutes les sociétés. + +Article 3. Définitions +Au sens de la présente loi, on entend par société... +""" + +# Extraire articles +articles = extract_articles(law_text) + +print(f"Nombre d'articles: {len(articles)}") + +for article in articles: + print(f"Article {article['number']}: {article['title']}") + print(f" Contenu: {article['content'][:50]}...") + print(f" Mots: {article['word_count']}, Chars: {article['char_count']}") + print(f" Parent: {article['parent_id']}, Position: {article['position']}") + print() + +# Résultat: +# Nombre d'articles: 3 +# Article 1: Objet de la loi +# Contenu: La présente loi régit les sociétés commerciales... +# Mots: 9, Chars: 58 +# Parent: None, Position: 0 +# ... +``` + +#### Paramètres d'Extraction + +```python +# Paramètres personnalisés +articles = extract_articles( + text=law_text, + min_article_length=10, # Minimum 10 caractères par article + preserve_formatting=False, # Nettoyer espaces multiples + strict=True # Erreur si <3 articles +) + +# Mode non-strict (pour documents incomplets) +articles = extract_articles( + text=draft_law_text, + strict=False # Accepte <3 articles +) +``` + +#### Patterns Supportés + +```python +# ✅ Article X. +"Article 1. Dispositions générales" +"Article 2: Champ d'application" + +# ✅ Art. X +"Art. 1. Titre" +"Art 1: Titre" + +# ✅ Article premier/première +"Article premier. Objet de la loi" +"Article première. Disposition initiale" + +# ✅ Numérotation hiérarchique +"Article 1.1. Sous-article" +"Article 1.2.3. Sous-sous-article" + +# ❌ Non supportés (à venir Phase 2) +"Section I. Titre de section" +"Chapitre II. Titre de chapitre" +"Alinéa 1. Paragraphe" +``` + +#### Intégration Pipeline + +```python +# Dans process_law task +from app.utils.text_chunker import extract_articles, ArticleExtractionError + +def process_law_task(law_id: int): + """Traiter document juridique uploadé.""" + + # 1. Récupérer loi + law = db.query(Law).filter(Law.id == law_id).first() + + # 2. Extraire articles + try: + articles = extract_articles(law.content_fr) + + # Validation + if len(articles) < 3: + raise ValueError(f"Minimum 3 articles requis, {len(articles)} trouvés") + + # 3. Sauvegarder en DB + for article_data in articles: + db_article = Article( + law_id=law_id, + number=article_data['number'], + title=article_data['title'], + content=article_data['content'], + order=article_data['position'], + # parent_id sera ajouté en Phase 2 + ) + db.add(db_article) + + db.commit() + logger.info(f"✅ {len(articles)} articles extraits pour loi #{law_id}") + + except ArticleExtractionError as e: + logger.error(f"❌ Extraction échouée: {e}") + raise +``` + +#### Format de Retour + +Chaque article extrait contient: + +```python +{ + 'number': '1', # VARCHAR(20) - Numéro article + 'title': 'Dispositions générales', # VARCHAR(200) - Titre (optionnel) + 'content': 'La présente loi...', # TEXT - Contenu article + 'position': 0, # INT - Ordre dans document (0-indexed) + 'parent_id': None, # VARCHAR(20) - Parent pour hiérarchie (1.1 → '1') + 'section': None, # VARCHAR(50) - Section/Chapitre (Phase 2) + 'word_count': 45, # INT - Nombre de mots + 'char_count': 234 # INT - Nombre de caractères +} +``` + +#### Tests + +```bash +# Tests unitaires (23 tests de base) +pytest backend/tests/test_utils/test_text_chunker.py -v + +# Classes de tests: +# - TestBasicExtraction: 3 tests (patterns standards) +# - TestHierarchicalNumbering: 2 tests (numérotation imbriquée) +# - TestValidation: 7 tests (validation stricte) +# - TestEdgeCases: 5 tests (cas limites) +# - TestStatistics: 2 tests (compteurs) +# - TestPositioning: 1 test (ordre séquentiel) +# - TestMinimumArticleLength: 2 tests (filtrage longueur) +# - TestMultiplePatterns: 1 test (patterns mixtes) + +# Tests d'intégration (7 tests) +pytest backend/tests/test_integration/test_article_extraction_integration.py -v + +# Avec coverage (cible: >90%) +pytest backend/tests/test_utils/test_text_chunker.py -v \ + --cov=app.utils.text_chunker \ + --cov-report=term-missing + +# Résultat attendu: +# ====== 23 passed, 96% coverage ====== +``` + +#### Validation Rules + +**Input:** +- Minimum: 200 caractères +- Maximum: 5,000,000 caractères (5MB) +- Minimum 3 articles requis (strict=True) + +**Article:** +- Contenu: 10-50,000 caractères +- Numéro: 1-20 caractères +- Titre: 0-200 caractères (optionnel) + +**Errors:** + +```python +# ValueError: Texte invalide +extract_articles("") # → "Le texte ne peut pas être vide" +extract_articles("x" * 50) # → "Texte trop court (50 chars, minimum 200)" + +# ArticleExtractionError: Extraction échouée +extract_articles("Long text without articles...") +# → "Aucun pattern d'article détecté" + +extract_articles("Article 1...\nArticle 2...", strict=True) +# → "Minimum 3 articles requis, 2 trouvés" +``` + +#### Troubleshooting + +**❌ Erreur: "Aucun pattern d'article détecté"** +- Vérifier format: "Article 1." ou "Art. 1" +- Patterns supportés: numérique (1, 2, 3) ou textuel (premier, première) + +**❌ Erreur: "Minimum 3 articles requis"** +```python +# Option 1: Mode non-strict +articles = extract_articles(text, strict=False) + +# Option 2: Ajouter plus d'articles au document +``` + +**❌ Articles trop courts filtrés** +```python +# Réduire min_article_length +articles = extract_articles(text, min_article_length=5) # Default: 10 +``` + +**❌ Titres mal détectés** +- Titres doivent être <100 caractères et sur première ligne +- Format attendu: "Article 1. Titre court\nContenu..." + +**❌ Hiérarchie non reconnue** +- Format requis: numérotation à points (1.1, 1.2.3) +- parent_id extrait automatiquement (1.2.3 → parent='1.2') + +#### Performance + +Benchmarks (Windows 11, Python 3.11): + +| Document | Articles | Temps | Performance | +|----------|----------|-------|-------------| +| Petit (10 articles) | 10 | <50ms | ✅ | +| Moyen (100 articles) | 100 | <100ms | ✅ | +| Large (1000 articles) | 1000 | <500ms | ✅ | + +```bash +# Test de performance +pytest backend/tests/test_integration/test_article_extraction_integration.py::TestArticleExtractionIntegration::test_performance_target_met -v + +# Résultat: +# test_performance_target_met PASSED [100%] +# Extraction took 0.045s (target: <0.5s for 100 articles) ✅ +``` + +## 🚀 Prochaines Étapes + +### Module 2: Classification Automatique (Section 5.6) +Service de catégorisation automatique de documents juridiques. + +### Module 4: Filtre Multilingue (Section 6.9) +Filtrage automatique des lois par langue détectée. + +### Module 5: Pipeline Complet (Section 4.2) +Intégration Celery pour traitement automatique des documents. + +## 📄 License + +Propriétaire - JuriX Team © 2025 + +## 👥 Équipe + +- **Développeurs:** JuriX Dev Team +- **Architecte:** Senior Software Engineer +- **QA:** Qodo AI Test Generation + +## 📞 Support + +Pour questions ou problèmes: +1. Vérifier documentation: `http://localhost:8000/docs` +2. Consulter logs: `docker-compose logs -f api` +3. Lancer health checks: `curl http://localhost:8000/api/v1/language/health` +4. Créer issue sur repository GitHub + +--- + +**Version:** 2.1.0 +**Dernière mise à jour:** 2025-01-05 +**Status:** ✅ LanguageDetector + ArticleSplitter Complets (30/30 tests passing, 96% coverage) diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000000000000000000000000000000000000..91db55772d726c96f553e4893279648bfe03c0f6 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,116 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# URL is set programmatically in alembic/env.py from DATABASE_URL environment variable +# Do NOT hardcode connection strings here — use env vars instead +sqlalchemy.url = placeholder://not-used + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000000000000000000000000000000000000..bb3b87361ac9012a1562e78112662c070c2c9489 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,81 @@ +from logging.config import fileConfig +import os + +from sqlalchemy import engine_from_config, pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +from app.core.database import Base +from app.models import Article, Category, Conversation, Law, Message, User + +target_metadata = Base.metadata + +# Override sqlalchemy.url from environment variable. +# Neon gives postgresql+asyncpg:// URLs — convert to psycopg2-compatible for migrations. +_db_url = os.environ.get("DATABASE_URL", "") +if _db_url: + # Alembic needs a sync driver: swap asyncpg with psycopg2 + _db_url = _db_url.replace("postgresql+asyncpg://", "postgresql://") + _db_url = _db_url.replace("postgres://", "postgresql://") # Neon sometimes gives bare postgres:// + config.set_main_option("sqlalchemy.url", _db_url) + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000000000000000000000000000000000000..55df2863d206fa1678abb4c92e90c45d3f85c114 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/001_initial_schema.py b/alembic/versions/001_initial_schema.py new file mode 100644 index 0000000000000000000000000000000000000000..159cba58c96a03814d1c88cf8a5bca5b39f880a1 --- /dev/null +++ b/alembic/versions/001_initial_schema.py @@ -0,0 +1,97 @@ +"""Initial schema + +Revision ID: 001 +Revises: +Create Date: 2025-01-03 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql +from pgvector.sqlalchemy import Vector + + +# revision identifiers, used by Alembic. +revision = '001' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Categories table + op.create_table( + 'categories', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(100), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + + # Laws table avec colonnes v2.1 + op.create_table( + 'laws', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('reference', sa.String(100), nullable=False, unique=True), + sa.Column('title', sa.String(500), nullable=False), + sa.Column('type', sa.String(50), nullable=False), + sa.Column('category_id', sa.Integer(), nullable=True), + # v2.1 Colonnes détection automatique + sa.Column('language', sa.String(2), nullable=True), # 'fr' | 'en' + sa.Column('language_confidence', sa.Float(), nullable=True), + sa.Column('detected_language', sa.String(2), nullable=True), + sa.Column('suggested_categories', postgresql.ARRAY(sa.Integer()), nullable=True), + sa.Column('category_confidence', sa.Float(), nullable=True), + # Autres colonnes + sa.Column('content', sa.Text(), nullable=False), + sa.Column('publication_date', sa.Date(), nullable=True), + sa.Column('status', sa.String(20), nullable=False, server_default='draft'), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['category_id'], ['categories.id']) + ) + + # Articles table + op.create_table( + 'articles', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('law_id', sa.Integer(), nullable=False), + sa.Column('number', sa.String(20), nullable=False), + sa.Column('title', sa.String(200), nullable=True), + sa.Column('content', sa.Text(), nullable=False), + sa.Column('embedding', Vector(3072), nullable=True), # pgvector 3072-dim Gemini + sa.Column('order', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['law_id'], ['laws.id'], ondelete='CASCADE') + ) + + # Index + op.create_index('idx_laws_language', 'laws', ['language']) + op.create_index('idx_laws_category', 'laws', ['category_id']) + op.create_index('idx_articles_law', 'articles', ['law_id']) + + # Insertion catégories initiales + op.execute(""" + INSERT INTO categories (name, description) VALUES + ('Droit Civil', 'Lois relatives au droit civil'), + ('Droit Pénal', 'Lois relatives au droit pénal'), + ('Droit Commercial', 'Lois relatives au commerce'), + ('Droit du Travail', 'Lois relatives au travail'), + ('Droit Fiscal', 'Lois relatives à la fiscalité'), + ('Droit Administratif', 'Lois relatives à l''administration'), + ('Droit de la Famille', 'Lois relatives à la famille'), + ('Droit des Affaires', 'Lois relatives aux affaires'), + ('Droit Constitutionnel', 'Constitution et lois constitutionnelles'), + ('Procédure Civile', 'Procédures judiciaires civiles'), + ('Procédure Pénale', 'Procédures judiciaires pénales'), + ('Droit OHADA', 'Actes uniformes OHADA') + """) + + +def downgrade() -> None: + op.drop_table('articles') + op.drop_table('laws') + op.drop_table('categories') diff --git a/alembic/versions/002_add_rag_tables.py b/alembic/versions/002_add_rag_tables.py new file mode 100644 index 0000000000000000000000000000000000000000..788ed2fe02fe8fe19494c48791a378ec45f51d81 --- /dev/null +++ b/alembic/versions/002_add_rag_tables.py @@ -0,0 +1,69 @@ +"""Add RAG conversation tables + +Revision ID: 002 +Revises: 001 +Create Date: 2026-01-10 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '002' +down_revision = '001' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Create conversations table + op.create_table( + 'conversations', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('session_id', sa.String(100), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('persona', sa.String(50), nullable=False), + sa.Column('language', sa.String(2), nullable=False, server_default='fr'), + sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.text('now()')), + sa.Column('updated_at', sa.DateTime(), nullable=False, server_default=sa.text('now()')), + sa.PrimaryKeyConstraint('id'), + # Note: Foreign key to users table commented out until users table is implemented + # sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='SET NULL'), + ) + + # Create indexes for conversations + op.create_index('idx_conversations_session_id', 'conversations', ['session_id'], unique=True) + op.create_index('idx_conversations_id', 'conversations', ['id']) + + # Create messages table + op.create_table( + 'messages', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('conversation_id', sa.Integer(), nullable=False), + sa.Column('role', sa.String(20), nullable=False), + sa.Column('content', sa.Text(), nullable=False), + sa.Column('sources', sa.JSON(), nullable=True), + sa.Column('confidence', sa.Float(), nullable=True), + sa.Column('retrieval_time_ms', sa.Integer(), nullable=True), + sa.Column('generation_time_ms', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.text('now()')), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['conversation_id'], ['conversations.id'], ondelete='CASCADE'), + ) + + # Create indexes for messages + op.create_index('idx_messages_conversation_id', 'messages', ['conversation_id']) + op.create_index('idx_messages_id', 'messages', ['id']) + op.create_index('idx_messages_created_at', 'messages', [sa.text('created_at DESC')]) + + +def downgrade() -> None: + op.drop_index('idx_messages_created_at', table_name='messages') + op.drop_index('idx_messages_id', table_name='messages') + op.drop_index('idx_messages_conversation_id', table_name='messages') + op.drop_table('messages') + + op.drop_index('idx_conversations_id', table_name='conversations') + op.drop_index('idx_conversations_session_id', table_name='conversations') + op.drop_table('conversations') diff --git a/alembic/versions/003_add_persona_analytics.py b/alembic/versions/003_add_persona_analytics.py new file mode 100644 index 0000000000000000000000000000000000000000..3e380d83caaaba947cb51b97f72ef0be3787fcb1 --- /dev/null +++ b/alembic/versions/003_add_persona_analytics.py @@ -0,0 +1,130 @@ +"""Add persona analytics tables + +Revision ID: 003 +Revises: 002 +Create Date: 2026-01-11 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '003' +down_revision = '002' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Create persona_stats table - Aggregated daily statistics + op.create_table( + 'persona_stats', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('persona', sa.String(50), nullable=False), + sa.Column('date', sa.Date(), nullable=False), + + # Usage Metrics + sa.Column('total_questions', sa.Integer(), nullable=False, server_default='0'), + sa.Column('total_conversations', sa.Integer(), nullable=False, server_default='0'), + sa.Column('unique_sessions', sa.Integer(), nullable=False, server_default='0'), + + # Performance Metrics + sa.Column('avg_confidence', sa.Float(), nullable=False, server_default='0.0'), + sa.Column('avg_retrieval_time_ms', sa.Integer(), nullable=False, server_default='0'), + sa.Column('avg_generation_time_ms', sa.Integer(), nullable=False, server_default='0'), + sa.Column('avg_total_time_ms', sa.Integer(), nullable=False, server_default='0'), + + # Engagement Metrics + sa.Column('avg_messages_per_conversation', sa.Float(), nullable=False, server_default='0.0'), + sa.Column('avg_session_duration_seconds', sa.Integer(), nullable=False, server_default='0'), + + # Quality Metrics + sa.Column('helpful_count', sa.Integer(), nullable=False, server_default='0'), + sa.Column('unhelpful_count', sa.Integer(), nullable=False, server_default='0'), + sa.Column('satisfaction_rate', sa.Float(), nullable=False, server_default='0.0'), + + # Metadata + sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.text('now()')), + sa.Column('updated_at', sa.DateTime(), nullable=False, server_default=sa.text('now()')), + + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('persona', 'date', name='uq_persona_date') + ) + + # Create indexes for persona_stats + op.create_index('idx_persona_stats_persona_date', 'persona_stats', ['persona', sa.text('date DESC')]) + op.create_index('idx_persona_stats_date', 'persona_stats', [sa.text('date DESC')]) + + # Create message_feedback table - User ratings on responses + op.create_table( + 'message_feedback', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('message_id', sa.Integer(), nullable=False), + sa.Column('helpful', sa.Boolean(), nullable=False), + sa.Column('rating', sa.Integer(), nullable=True), + sa.Column('comment', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.text('now()')), + + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['message_id'], ['messages.id'], ondelete='CASCADE'), + sa.UniqueConstraint('message_id', name='uq_message_feedback_message_id'), + sa.CheckConstraint('rating >= 1 AND rating <= 5', name='ck_rating_range') + ) + + # Create indexes for message_feedback + op.create_index('idx_message_feedback_message_id_helpful', 'message_feedback', ['message_id', 'helpful']) + op.create_index('idx_message_feedback_created_at', 'message_feedback', [sa.text('created_at DESC')]) + op.create_index('idx_message_feedback_helpful', 'message_feedback', ['helpful']) + + # Create persona_interactions table - Per-session analytics + op.create_table( + 'persona_interactions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('conversation_id', sa.Integer(), nullable=False), + sa.Column('persona', sa.String(50), nullable=False), + sa.Column('language', sa.String(2), nullable=False), + + # Interaction Counts + sa.Column('question_count', sa.Integer(), nullable=False, server_default='0'), + sa.Column('answer_count', sa.Integer(), nullable=False, server_default='0'), + + # Performance + sa.Column('avg_confidence', sa.Float(), nullable=False, server_default='0.0'), + sa.Column('total_response_time_ms', sa.Integer(), nullable=False, server_default='0'), + + # Persona Behavior + sa.Column('personas_used', sa.JSON(), nullable=True), + sa.Column('persona_switches', sa.Integer(), nullable=False, server_default='0'), + + # Metadata + sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.text('now()')), + sa.Column('updated_at', sa.DateTime(), nullable=False, server_default=sa.text('now()')), + + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['conversation_id'], ['conversations.id'], ondelete='CASCADE') + ) + + # Create indexes for persona_interactions + op.create_index('idx_persona_interactions_persona_created', 'persona_interactions', ['persona', sa.text('created_at DESC')]) + op.create_index('idx_persona_interactions_conversation_id', 'persona_interactions', ['conversation_id']) + op.create_index('idx_persona_interactions_created_at', 'persona_interactions', [sa.text('created_at DESC')]) + + +def downgrade() -> None: + # Drop persona_interactions table + op.drop_index('idx_persona_interactions_created_at', table_name='persona_interactions') + op.drop_index('idx_persona_interactions_conversation_id', table_name='persona_interactions') + op.drop_index('idx_persona_interactions_persona_created', table_name='persona_interactions') + op.drop_table('persona_interactions') + + # Drop message_feedback table + op.drop_index('idx_message_feedback_helpful', table_name='message_feedback') + op.drop_index('idx_message_feedback_created_at', table_name='message_feedback') + op.drop_index('idx_message_feedback_message_id_helpful', table_name='message_feedback') + op.drop_table('message_feedback') + + # Drop persona_stats table + op.drop_index('idx_persona_stats_date', table_name='persona_stats') + op.drop_index('idx_persona_stats_persona_date', table_name='persona_stats') + op.drop_table('persona_stats') diff --git a/alembic/versions/004_add_processing_status.py b/alembic/versions/004_add_processing_status.py new file mode 100644 index 0000000000000000000000000000000000000000..b06d328b3844e2a7aeef8f1130cba9ed293420de --- /dev/null +++ b/alembic/versions/004_add_processing_status.py @@ -0,0 +1,36 @@ +"""Add processing status fields to laws table + +Revision ID: 004 +Revises: 003 +Create Date: 2026-01-13 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '004' +down_revision = '003' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Add processing status tracking fields to laws table + op.add_column('laws', sa.Column('processing_progress', sa.Integer(), nullable=True)) + op.add_column('laws', sa.Column('processing_error', sa.Text(), nullable=True)) + op.add_column('laws', sa.Column('processing_started_at', sa.DateTime(), nullable=True)) + + # Create index for processing_progress for efficient filtering + op.create_index('idx_laws_processing_progress', 'laws', ['processing_progress']) + + +def downgrade() -> None: + # Drop index + op.drop_index('idx_laws_processing_progress', table_name='laws') + + # Drop columns + op.drop_column('laws', 'processing_started_at') + op.drop_column('laws', 'processing_error') + op.drop_column('laws', 'processing_progress') diff --git a/alembic/versions/005_add_category_icon.py b/alembic/versions/005_add_category_icon.py new file mode 100644 index 0000000000000000000000000000000000000000..0135ffef56d6492b76713b8647803015f7a6ae51 --- /dev/null +++ b/alembic/versions/005_add_category_icon.py @@ -0,0 +1,26 @@ +"""Add icon field to categories table + +Revision ID: 005 +Revises: 1e5feee465ba +Create Date: 2026-01-13 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '005' +down_revision = '1e5feee465ba' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Add icon field to categories table + op.add_column('categories', sa.Column('icon', sa.String(10), nullable=True)) + + +def downgrade() -> None: + # Drop icon column + op.drop_column('categories', 'icon') diff --git a/alembic/versions/006_add_display_order.py b/alembic/versions/006_add_display_order.py new file mode 100644 index 0000000000000000000000000000000000000000..21ba72511932d69d17dfd9346a5fca56149882eb --- /dev/null +++ b/alembic/versions/006_add_display_order.py @@ -0,0 +1,27 @@ +"""Add display_order to categories + +Revision ID: 006 +Revises: 005 +Create Date: 2026-01-14 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '006' +down_revision = '005' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column('categories', sa.Column('display_order', sa.Integer(), nullable=True, server_default='0')) + # Make it non-nullable after setting defaults + op.execute("UPDATE categories SET display_order = id WHERE display_order IS NULL") + op.alter_column('categories', 'display_order', nullable=False) + + +def downgrade() -> None: + op.drop_column('categories', 'display_order') diff --git a/alembic/versions/0b21fb6a3651_add_user_model_and_update_relationships.py b/alembic/versions/0b21fb6a3651_add_user_model_and_update_relationships.py new file mode 100644 index 0000000000000000000000000000000000000000..192ad2eeddd00f7e8862efd6d0594d6df21fa04c --- /dev/null +++ b/alembic/versions/0b21fb6a3651_add_user_model_and_update_relationships.py @@ -0,0 +1,141 @@ +"""Add User model and update relationships + +Revision ID: 0b21fb6a3651 +Revises: 003 +Create Date: 2026-01-12 07:03:08.986151 + +""" + +import pgvector.sqlalchemy +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "0b21fb6a3651" +down_revision = "003" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "users", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("email", sa.String(length=255), nullable=False), + sa.Column("username", sa.String(length=100), nullable=False), + sa.Column("hashed_password", sa.String(length=255), nullable=False), + sa.Column("full_name", sa.String(length=255), nullable=True), + sa.Column( + "role", sa.String(length=50), nullable=False, comment="Role: user, admin, superadmin" + ), + sa.Column("is_active", sa.Boolean(), nullable=False), + sa.Column("is_verified", sa.Boolean(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.Column("last_login_at", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_users_email"), "users", ["email"], unique=True) + op.create_index(op.f("ix_users_id"), "users", ["id"], unique=False) + op.create_index(op.f("ix_users_role"), "users", ["role"], unique=False) + op.create_index(op.f("ix_users_username"), "users", ["username"], unique=True) + # Keep embedding as VECTOR type (don't convert to JSON) + op.create_index(op.f("ix_articles_id"), "articles", ["id"], unique=False) + op.create_index(op.f("ix_articles_law_id"), "articles", ["law_id"], unique=False) + op.create_index(op.f("ix_categories_id"), "categories", ["id"], unique=False) + op.drop_index(op.f("idx_conversations_id"), table_name="conversations") + op.drop_index(op.f("idx_conversations_session_id"), table_name="conversations") + op.create_index("idx_conversations_session_id", "conversations", ["session_id"], unique=False) + op.create_index(op.f("ix_conversations_id"), "conversations", ["id"], unique=False) + op.create_index( + op.f("ix_conversations_session_id"), "conversations", ["session_id"], unique=True + ) + op.create_index(op.f("ix_conversations_user_id"), "conversations", ["user_id"], unique=False) + op.create_foreign_key(None, "conversations", "users", ["user_id"], ["id"], ondelete="SET NULL") + # Keep suggested_categories as ARRAY type (don't convert to JSON) + op.drop_constraint(op.f("laws_reference_key"), "laws", type_="unique") + op.create_index("idx_laws_reference", "laws", ["reference"], unique=False) + op.create_index("idx_laws_status", "laws", ["status"], unique=False) + op.create_index(op.f("ix_laws_category_id"), "laws", ["category_id"], unique=False) + op.create_index(op.f("ix_laws_id"), "laws", ["id"], unique=False) + op.create_index(op.f("ix_laws_language"), "laws", ["language"], unique=False) + op.create_index(op.f("ix_laws_reference"), "laws", ["reference"], unique=True) + op.create_index(op.f("ix_laws_status"), "laws", ["status"], unique=False) + op.create_index(op.f("ix_message_feedback_id"), "message_feedback", ["id"], unique=False) + op.drop_index(op.f("idx_messages_id"), table_name="messages") + op.create_index( + op.f("ix_messages_conversation_id"), "messages", ["conversation_id"], unique=False + ) + op.create_index(op.f("ix_messages_created_at"), "messages", ["created_at"], unique=False) + op.create_index(op.f("ix_messages_id"), "messages", ["id"], unique=False) + op.create_index( + op.f("ix_persona_interactions_conversation_id"), + "persona_interactions", + ["conversation_id"], + unique=False, + ) + op.create_index( + op.f("ix_persona_interactions_id"), "persona_interactions", ["id"], unique=False + ) + op.create_index(op.f("ix_persona_stats_id"), "persona_stats", ["id"], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_persona_stats_id"), table_name="persona_stats") + op.drop_index(op.f("ix_persona_interactions_id"), table_name="persona_interactions") + op.drop_index( + op.f("ix_persona_interactions_conversation_id"), table_name="persona_interactions" + ) + op.drop_index(op.f("ix_messages_id"), table_name="messages") + op.drop_index(op.f("ix_messages_created_at"), table_name="messages") + op.drop_index(op.f("ix_messages_conversation_id"), table_name="messages") + op.create_index(op.f("idx_messages_id"), "messages", ["id"], unique=False) + op.drop_index(op.f("ix_message_feedback_id"), table_name="message_feedback") + op.drop_index(op.f("ix_laws_status"), table_name="laws") + op.drop_index(op.f("ix_laws_reference"), table_name="laws") + op.drop_index(op.f("ix_laws_language"), table_name="laws") + op.drop_index(op.f("ix_laws_id"), table_name="laws") + op.drop_index(op.f("ix_laws_category_id"), table_name="laws") + op.drop_index("idx_laws_status", table_name="laws") + op.drop_index("idx_laws_reference", table_name="laws") + op.create_unique_constraint( + op.f("laws_reference_key"), "laws", ["reference"], postgresql_nulls_not_distinct=False + ) + op.alter_column( + "laws", + "suggested_categories", + existing_type=sa.JSON(), + type_=postgresql.ARRAY(sa.INTEGER()), + existing_nullable=True, + ) + op.drop_constraint(None, "conversations", type_="foreignkey") + op.drop_index(op.f("ix_conversations_user_id"), table_name="conversations") + op.drop_index(op.f("ix_conversations_session_id"), table_name="conversations") + op.drop_index(op.f("ix_conversations_id"), table_name="conversations") + op.drop_index("idx_conversations_session_id", table_name="conversations") + op.create_index( + op.f("idx_conversations_session_id"), "conversations", ["session_id"], unique=True + ) + op.create_index(op.f("idx_conversations_id"), "conversations", ["id"], unique=False) + op.drop_index(op.f("ix_categories_id"), table_name="categories") + op.drop_index(op.f("ix_articles_law_id"), table_name="articles") + op.drop_index(op.f("ix_articles_id"), table_name="articles") + op.alter_column( + "articles", + "embedding", + existing_type=sa.JSON(), + type_=pgvector.sqlalchemy.vector.VECTOR(dim=768), + existing_nullable=True, + ) + op.drop_index(op.f("ix_users_username"), table_name="users") + op.drop_index(op.f("ix_users_role"), table_name="users") + op.drop_index(op.f("ix_users_id"), table_name="users") + op.drop_index(op.f("ix_users_email"), table_name="users") + op.drop_table("users") + op.drop_table("users") + # ### end Alembic commands ### diff --git a/alembic/versions/1e5feee465ba_merge_heads.py b/alembic/versions/1e5feee465ba_merge_heads.py new file mode 100644 index 0000000000000000000000000000000000000000..019131e95687fade94bcfb17bfde76b2e8a2e43f --- /dev/null +++ b/alembic/versions/1e5feee465ba_merge_heads.py @@ -0,0 +1,24 @@ +"""merge_heads + +Revision ID: 1e5feee465ba +Revises: 004, 0b21fb6a3651 +Create Date: 2026-01-13 16:01:59.279535 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '1e5feee465ba' +down_revision = ('004', '0b21fb6a3651') +branch_labels = None +depends_on = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/alembic/versions/3f0760423953_add_file_tracking_real.py b/alembic/versions/3f0760423953_add_file_tracking_real.py new file mode 100644 index 0000000000000000000000000000000000000000..49604b7c8ae35fd0d0b817a712e00fbe2b70a6d3 --- /dev/null +++ b/alembic/versions/3f0760423953_add_file_tracking_real.py @@ -0,0 +1,62 @@ +"""add_file_tracking_real + +Revision ID: 3f0760423953 +Revises: 006 +Create Date: 2026-01-15 04:34:38.060616 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +"""add_file_tracking_real + +Revision ID: 3f0760423953 +Revises: 006 +Create Date: 2026-01-15 04:34:38.060616 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '3f0760423953' +down_revision = '006' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # op.alter_column('articles', 'embedding', + # existing_type=pgvector.sqlalchemy.vector.VECTOR(dim=768), + # type_=sa.JSON(), + # existing_nullable=True) + op.add_column('laws', sa.Column('file_id', sa.String(length=100), nullable=True)) + op.add_column('laws', sa.Column('original_filename', sa.String(length=500), nullable=True)) + # op.alter_column('laws', 'suggested_categories', + # existing_type=postgresql.ARRAY(sa.INTEGER()), + # type_=sa.JSON(), + # existing_nullable=True) + # op.drop_index(op.f('idx_laws_processing_progress'), table_name='laws') + op.create_index(op.f('ix_laws_file_id'), 'laws', ['file_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_laws_file_id'), table_name='laws') + # op.create_index(op.f('idx_laws_processing_progress'), 'laws', ['processing_progress'], unique=False) + # op.alter_column('laws', 'suggested_categories', + # existing_type=sa.JSON(), + # type_=postgresql.ARRAY(sa.INTEGER()), + # existing_nullable=True) + op.drop_column('laws', 'original_filename') + op.drop_column('laws', 'file_id') + # op.alter_column('articles', 'embedding', + # existing_type=sa.JSON(), + # type_=pgvector.sqlalchemy.vector.VECTOR(dim=768), + # existing_nullable=True) + # ### end Alembic commands ### diff --git a/alembic/versions/668e3740d3b5_add_page_number_to_articles.py b/alembic/versions/668e3740d3b5_add_page_number_to_articles.py new file mode 100644 index 0000000000000000000000000000000000000000..de3602e48b3d39c7cdc10710105c35fb95235c31 --- /dev/null +++ b/alembic/versions/668e3740d3b5_add_page_number_to_articles.py @@ -0,0 +1,24 @@ +"""Add page_number to articles + +Revision ID: 668e3740d3b5 +Revises: 3f0760423953 +Create Date: 2026-01-21 03:44:42.433227 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '668e3740d3b5' +down_revision = '3f0760423953' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Add page_number column to articles table + op.add_column('articles', sa.Column('page_number', sa.Integer(), nullable=True)) + + +def downgrade() -> None: + op.drop_column('articles', 'page_number') diff --git a/alembic/versions/7a8b9c0d1e2f_enable_pgvector.py b/alembic/versions/7a8b9c0d1e2f_enable_pgvector.py new file mode 100644 index 0000000000000000000000000000000000000000..4e5c67a471134bcfaacdb4d37d09c5225aa66b7a --- /dev/null +++ b/alembic/versions/7a8b9c0d1e2f_enable_pgvector.py @@ -0,0 +1,65 @@ +"""Enable pgvector and migrate embeddings column + +Revision ID: 7a8b9c0d1e2f +Revises: 668e3740d3b5 +Create Date: 2026-01-27 13:10:00.000000 + +""" +from alembic import op +import sqlalchemy as sa +from pgvector.sqlalchemy import Vector + +# revision identifiers, used by Alembic. +revision = '7a8b9c0d1e2f' +down_revision = '668e3740d3b5' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """ + Enable pgvector extension and migrate embedding column. + + Steps: + 1. Enable pgvector extension + 2. Convert embedding column to Vector(3072) for Gemini embeddings + + Note: No vector index is created because 3072 dimensions exceeds + pgvector's 2000-dimension limit for HNSW/IVFFlat indexes. + Sequential scan is acceptable for <20k articles. + """ + # Step 1: Enable pgvector extension + op.execute('CREATE EXTENSION IF NOT EXISTS vector') + + # Step 2: Convert embedding column type to vector(3072) + # 3072 dimensions = native Gemini gemini-embedding-001 output + op.execute(""" + ALTER TABLE articles + ALTER COLUMN embedding + TYPE vector(3072) + USING NULL::vector(3072) + """) + + +def downgrade() -> None: + """ + Rollback pgvector changes. + + Steps: + 1. Drop HNSW index + 2. Convert embedding column back to JSON + 3. Drop pgvector extension (if no other tables use it) + """ + # Step 1: Drop index + op.execute('DROP INDEX IF EXISTS idx_articles_embedding_cosine') + + # Step 2: Convert back to JSON + op.execute(""" + ALTER TABLE articles + ALTER COLUMN embedding + TYPE json + USING NULL::json + """) + + # Step 3: Drop extension (commented out for safety - may affect other tables) + # op.execute('DROP EXTENSION IF EXISTS vector') diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/app/api/dependencies/__init__.py b/app/api/dependencies/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fbc45ac065fc6553579f65f10925db1aae436dff --- /dev/null +++ b/app/api/dependencies/__init__.py @@ -0,0 +1,5 @@ +"""API dependencies.""" + +from app.core.database import get_db + +__all__ = ["get_db"] diff --git a/app/api/routes/__init__.py b/app/api/routes/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/app/api/routes/admin.py b/app/api/routes/admin.py new file mode 100644 index 0000000000000000000000000000000000000000..0adcd0b100023ceff389c9f0f4da6e27d279368a --- /dev/null +++ b/app/api/routes/admin.py @@ -0,0 +1,335 @@ +""" +API routes for Admin operations. + +Endpoints: +- GET /api/v1/admin/users - List users +- POST /api/v1/admin/users - Create user +- PUT /api/v1/admin/users/{id} - Update user +- DELETE /api/v1/admin/users/{id} - Delete user +- GET /api/v1/admin/system - System information + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +import logging +import platform +from datetime import datetime +from typing import Dict, List + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel, EmailStr, Field +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.database import get_db, health_check_db + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/v1/admin", tags=["Admin"]) + + +# ==================== SCHEMAS ==================== + + +class UserCreate(BaseModel): + """Schema for creating a user.""" + + email: EmailStr = Field(..., description="User email") + username: str = Field(..., min_length=3, max_length=50, description="Username") + role: str = Field(default="user", description="User role (user or admin)") + is_active: bool = Field(default=True, description="User active status") + + +class UserUpdate(BaseModel): + """Schema for updating a user.""" + + email: EmailStr | None = None + username: str | None = Field(None, min_length=3, max_length=50) + role: str | None = None + is_active: bool | None = None + + +class UserResponse(BaseModel): + """Schema for user response.""" + + id: int + email: str + username: str + role: str + is_active: bool + created_at: datetime + + +# ==================== MOCK USER STORAGE ==================== +# TODO: Replace with actual database model + + +_mock_users = [ + { + "id": 1, + "email": "admin@jurix.cm", + "username": "admin", + "role": "admin", + "is_active": True, + "created_at": datetime.now(), + }, + { + "id": 2, + "email": "user@jurix.cm", + "username": "user1", + "role": "user", + "is_active": True, + "created_at": datetime.now(), + }, +] + +_next_user_id = 3 + + +# ==================== AUTHENTICATION ==================== + + +async def get_current_admin(): + """ + Dependency for admin authentication. + + TODO: Implement actual JWT authentication. + For now, returns mock admin user. + """ + return {"id": 1, "role": "admin", "username": "admin"} + + +# ==================== USER MANAGEMENT ==================== + + +@router.get("/users", response_model=List[UserResponse]) +async def list_users( + skip: int = 0, + limit: int = 100, + current_admin: dict = Depends(get_current_admin), +) -> List[UserResponse]: + """ + List all users (admin only). + + **Parameters:** + - skip: Number of records to skip + - limit: Maximum records to return + + **Returns:** + - List of users + + **Requires:** Admin authentication + """ + logger.info(f"📋 GET /admin/users - skip={skip}, limit={limit}") + + users = _mock_users[skip : skip + limit] + return users + + +@router.post("/users", response_model=UserResponse, status_code=status.HTTP_201_CREATED) +async def create_user( + user: UserCreate, + current_admin: dict = Depends(get_current_admin), +) -> UserResponse: + """ + Create a new user (admin only). + + **Parameters:** + - email: User email (unique) + - username: Username (unique) + - role: User role (user or admin) + - is_active: Active status + + **Returns:** + - Created user + + **Raises:** + - 409: Email or username already exists + + **Requires:** Admin authentication + """ + global _next_user_id + + assert user is not None, "UserCreate must not be None" + assert isinstance(user.email, str) and len(user.email) > 0, "Email must be a non-empty string" + assert isinstance(user.username, str) and len(user.username) >= 3, "Username must be at least 3 characters" + + logger.info(f"➕ POST /admin/users - Creating user: {user.email}") + + # Check for duplicate email or username + for existing_user in _mock_users: + if existing_user["email"] == user.email: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Email already exists: {user.email}", + ) + if existing_user["username"] == user.username: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Username already exists: {user.username}", + ) + + # Create new user + new_user = { + "id": _next_user_id, + "email": user.email, + "username": user.username, + "role": user.role, + "is_active": user.is_active, + "created_at": datetime.now(), + } + + _mock_users.append(new_user) + _next_user_id += 1 + + logger.info(f"✅ User created: ID={new_user['id']}, email={new_user['email']}") + return new_user + + +@router.put("/users/{user_id}", response_model=UserResponse) +async def update_user( + user_id: int, + user_update: UserUpdate, + current_admin: dict = Depends(get_current_admin), +) -> UserResponse: + """ + Update a user (admin only). + + **Parameters:** + - user_id: User ID + - Partial update fields + + **Returns:** + - Updated user + + **Raises:** + - 404: User not found + + **Requires:** Admin authentication + """ + assert isinstance(user_id, int) and user_id > 0, "user_id must be a positive integer" + assert user_update is not None, "UserUpdate must not be None" + + logger.info(f"✏️ PUT /admin/users/{user_id}") + + # Find user + user = next((u for u in _mock_users if u["id"] == user_id), None) + + if not user: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"User not found: {user_id}", + ) + + # Update fields + update_data = user_update.model_dump(exclude_unset=True) + for field, value in update_data.items(): + user[field] = value + + logger.info(f"✅ User {user_id} updated") + return user + + +@router.delete("/users/{user_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_user( + user_id: int, + current_admin: dict = Depends(get_current_admin), +): + """ + Delete a user (admin only). + + **Parameters:** + - user_id: User ID + + **Returns:** + - 204 No Content + + **Raises:** + - 404: User not found + + **Requires:** Admin authentication + """ + assert isinstance(user_id, int) and user_id > 0, "user_id must be a positive integer" + + logger.info(f"🗑️ DELETE /admin/users/{user_id}") + + # Find user + user = next((u for u in _mock_users if u["id"] == user_id), None) + + if not user: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"User not found: {user_id}", + ) + + # Remove user + _mock_users.remove(user) + + logger.info(f"✅ User {user_id} deleted") + return None + + +# ==================== SYSTEM INFORMATION ==================== + + +@router.get("/system") +async def get_system_info( + db: AsyncSession = Depends(get_db), + current_admin: dict = Depends(get_current_admin), +) -> Dict: + """ + Get system information (admin only). + + **Returns:** + - Database status + - System info + - Service health + - Version info + + **Requires:** Admin authentication + """ + logger.info("🖥️ GET /admin/system") + + # Check database health + db_healthy = await health_check_db() + + return { + "system": { + "platform": platform.system(), + "platform_version": platform.version(), + "python_version": platform.python_version(), + "hostname": platform.node(), + }, + "database": { + "status": "healthy" if db_healthy else "unhealthy", + "connected": db_healthy, + }, + "services": { + "api": "running", + "search": "running", + "rag": "running", + }, + "version": { + "api": "2.1.0", + "build": "production", + }, + "timestamp": datetime.now().isoformat(), + } + + +# ==================== HEALTH CHECK ==================== + + +@router.get("/health") +async def health_check() -> Dict: + """ + Check admin service health. + + **Returns:** + - Service status + """ + return { + "service": "Admin", + "status": "healthy", + "timestamp": datetime.now().isoformat(), + } diff --git a/app/api/routes/analytics.py b/app/api/routes/analytics.py new file mode 100644 index 0000000000000000000000000000000000000000..a04f20cdc7ff6d32a6fc1094e0f43e02388e6f9e --- /dev/null +++ b/app/api/routes/analytics.py @@ -0,0 +1,291 @@ +""" +API routes for Analytics and Statistics. + +Endpoints: +- GET /api/v1/analytics/overview - Dashboard overview +- GET /api/v1/analytics/laws - Law statistics +- GET /api/v1/analytics/search - Search analytics +- GET /api/v1/analytics/usage - Usage metrics + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +import logging +from datetime import datetime, timedelta +from typing import Dict, List + +from fastapi import APIRouter, Depends, Query +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.database import get_db +from app.models.law import Law + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/v1/analytics", tags=["Analytics"]) + + +# ==================== DASHBOARD OVERVIEW ==================== + + +@router.get("/overview") +async def get_overview( + db: AsyncSession = Depends(get_db), +) -> Dict: + """ + Get dashboard overview with key metrics. + + **Returns:** + - Total laws count + - Laws by language + - Laws by status + - Recent activity + + **Example Response:** + ```json + { + "total_laws": 150, + "by_language": {"fr": 120, "en": 30}, + "by_status": {"active": 145, "archived": 5}, + "recent_laws": 10 + } + ``` + """ + logger.info("📊 GET /analytics/overview") + + try: + # Total laws + total_query = select(func.count(Law.id)) + total_result = await db.execute(total_query) + total_laws = total_result.scalar() + + # Laws by language + lang_query = select( + Law.language, + func.count(Law.id) + ).group_by(Law.language) + lang_result = await db.execute(lang_query) + by_language = {row[0]: row[1] for row in lang_result.all()} + + # Laws by status + status_query = select( + Law.status, + func.count(Law.id) + ).group_by(Law.status) + status_result = await db.execute(status_query) + by_status = {row[0]: row[1] for row in status_result.all()} + + # Recent laws (last 30 days) + thirty_days_ago = datetime.now() - timedelta(days=30) + recent_query = select(func.count(Law.id)).where( + Law.created_at >= thirty_days_ago + ) + recent_result = await db.execute(recent_query) + recent_laws = recent_result.scalar() + + return { + "total_laws": total_laws or 0, + "by_language": by_language, + "by_status": by_status, + "recent_laws": recent_laws or 0, + "timestamp": datetime.now().isoformat(), + } + + except Exception as e: + logger.error(f"❌ Error getting overview: {e}", exc_info=True) + return { + "total_laws": 0, + "by_language": {}, + "by_status": {}, + "recent_laws": 0, + "error": str(e), + } + + +# ==================== LAW STATISTICS ==================== + + +@router.get("/laws") +async def get_law_statistics( + db: AsyncSession = Depends(get_db), +) -> Dict: + """ + Get detailed law statistics. + + **Returns:** + - Laws by language (FR/EN) + - Laws by type (loi, décret, etc.) + - Average content length + - Publication trends + + **Example Response:** + ```json + { + "by_language": {"fr": 120, "en": 30}, + "by_type": {"loi": 80, "décret": 50, "ordonnance": 20}, + "avg_content_length": 5000, + "total": 150 + } + ``` + """ + logger.info("📊 GET /analytics/laws") + + try: + # Laws by language + lang_query = select( + Law.language, + func.count(Law.id) + ).group_by(Law.language) + lang_result = await db.execute(lang_query) + by_language = {row[0]: row[1] for row in lang_result.all()} + + # Laws by type + type_query = select( + Law.type, + func.count(Law.id) + ).group_by(Law.type) + type_result = await db.execute(type_query) + by_type = {row[0]: row[1] for row in type_result.all()} + + # Average content length + avg_query = select(func.avg(func.length(Law.content))) + avg_result = await db.execute(avg_query) + avg_length = avg_result.scalar() or 0 + + # Total + total_query = select(func.count(Law.id)) + total_result = await db.execute(total_query) + total = total_result.scalar() + + return { + "by_language": by_language, + "by_type": by_type, + "avg_content_length": int(avg_length), + "total": total or 0, + "timestamp": datetime.now().isoformat(), + } + + except Exception as e: + logger.error(f"❌ Error getting law statistics: {e}", exc_info=True) + return { + "by_language": {}, + "by_type": {}, + "avg_content_length": 0, + "total": 0, + "error": str(e), + } + + +# ==================== SEARCH ANALYTICS ==================== + + +@router.get("/search") +async def get_search_analytics() -> Dict: + """ + Get search analytics. + + **Returns:** + - Search modes usage (text/semantic/hybrid) + - Average response time + - Popular queries (mock data for now) + + **Note:** This is a simplified version. In production, + integrate with actual search logs. + + **Example Response:** + ```json + { + "modes_usage": { + "text": 100, + "semantic": 50, + "hybrid": 200 + }, + "avg_response_time_ms": 150, + "total_searches": 350 + } + ``` + """ + logger.info("📊 GET /analytics/search") + + # TODO: Integrate with actual search logs + # For now, return mock data + return { + "modes_usage": { + "text": 100, + "semantic": 50, + "hybrid": 200, + }, + "avg_response_time_ms": 150, + "total_searches": 350, + "timestamp": datetime.now().isoformat(), + "note": "Mock data - integrate with search logs in production", + } + + +# ==================== USAGE METRICS ==================== + + +@router.get("/usage") +async def get_usage_metrics() -> Dict: + """ + Get API usage metrics. + + **Returns:** + - API calls per endpoint + - Peak usage times + - Active users (mock data for now) + + **Note:** This is a simplified version. In production, + integrate with API gateway logs or monitoring system. + + **Example Response:** + ```json + { + "api_calls": { + "/api/v1/laws": 500, + "/api/v1/search": 350, + "/api/v1/rag": 200 + }, + "total_calls": 1050, + "active_users": 25 + } + ``` + """ + logger.info("📊 GET /analytics/usage") + + # TODO: Integrate with API gateway logs or monitoring + # For now, return mock data + return { + "api_calls": { + "/api/v1/laws": 500, + "/api/v1/search": 350, + "/api/v1/rag": 200, + "/api/v1/upload": 100, + }, + "total_calls": 1150, + "active_users": 25, + "peak_hours": [9, 10, 14, 15, 16], + "timestamp": datetime.now().isoformat(), + "note": "Mock data - integrate with monitoring system in production", + } + + +# ==================== HEALTH CHECK ==================== + + +@router.get("/health") +async def health_check() -> Dict: + """ + Check analytics service health. + + **Returns:** + - Service status + - Database connectivity + """ + return { + "service": "Analytics", + "status": "healthy", + "timestamp": datetime.now().isoformat(), + } diff --git a/app/api/routes/articles.py b/app/api/routes/articles.py new file mode 100644 index 0000000000000000000000000000000000000000..32f14e7ef5264bf6606a5535116fed4b8225d370 --- /dev/null +++ b/app/api/routes/articles.py @@ -0,0 +1,167 @@ +""" +Article extraction API endpoint (utility). + +For testing/debugging article extraction on arbitrary text. +""" + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field +from typing import List, Dict, Any +import time + +from app.utils.text_chunker import extract_articles, ArticleExtractionError + + +router = APIRouter(prefix="/api/v1/articles", tags=["articles"]) + + +class ExtractRequest(BaseModel): + """Request model for article extraction.""" + text: str = Field(..., min_length=200, max_length=1_000_000, description="Legal document text") + min_article_length: int = Field(default=10, ge=5, le=1000, description="Minimum article length in characters") + preserve_formatting: bool = Field(default=False, description="Preserve original formatting") + strict: bool = Field(default=True, description="Strict validation (require minimum 3 articles)") + + class Config: + json_schema_extra = { + "example": { + "text": "Article 1. Dispositions générales\nLa présente loi régit...\n\nArticle 2. Champ d'application\nLes dispositions s'appliquent...\n\nArticle 3. Définitions\nAu sens de la présente loi...", + "min_article_length": 10, + "preserve_formatting": False, + "strict": True + } + } + + +class ArticleResult(BaseModel): + """Single extracted article.""" + number: str + title: str | None + content: str + position: int + parent_id: str | None + section: str | None + word_count: int + char_count: int + + +class ExtractResponse(BaseModel): + """Response model for article extraction.""" + articles: List[Dict[str, Any]] + count: int + processing_time_ms: int + + class Config: + json_schema_extra = { + "example": { + "articles": [ + { + "number": "1", + "title": "Dispositions générales", + "content": "La présente loi régit...", + "position": 0, + "parent_id": None, + "section": None, + "word_count": 45, + "char_count": 234 + } + ], + "count": 3, + "processing_time_ms": 125 + } + } + + +class HealthResponse(BaseModel): + """Health check response.""" + service: str + status: str + version: str + + +@router.post("/extract", response_model=ExtractResponse, summary="Extract articles from legal text") +async def extract_articles_endpoint(request: ExtractRequest) -> ExtractResponse: + """ + Extract articles from legal document text. + + This utility endpoint allows testing/debugging of article extraction + on arbitrary legal text without uploading full documents. + + **Supported Patterns:** + - Article X. / Article X: + - Art. X / Art X. + - Article premier / Article première + - Hierarchical: Article 1.1, Article 1.2.3 + + **Validation:** + - Text length: 200 characters - 1MB + - Minimum 3 articles (if strict=True) + - Articles must have minimum length + + **Returns:** + - List of extracted articles with metadata + - Processing time in milliseconds + - Total article count + """ + assert request is not None, "Extract request must not be None" + assert isinstance(request.text, str) and len(request.text) >= 200, "Text must be at least 200 characters" + + try: + start = time.time() + + articles = extract_articles( + text=request.text, + min_article_length=request.min_article_length, + preserve_formatting=request.preserve_formatting, + strict=request.strict + ) + + elapsed_ms = int((time.time() - start) * 1000) + + return ExtractResponse( + articles=articles, + count=len(articles), + processing_time_ms=elapsed_ms + ) + + except ValueError as e: + raise HTTPException( + status_code=400, + detail={ + "error": "Invalid input", + "message": str(e), + "type": "ValueError" + } + ) + except ArticleExtractionError as e: + raise HTTPException( + status_code=422, + detail={ + "error": "Extraction failed", + "message": str(e), + "type": "ArticleExtractionError" + } + ) + except Exception as e: + raise HTTPException( + status_code=500, + detail={ + "error": "Internal server error", + "message": f"Unexpected error: {str(e)}", + "type": type(e).__name__ + } + ) + + +@router.get("/health", response_model=HealthResponse, summary="Health check") +async def health_check() -> HealthResponse: + """ + Health check for article extraction service. + + Returns service status and version information. + """ + return HealthResponse( + service="ArticleExtractor", + status="healthy", + version="1.0.0" + ) diff --git a/app/api/routes/batch_upload.py b/app/api/routes/batch_upload.py new file mode 100644 index 0000000000000000000000000000000000000000..bfc10f8e05c6ee713983cc859b29de1db29eea13 --- /dev/null +++ b/app/api/routes/batch_upload.py @@ -0,0 +1,302 @@ +""" +Batch upload router with WebSocket support for real-time progress tracking. + +Handles: +- Batch PDF upload (multiple files) +- Real-time progress updates via WebSocket +- Status tracking (PENDING → PROCESSING → PUBLISHED/REFUSED) +""" + +from fastapi import APIRouter, UploadFile, File, WebSocket, WebSocketDisconnect, HTTPException, Depends +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select, and_ +from typing import List +import asyncio +import json +import uuid +from datetime import datetime + +from app.core.database import get_db +from app.models.law import Law +from app.tasks.process_law import process_law_sync +from app.core.config import settings + +router = APIRouter(prefix="/admin/batch", tags=["batch-upload"]) + +# WebSocket connection manager +class ConnectionManager: + def __init__(self): + self.active_connections: dict[str, WebSocket] = {} + + async def connect(self, session_id: str, websocket: WebSocket): + await websocket.accept() + self.active_connections[session_id] = websocket + + def disconnect(self, session_id: str): + if session_id in self.active_connections: + del self.active_connections[session_id] + + async def send_progress(self, session_id: str, data: dict): + if session_id in self.active_connections: + try: + await self.active_connections[session_id].send_json(data) + except: + self.disconnect(session_id) + +manager = ConnectionManager() + + +@router.websocket("/ws/{session_id}") +async def websocket_endpoint(websocket: WebSocket, session_id: str): + """ + WebSocket endpoint for real-time upload progress. + + Client receives JSON messages: + { + "type": "progress", + "law_id": 123, + "filename": "law.pdf", + "progress": 45, + "status": "processing", + "message": "Extracting text..." + } + """ + assert session_id and isinstance(session_id, str), "session_id must be a non-empty string" + await manager.connect(session_id, websocket) + MAX_KEEPALIVE_CYCLES = 3600 # ~1 hour at 1s keepalive cadence (NASA Rule 2: bounded loops) + try: + for _ in range(MAX_KEEPALIVE_CYCLES): + # Keep connection alive, bounded to prevent infinite hang + await websocket.receive_text() + # Gracefully close after max cycles + await websocket.close(code=1000, reason="Session timeout (max keepalive reached)") + except WebSocketDisconnect: + pass + finally: + manager.disconnect(session_id) + + +@router.post("/upload") +async def batch_upload( + files: List[UploadFile] = File(...), + session_id: str = None, + db: AsyncSession = Depends(get_db) +): + """ + Batch upload multiple PDF files. + + Args: + files: List of PDF files to upload + session_id: WebSocket session ID for progress updates + + Returns: + { + "session_id": "uuid", + "total_files": 10, + "created_laws": [{"id": 1, "filename": "law1.pdf"}, ...] + } + """ + assert files is not None and len(files) > 0, "At least one file must be uploaded" + assert all(hasattr(f, 'filename') for f in files), "All uploaded items must be valid files" + + # Generate session ID if not provided + if not session_id: + session_id = str(uuid.uuid4()) + + # Validate files + for file in files: + if not file.filename.endswith('.pdf'): + raise HTTPException(400, f"File {file.filename} is not a PDF") + + # Check file size + file.file.seek(0, 2) # Seek to end + size = file.file.tell() + file.file.seek(0) # Reset + + if size > settings.MAX_UPLOAD_SIZE: + raise HTTPException(400, f"File {file.filename} exceeds {settings.MAX_UPLOAD_SIZE / 1024 / 1024}MB limit") + + created_laws = [] + + # Process each file + for idx, file in enumerate(files): + try: + # Send progress update + await manager.send_progress(session_id, { + "type": "upload_progress", + "current": idx + 1, + "total": len(files), + "filename": file.filename, + "status": "uploading" + }) + + # Read file content + content = await file.read() + + # Create law entry with PENDING status + law = Law( + reference=f"PENDING-{uuid.uuid4().hex[:8]}", + title=f"PENDING-{uuid.uuid4().hex[:8]}", + type="loi", + content="", # Will be filled by processing + status="pending", + processing_progress=0, + processing_started_at=None + ) + + db.add(law) + await db.flush() # Get ID + + # Save file temporarily + from pathlib import Path + from app.services.file_upload_service import get_upload_service + + upload_service = get_upload_service() + file_id = str(uuid.uuid4()) + file_path = upload_service.storage_path / f"{file_id}.pdf" + + with open(file_path, "wb") as f: + f.write(content) + + created_laws.append({ + "id": law.id, + "filename": file.filename, + "file_id": file_id, + "status": "pending" + }) + + # Send creation confirmation + await manager.send_progress(session_id, { + "type": "file_created", + "law_id": law.id, + "filename": file.filename, + "status": "pending" + }) + + except Exception as e: + await manager.send_progress(session_id, { + "type": "error", + "filename": file.filename, + "error": str(e) + }) + + await db.commit() + + # Start background processing for all files + asyncio.create_task(process_batch(created_laws, session_id, db)) + + return { + "session_id": session_id, + "total_files": len(files), + "created_laws": created_laws + } + + +async def process_batch(laws: List[dict], session_id: str, db: AsyncSession): + """ + Background task to process all uploaded files. + """ + for law_data in laws: + try: + law_id = law_data["id"] + file_id = law_data["file_id"] + + # Update status to PROCESSING + result = await db.execute(select(Law).where(Law.id == law_id)) + law = result.scalar_one_or_none() + + if law: + law.status = "processing" + law.processing_progress = 0 + law.processing_started_at = datetime.utcnow() + await db.commit() + + # Send processing start + await manager.send_progress(session_id, { + "type": "processing_start", + "law_id": law_id, + "filename": law_data["filename"], + "status": "processing" + }) + + # Call processing pipeline (synchronous) + # We'll need to update process_law_sync to emit progress + result = process_law_sync(law_id, file_id) + + # Update final status + if result.get("status") == "completed": + law.status = "published" + law.processing_progress = 100 + else: + law.status = "refused" + law.processing_error = str(result.get("errors", [])) + + await db.commit() + + # Send completion + await manager.send_progress(session_id, { + "type": "processing_complete", + "law_id": law_id, + "filename": law_data["filename"], + "status": law.status, + "progress": 100 + }) + + except Exception as e: + # Mark as refused + law.status = "refused" + law.processing_error = str(e) + await db.commit() + + await manager.send_progress(session_id, { + "type": "processing_error", + "law_id": law_id, + "filename": law_data["filename"], + "error": str(e) + }) + + +@router.get("/status") +async def get_batch_status( + status: str = None, + db: AsyncSession = Depends(get_db) +): + """ + Get all laws with optional status filter. + + Args: + status: Filter by status (pending, processing, published, refused) + + Returns: + List of laws with their processing status + """ + assert status is None or isinstance(status, str), "Status filter must be a string or None" + assert status is None or status in ("pending", "processing", "published", "refused"), \ + f"Invalid status: {status}. Must be one of: pending, processing, published, refused" + + query = select(Law) + + if status: + query = query.where(Law.status == status) + + query = query.order_by(Law.created_at.desc()) + + result = await db.execute(query) + laws = result.scalars().all() + + return { + "total": len(laws), + "laws": [ + { + "id": law.id, + "title": law.title, + "reference": law.reference, + "status": law.status, + "processing_progress": law.processing_progress, + "processing_error": law.processing_error, + "created_at": law.created_at.isoformat() if law.created_at else None, + "processing_started_at": law.processing_started_at.isoformat() if law.processing_started_at else None + } + for law in laws + ] + } diff --git a/app/api/routes/categories.py b/app/api/routes/categories.py new file mode 100644 index 0000000000000000000000000000000000000000..badb6b026614025278ba9fe4f4d40c4c55e4f72c --- /dev/null +++ b/app/api/routes/categories.py @@ -0,0 +1,600 @@ +""" +API Routes for Category Management (Service 8). + +Endpoints: +- POST /api/v1/categories - Create category +- GET /api/v1/categories - List categories +- GET /api/v1/categories/{id} - Get single category +- PUT /api/v1/categories/{id} - Update category +- DELETE /api/v1/categories/{id} - Delete category +- GET /api/v1/categories/{id}/stats - Get category statistics +- GET /api/v1/categories/stats/all - Get all category statistics +- GET /api/v1/categories/mapping/id-to-name - Get ID → Name mapping +- GET /api/v1/categories/mapping/name-to-id - Get Name → ID mapping +- GET /api/v1/categories/health - Health check + +Author: JuriX Team +Version: 2.1.0 +""" + +import logging +from typing import List, Dict +from fastapi import APIRouter, Depends, HTTPException, status, Query +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.database import get_db +from app.services.category_service import ( + CategoryService, + CategoryNotFoundError, + DuplicateCategoryNameError, + CategoryInUseError, + CategoryServiceError +) +from app.schemas.law import CategoryCreate, CategoryUpdate, CategoryResponse, CategoryStats + +# Configure logger +logger = logging.getLogger(__name__) + +# Create router +router = APIRouter( + prefix="/api/v1/categories", + tags=["categories"], + responses={ + 400: {"description": "Bad Request - Validation error"}, + 404: {"description": "Not Found - Category doesn't exist"}, + 409: {"description": "Conflict - Category in use or duplicate name"}, + 500: {"description": "Internal Server Error"} + } +) + + +# ============================================================================ +# CRUD ENDPOINTS +# ============================================================================ + +@router.post( + "", + response_model=CategoryResponse, + status_code=status.HTTP_201_CREATED, + summary="Create a new category" +) +async def create_category( + category_data: CategoryCreate, + db: AsyncSession = Depends(get_db) +) -> CategoryResponse: + """ + Create a new legal category. + + **Parameters:** + - **name**: Category name (unique, 1-100 characters) + - **description**: Optional description + + **Returns:** + - Created category with ID and metadata + + **Raises:** + - 400: Invalid input data + - 409: Category name already exists + - 500: Server error + """ + assert category_data is not None, "CategoryCreate must not be None" + assert isinstance(category_data.name, str) and len(category_data.name) > 0, "Category name must be a non-empty string" + + try: + logger.info(f"📥 POST /categories - Creating: {category_data.name}") + service = CategoryService(db) + category = await service.create_category(category_data) + + # Build response + law_count = await service._get_law_count_for_category(category.id) + response = CategoryResponse( + id=category.id, + name=category.name, + description=category.description, + created_at=category.created_at, + law_count=law_count + ) + + logger.info(f"✅ Category created: {category.name} (ID: {category.id})") + return response + + except DuplicateCategoryNameError as e: + logger.warning(f"⚠️ Duplicate category name: {e}") + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=str(e) + ) + except CategoryServiceError as e: + logger.error(f"❌ Service error: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to create category: {str(e)}" + ) + + +@router.get( + "", + response_model=List[CategoryResponse], + summary="List all categories" +) +async def list_categories( + skip: int = Query(0, ge=0, description="Number of records to skip"), + limit: int = Query(100, ge=1, le=1000, description="Maximum records to return"), + db: AsyncSession = Depends(get_db) +) -> List[CategoryResponse]: + """ + List all legal categories with law counts. + + **Parameters:** + - **skip**: Pagination offset (default: 0) + - **limit**: Max results (default: 100, max: 1000) + + **Returns:** + - List of categories ordered by name + - Each category includes law_count + + **Example Response:** + ```json + [ + { + "id": 1, + "name": "Droit Civil", + "description": "Civil law", + "created_at": "2024-01-01T00:00:00", + "law_count": 45 + } + ] + ``` + """ + try: + logger.info(f"📥 GET /categories (skip={skip}, limit={limit})") + service = CategoryService(db) + categories = await service.list_categories(skip=skip, limit=limit) + + # DEBUG: Print first category details + if categories: + first = categories[0] + logger.info(f"DEBUG: First category: ID={first.id}, Name={first.name}, Icon={getattr(first, 'icon', 'MISSING')}") + + logger.info(f"✅ Returning {len(categories)} categories") + return categories + + except CategoryServiceError as e: + logger.error(f"❌ Service error: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to list categories: {str(e)}" + ) + + +@router.get( + "/{category_id}", + response_model=CategoryResponse, + summary="Get a single category" +) +async def get_category( + category_id: int, + db: AsyncSession = Depends(get_db) +) -> CategoryResponse: + """ + Get a category by ID. + + **Parameters:** + - **category_id**: Category ID + + **Returns:** + - Category details with law count + + **Raises:** + - 404: Category not found + - 500: Server error + """ + assert isinstance(category_id, int) and category_id > 0, "Category ID must be a positive integer" + + try: + logger.info(f"�� GET /categories/{category_id}") + service = CategoryService(db) + category = await service.get_category(category_id) + + # Build response + law_count = await service._get_law_count_for_category(category.id) + response = CategoryResponse( + id=category.id, + name=category.name, + description=category.description, + icon=category.icon, + created_at=category.created_at, + law_count=law_count + ) + + logger.info(f"✅ Category found: {category.name}") + return response + + except CategoryNotFoundError as e: + logger.warning(f"⚠️ Category not found: {e}") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(e) + ) + except CategoryServiceError as e: + logger.error(f"❌ Service error: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to fetch category: {str(e)}" + ) + + +@router.put( + "/{category_id}", + response_model=CategoryResponse, + summary="Update a category" +) +async def update_category( + category_id: int, + category_data: CategoryUpdate, + db: AsyncSession = Depends(get_db) +) -> CategoryResponse: + """ + Update a category. + + **Parameters:** + - **category_id**: Category ID + - **name**: New name (optional) + - **description**: New description (optional) + + **Returns:** + - Updated category + + **Raises:** + - 404: Category not found + - 409: New name conflicts with existing category + - 500: Server error + """ + assert isinstance(category_id, int) and category_id > 0, "category_id must be a positive integer" + assert category_data is not None, "CategoryUpdate must not be None" + + try: + logger.info(f"📥 PUT /categories/{category_id}") + service = CategoryService(db) + category = await service.update_category(category_id, category_data) + + # Build response + law_count = await service._get_law_count_for_category(category.id) + response = CategoryResponse( + id=category.id, + name=category.name, + description=category.description, + created_at=category.created_at, + law_count=law_count + ) + + logger.info(f"✅ Category updated: {category.name}") + return response + + except CategoryNotFoundError as e: + logger.warning(f"⚠️ Category not found: {e}") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(e) + ) + except DuplicateCategoryNameError as e: + logger.warning(f"⚠️ Duplicate category name: {e}") + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=str(e) + ) + except CategoryServiceError as e: + logger.error(f"❌ Service error: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update category: {str(e)}" + ) + + +@router.delete( + "/{category_id}", + status_code=status.HTTP_204_NO_CONTENT, + summary="Delete a category" +) +async def delete_category( + category_id: int, + force: bool = Query(False, description="Force delete even if laws are associated"), + db: AsyncSession = Depends(get_db) +): + """ + Delete a category. + + **Parameters:** + - **category_id**: Category ID + - **force**: If true, deletes category even if laws are associated + (laws will have category_id set to NULL) + + **Returns:** + - 204 No Content on success + + **Raises:** + - 404: Category not found + - 409: Category has associated laws and force=false + - 500: Server error + + **Warning:** + Using force=true will set category_id to NULL for all associated laws. + Use with caution in production! + """ + assert isinstance(category_id, int) and category_id > 0, "category_id must be a positive integer" + + try: + logger.info(f"📥 DELETE /categories/{category_id} (force={force})") + service = CategoryService(db) + await service.delete_category(category_id, force=force) + + logger.info(f"✅ Category deleted: ID {category_id}") + return None # 204 No Content + + except CategoryNotFoundError as e: + logger.warning(f"⚠️ Category not found: {e}") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(e) + ) + except CategoryInUseError as e: + logger.warning(f"⚠️ Category in use: {e}") + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=str(e) + ) + except CategoryServiceError as e: + logger.error(f"❌ Service error: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to delete category: {str(e)}" + ) + + +# ============================================================================ +# STATISTICS ENDPOINTS +# ============================================================================ + +@router.get( + "/{category_id}/stats", + response_model=CategoryStats, + summary="Get category statistics" +) +async def get_category_stats( + category_id: int, + db: AsyncSession = Depends(get_db) +) -> CategoryStats: + """ + Get statistics for a single category. + + **Parameters:** + - **category_id**: Category ID + + **Returns:** + - Category statistics (law count, percentage) + + **Example Response:** + ```json + { + "category_id": 1, + "category_name": "Droit Civil", + "law_count": 45, + "percentage": 26.47 + } + ``` + + **Raises:** + - 404: Category not found + - 500: Server error + """ + try: + logger.info(f"📥 GET /categories/{category_id}/stats") + service = CategoryService(db) + stats = await service.get_category_stats(category_id) + + logger.info(f"✅ Stats returned: {stats.law_count} laws ({stats.percentage:.2f}%)") + return stats + + except CategoryNotFoundError as e: + logger.warning(f"⚠️ Category not found: {e}") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(e) + ) + except CategoryServiceError as e: + logger.error(f"❌ Service error: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to fetch category stats: {str(e)}" + ) + + +@router.get( + "/stats/all", + response_model=List[CategoryStats], + summary="Get statistics for all categories" +) +async def get_all_category_stats( + db: AsyncSession = Depends(get_db) +) -> List[CategoryStats]: + """ + Get statistics for all categories. + + **Returns:** + - List of category statistics ordered by law count (descending) + - Each includes: category ID, name, law count, percentage + + **Example Response:** + ```json + [ + { + "category_id": 1, + "category_name": "Droit Civil", + "law_count": 45, + "percentage": 26.47 + }, + { + "category_id": 2, + "category_name": "Droit Pénal", + "law_count": 38, + "percentage": 22.35 + } + ] + ``` + + **Use Case:** + - Dashboard analytics + - Category distribution charts + - Admin overview + """ + try: + logger.info("📥 GET /categories/stats/all") + service = CategoryService(db) + stats_list = await service.get_all_category_stats() + + logger.info(f"✅ Stats returned for {len(stats_list)} categories") + return stats_list + + except CategoryServiceError as e: + logger.error(f"❌ Service error: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to fetch all category stats: {str(e)}" + ) + + +# ============================================================================ +# MAPPING UTILITY ENDPOINTS +# ============================================================================ + +@router.get( + "/mapping/id-to-name", + response_model=Dict[int, str], + summary="Get ID → Name mapping" +) +async def get_id_to_name_mapping( + db: AsyncSession = Depends(get_db) +) -> Dict[int, str]: + """ + Get ID → Name mapping for all categories. + + **Returns:** + - Dictionary mapping category IDs to names + + **Example Response:** + ```json + { + "1": "Droit Civil", + "2": "Droit Pénal", + "3": "Droit Commercial" + } + ``` + + **Use Case:** + - Fast lookups without database queries + - Frontend dropdown populations + - DocumentClassifier integration + """ + try: + logger.info("📥 GET /categories/mapping/id-to-name") + service = CategoryService(db) + mapping = await service.get_category_mapping() + + logger.info(f"✅ Mapping returned: {len(mapping)} categories") + return mapping + + except CategoryServiceError as e: + logger.error(f"❌ Service error: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to fetch ID → Name mapping: {str(e)}" + ) + + +@router.get( + "/mapping/name-to-id", + response_model=Dict[str, int], + summary="Get Name → ID mapping" +) +async def get_name_to_id_mapping( + db: AsyncSession = Depends(get_db) +) -> Dict[str, int]: + """ + Get Name → ID mapping for all categories. + + **Returns:** + - Dictionary mapping category names to IDs + + **Example Response:** + ```json + { + "Droit Civil": 1, + "Droit Pénal": 2, + "Droit Commercial": 3 + } + ``` + + **Use Case:** + - Reverse lookups by name + - Category name validation + - Import/export utilities + """ + try: + logger.info("📥 GET /categories/mapping/name-to-id") + service = CategoryService(db) + mapping = await service.get_reverse_category_mapping() + + logger.info(f"✅ Reverse mapping returned: {len(mapping)} categories") + return mapping + + except CategoryServiceError as e: + logger.error(f"❌ Service error: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to fetch Name → ID mapping: {str(e)}" + ) + + +# ============================================================================ +# HEALTH CHECK +# ============================================================================ + +@router.get( + "/health", + response_model=Dict[str, str], + summary="Health check" +) +async def health_check( + db: AsyncSession = Depends(get_db) +) -> Dict[str, str]: + """ + Check CategoryService health. + + **Returns:** + - Service status and metadata + + **Example Response:** + ```json + { + "service": "CategoryService", + "status": "healthy", + "version": "2.1.0", + "timestamp": "2024-01-11T12:00:00" + } + ``` + """ + try: + logger.debug("📥 GET /categories/health") + service = CategoryService(db) + health = service.health_check() + + logger.debug("✅ Health check passed") + return health + + except Exception as e: + logger.error(f"❌ Health check failed: {e}") + return { + "service": "CategoryService", + "status": "unhealthy", + "error": str(e) + } diff --git a/app/api/routes/classifier.py b/app/api/routes/classifier.py new file mode 100644 index 0000000000000000000000000000000000000000..8b78e46e42f054d013c5e3793c0f0d6877e68215 --- /dev/null +++ b/app/api/routes/classifier.py @@ -0,0 +1,392 @@ +""" +API Routes pour catégorisation de documents juridiques. + +Endpoints: +- POST /api/v1/classifier/classify - Classifie un document +- GET /api/v1/classifier/health - Vérifie santé du service +- GET /api/v1/classifier/categories - Liste toutes les catégories + +Usage: + curl -X POST http://localhost:8000/api/v1/classifier/classify \ + -H "Content-Type: application/json" \ + -d '{"text": "Article 1. La SARL est une société commerciale..."}' +""" + +from typing import Dict, Any, List +import logging + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel, Field, validator + +from app.core.dependencies import get_document_classifier +from app.services.document_classifier import DocumentClassifier, DocumentClassificationError + +logger = logging.getLogger(__name__) + +# Router configuration +router = APIRouter( + prefix="/api/v1/classifier", + tags=["classifier"], + responses={ + 500: {"description": "Erreur interne du service de classification"} + } +) + + +# ==================== MODELS ==================== + +class ClassifyDocumentRequest(BaseModel): + """Requête de classification de document.""" + + text: str = Field( + ..., + min_length=50, + max_length=50000, + description="Texte du document à classifier (50-50000 caractères)", + example="Article 1. Les sociétés commerciales sont régies par l'acte uniforme OHADA. La SARL doit avoir un capital social minimum." + ) + + language: str = Field( + default="fr", + description="Langue du document ('fr' ou 'en')", + example="fr" + ) + + top_k: int = Field( + default=3, + ge=1, + le=12, + description="Nombre de suggestions à retourner (1-12)", + example=3 + ) + + min_confidence: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Seuil de confiance minimum (0-1)", + example=0.0 + ) + + @validator('text') + def text_not_empty(cls, v): + """Valide que le texte n'est pas vide.""" + if not v or not v.strip(): + raise ValueError("Le texte ne peut pas être vide") + return v + + @validator('language') + def language_valid(cls, v): + """Valide que la langue est supportée.""" + if v not in ['fr', 'en']: + raise ValueError("Langue doit être 'fr' ou 'en'") + return v + + class Config: + json_schema_extra = { + "example": { + "text": "Article 1. La SARL est régie par l'acte uniforme OHADA relatif au droit des sociétés commerciales. Le capital social est divisé en parts sociales.", + "language": "fr", + "top_k": 3, + "min_confidence": 0.0 + } + } + + +class CategorySuggestion(BaseModel): + """Suggestion de catégorie avec confiance.""" + + category_id: int = Field( + ..., + ge=1, + le=12, + description="ID de la catégorie (1-12)", + example=4 + ) + + category_name: str = Field( + ..., + description="Nom de la catégorie", + example="Droit Commercial OHADA" + ) + + confidence: float = Field( + ..., + ge=0.0, + le=1.0, + description="Score de confiance (0-1)", + example=0.92 + ) + + method: str = Field( + ..., + description="Méthode utilisée ('keyword', 'ml', 'hybrid')", + example="keyword" + ) + + class Config: + json_schema_extra = { + "example": { + "category_id": 4, + "category_name": "Droit Commercial OHADA", + "confidence": 0.92, + "method": "keyword" + } + } + + +class ClassifyDocumentResponse(BaseModel): + """Réponse de classification.""" + + suggestions: List[CategorySuggestion] = Field( + ..., + description="Liste des suggestions classées par confiance décroissante" + ) + + processing_time_ms: int = Field( + ..., + description="Temps de traitement en millisecondes", + example=125 + ) + + class Config: + json_schema_extra = { + "example": { + "suggestions": [ + { + "category_id": 4, + "category_name": "Droit Commercial OHADA", + "confidence": 0.92, + "method": "keyword" + }, + { + "category_id": 12, + "category_name": "Droit des Affaires", + "confidence": 0.68, + "method": "keyword" + }, + { + "category_id": 2, + "category_name": "Droit Civil", + "confidence": 0.45, + "method": "keyword" + } + ], + "processing_time_ms": 125 + } + } + + +class HealthResponse(BaseModel): + """Réponse du health check.""" + + service: str = Field(..., example="DocumentClassifier") + status: str = Field(..., example="healthy") + mode: str = Field(..., example="keywords-only") + ml_model_loaded: bool = Field(..., example=False) + vectorizer_loaded: bool = Field(..., example=False) + categories_count: int = Field(..., example=12) + version: str = Field(..., example="1.0.0-mvp") + + +class Category(BaseModel): + """Catégorie juridique.""" + + id: int = Field(..., ge=1, le=12, example=4) + name: str = Field(..., example="Droit Commercial OHADA") + + +class CategoriesResponse(BaseModel): + """Liste des catégories disponibles.""" + + categories: List[Category] = Field( + ..., + description="Liste des 12 catégories juridiques" + ) + + +# ==================== ENDPOINTS ==================== + +@router.post( + "/classify", + response_model=ClassifyDocumentResponse, + status_code=status.HTTP_200_OK, + summary="Classifier un document juridique", + description=""" +Classifie un document juridique dans une ou plusieurs des 12 catégories. + +**Méthodes:** +- Phase 1 (MVP): Keywords-only (70-75% précision) +- Phase 3: Hybrid 40% keywords + 60% ML (87% top-1, 96% top-3) + +**Catégories disponibles:** +1. Droit Constitutionnel +2. Droit Civil +3. Droit Pénal +4. Droit Commercial OHADA +5. Droit du Travail +6. Droit Fiscal +7. Droit Administratif +8. Droit Foncier +9. Droit de la Famille +10. Droit de l'Environnement +11. Droit International +12. Droit des Affaires + +**Performance:** +- Temps de réponse: <500ms (MVP), <2s (ML) +- Top-k suggestions classées par confiance décroissante +""" +) +async def classify_document( + request: ClassifyDocumentRequest, + classifier: DocumentClassifier = Depends(get_document_classifier) +) -> ClassifyDocumentResponse: + """ + Classifie un document juridique. + + Args: + request: Requête avec texte et paramètres + classifier: Instance du classifier (injectée) + + Returns: + Suggestions de catégories avec scores de confiance + + Raises: + HTTPException 400: Texte invalide + HTTPException 500: Erreur interne de classification + """ + assert request is not None, "ClassifyDocumentRequest must not be None" + assert isinstance(request.text, str) and len(request.text) >= 50, "Text must be at least 50 characters" + + try: + import time + start_time = time.time() + + # Classification + results = classifier.classify( + text=request.text, + language=request.language, + top_k=request.top_k, + min_confidence=request.min_confidence + ) + + # Formatter réponse + suggestions = [] + for category_id, confidence, method in results: + category_name = classifier.get_category_name(category_id) + suggestions.append( + CategorySuggestion( + category_id=category_id, + category_name=category_name, + confidence=confidence, + method=method + ) + ) + + processing_time_ms = int((time.time() - start_time) * 1000) + + logger.info( + f"Classification réussie: {len(suggestions)} suggestions, " + f"top={suggestions[0].category_name if suggestions else 'N/A'}, " + f"temps={processing_time_ms}ms" + ) + + return ClassifyDocumentResponse( + suggestions=suggestions, + processing_time_ms=processing_time_ms + ) + + except ValueError as e: + logger.warning(f"Requête invalide: {e}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(e) + ) + + except DocumentClassificationError as e: + logger.error(f"Erreur classification: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Erreur de classification: {str(e)}" + ) + + except Exception as e: + logger.error(f"Erreur inattendue: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Erreur interne du service de classification" + ) + + +@router.get( + "/health", + response_model=HealthResponse, + status_code=status.HTTP_200_OK, + summary="Vérifier la santé du service", + description="Vérifie l'état de santé du service de classification et les modèles chargés." +) +async def health_check( + classifier: DocumentClassifier = Depends(get_document_classifier) +) -> HealthResponse: + """ + Vérifie la santé du service de classification. + + Args: + classifier: Instance du classifier (injectée) + + Returns: + État du service et des modèles + """ + try: + health_status = classifier.health_check() + + logger.debug(f"Health check: {health_status}") + + return HealthResponse(**health_status) + + except Exception as e: + logger.error(f"Erreur health check: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Impossible de vérifier la santé du service" + ) + + +@router.get( + "/categories", + response_model=CategoriesResponse, + status_code=status.HTTP_200_OK, + summary="Lister les catégories", + description="Retourne la liste des 12 catégories juridiques disponibles." +) +async def get_categories( + classifier: DocumentClassifier = Depends(get_document_classifier) +) -> CategoriesResponse: + """ + Retourne toutes les catégories juridiques disponibles. + + Args: + classifier: Instance du classifier (injectée) + + Returns: + Liste des catégories avec IDs et noms + """ + try: + all_categories = classifier.get_all_categories() + + categories = [ + Category(id=cat_id, name=name) + for cat_id, name in sorted(all_categories.items()) + ] + + logger.debug(f"Retour de {len(categories)} catégories") + + return CategoriesResponse(categories=categories) + + except Exception as e: + logger.error(f"Erreur récupération catégories: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Impossible de récupérer les catégories" + ) diff --git a/app/api/routes/language.py b/app/api/routes/language.py new file mode 100644 index 0000000000000000000000000000000000000000..c52ec26a790f20642d09fb9ce728c771fbbebc3b --- /dev/null +++ b/app/api/routes/language.py @@ -0,0 +1,284 @@ +""" +API Routes pour détection de langue. + +Endpoints: +- POST /api/v1/language/detect - Détecte langue d'un texte +- GET /api/v1/language/health - Vérifie santé du service + +Usage: + curl -X POST http://localhost:8000/api/v1/language/detect \ + -H "Content-Type: application/json" \ + -d '{"text": "Article 1. La présente loi..."}' +""" + +from typing import Dict, Any +import logging + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel, Field, validator + +from app.core.dependencies import get_language_detector +from app.services.language_detector import LanguageDetector, LanguageDetectionError + +logger = logging.getLogger(__name__) + +# Router configuration +router = APIRouter( + prefix="/api/v1/language", + tags=["language"], + responses={ + 500: {"description": "Erreur interne du service de détection"} + } +) + + +# ==================== MODELS ==================== + +class DetectLanguageRequest(BaseModel): + """Requête de détection de langue.""" + + text: str = Field( + ..., + min_length=50, + max_length=50000, + description="Texte à analyser (50-50000 caractères)", + example="Article 1. La présente loi régit les conditions de création des sociétés commerciales au Cameroun." + ) + + min_confidence: float = Field( + default=0.80, + ge=0.0, + le=1.0, + description="Seuil de confiance minimum (0-1)", + example=0.80 + ) + + @validator('text') + def text_not_empty(cls, v): + """Valide que le texte n'est pas vide.""" + if not v or not v.strip(): + raise ValueError("Le texte ne peut pas être vide") + return v.strip() + + class Config: + schema_extra = { + "example": { + "text": "Article 1. La présente loi régit les conditions de création des sociétés commerciales au Cameroun. Les dispositions du Code civil s'appliquent en matière de contrats.", + "min_confidence": 0.80 + } + } + + +class DetectLanguageResponse(BaseModel): + """Réponse de détection de langue.""" + + language: str = Field( + ..., + description="Code langue détectée ('fr' ou 'en')", + example="fr" + ) + + confidence: float = Field( + ..., + ge=0.0, + le=1.0, + description="Score de confiance (0-1)", + example=0.98 + ) + + method_votes: Dict[str, str] = Field( + ..., + description="Votes de chaque méthode de détection", + example={ + "langdetect": "fr", + "spacy": "fr", + "fasttext": "fr" + } + ) + + consensus: bool = Field( + ..., + description="True si au moins 2/3 méthodes d'accord", + example=True + ) + + processing_time_ms: int = Field( + ..., + ge=0, + description="Temps de traitement en millisecondes", + example=450 + ) + + text_length: int = Field( + ..., + ge=0, + description="Longueur du texte analysé", + example=156 + ) + + class Config: + schema_extra = { + "example": { + "language": "fr", + "confidence": 0.98, + "method_votes": { + "langdetect": "fr", + "spacy": "fr", + "fasttext": "fr" + }, + "consensus": True, + "processing_time_ms": 450, + "text_length": 156 + } + } + + +class HealthResponse(BaseModel): + """Réponse du health check.""" + + service: str = Field(..., example="LanguageDetector") + status: str = Field(..., example="healthy") + models: Dict[str, str] = Field( + ..., + example={ + "spacy_fr": "✅ OK", + "spacy_en": "✅ OK", + "fasttext": "✅ OK" + } + ) + + +# ==================== ENDPOINTS ==================== + +@router.post( + "/detect", + response_model=DetectLanguageResponse, + status_code=status.HTTP_200_OK, + summary="Détecte la langue d'un texte", + description=""" + Détecte automatiquement la langue (français ou anglais) d'un texte juridique. + + **Méthode:** Triple ensemble (langdetect + spaCy + fastText) avec vote majoritaire + + **Performance:** <1 seconde pour textes jusqu'à 5000 caractères + + **Précision:** >99% sur documents juridiques + """, + response_description="Langue détectée avec métadonnées de confiance" +) +async def detect_language( + request: DetectLanguageRequest, + detector: LanguageDetector = Depends(get_language_detector) +) -> DetectLanguageResponse: + """ + Détecte la langue d'un texte. + + Args: + request: Requête contenant le texte à analyser + detector: Service de détection (injecté) + + Returns: + Résultat de détection avec langue, confiance et votes + + Raises: + HTTPException 400: Si texte invalide (trop court, vide) + HTTPException 500: Si erreur interne de détection + """ + assert request is not None, "DetectLanguageRequest must not be None" + assert isinstance(request.text, str) and len(request.text) >= 50, "Text must be at least 50 characters" + + try: + logger.info(f"📥 Requête détection langue (texte: {len(request.text)} chars)") + + # Appeler le service de détection + result = detector.detect( + text=request.text, + min_confidence=request.min_confidence + ) + + logger.info( + f"✅ Détection réussie: {result['language']} " + f"({result['confidence']:.2%}, {result['processing_time_ms']}ms)" + ) + + return DetectLanguageResponse(**result) + + except ValueError as e: + # Erreur de validation (texte trop court, etc.) + logger.warning(f"⚠️ Validation error: {e}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(e) + ) + + except LanguageDetectionError as e: + # Erreur du service de détection + logger.error(f"❌ Detection error: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Erreur lors de la détection: {str(e)}" + ) + + except Exception as e: + # Erreur inattendue + logger.exception(f"❌ Unexpected error: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Erreur interne du serveur" + ) + + +@router.get( + "/health", + response_model=HealthResponse, + status_code=status.HTTP_200_OK, + summary="Vérifie la santé du service", + description=""" + Vérifie que tous les modèles NLP sont chargés et fonctionnels. + + **Modèles vérifiés:** + - spaCy français (fr_core_news_sm) + - spaCy anglais (en_core_web_sm) + - fastText (lid.176.bin) + """, + response_description="État de santé de chaque composant" +) +async def health_check( + detector: LanguageDetector = Depends(get_language_detector) +) -> HealthResponse: + """ + Vérifie l'état de santé du service de détection. + + Args: + detector: Service de détection (injecté) + + Returns: + État de santé avec status de chaque modèle + + Raises: + HTTPException 503: Si un ou plusieurs modèles sont défaillants + """ + try: + health_status = detector.health_check() + + # Si status != "healthy", retourner 503 Service Unavailable + if health_status["status"] != "healthy": + logger.warning(f"⚠️ Service degradé: {health_status}") + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=health_status + ) + + logger.info("✅ Health check OK") + return HealthResponse(**health_status) + + except HTTPException: + # Re-raise HTTP exceptions + raise + + except Exception as e: + logger.exception(f"❌ Health check error: {e}") + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=f"Erreur lors du health check: {str(e)}" + ) diff --git a/app/api/routes/laws.py b/app/api/routes/laws.py new file mode 100644 index 0000000000000000000000000000000000000000..81f4f1f44fabc53224c93f0103f95539a15bf478 --- /dev/null +++ b/app/api/routes/laws.py @@ -0,0 +1,810 @@ +""" +API routes for Laws management. + +Endpoints: +- GET /api/v1/laws - List laws with filters +- GET /api/v1/laws/{id} - Get law detail with articles +- POST /api/v1/admin/laws - Create law (admin only) +- PUT /api/v1/admin/laws/{id} - Update law (admin only) +- DELETE /api/v1/admin/laws/{id} - Delete law (admin only) + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +import logging +import time +from typing import List, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query, status +from fastapi.responses import FileResponse +from pydantic import BaseModel +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.core.database import get_db +from app.models.law import Category, Law +from app.schemas.law import ( + LawCreate, + LawResponse, + LawUpdate, +) +from app.services.search_service import invalidate_search_cache +from app.tasks.process_law import delete_from_meilisearch + + +class LawIngestRequest(BaseModel): + file_id: str + title: Optional[str] = None + original_filename: Optional[str] = None + reference: Optional[str] = None # Optional custom reference + category_id: Optional[int] = None # Optional category ID + + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/v1", tags=["Laws"]) + + +# ==================== DEPENDENCIES ==================== + + +# TODO: Implement authentication +# For now, this is a placeholder +async def get_current_admin_user(): + """ + Dependency for admin authentication. + + TODO: Implement actual authentication with JWT tokens. + For now, returns a mock admin user. + """ + # In production, verify JWT token and check admin role + return {"id": 1, "role": "admin"} + + +# ==================== PUBLIC ENDPOINTS ==================== + + +@router.get("/laws", response_model=List[LawResponse]) +async def get_laws( + language: Optional[str] = Query(None, description="Filter by language (fr or en)"), + category_id: Optional[int] = Query(None, description="Filter by category ID"), + law_status: str = Query( + "published", description="Filter by status (published, draft, or archived)" + ), + skip: int = Query(0, ge=0, description="Number of records to skip"), + limit: int = Query(100, ge=1, le=10000, description="Maximum number of records to return"), + db: AsyncSession = Depends(get_db), +): + """ + Get list of laws with optional filters. + + **Filters:** + - `language`: Filter by language code (fr or en) + - `category_id`: Filter by category ID + - `law_status`: Filter by status (active or archived) + + **Pagination:** + - `skip`: Number of records to skip (default: 0) + - `limit`: Maximum records to return (default: 50, max: 100) + + **Example:** + ``` + GET /api/v1/laws?language=fr&category_id=1&skip=0&limit=10 + ``` + + Returns: + List of laws matching the filters + """ + logger.info( + f"📋 GET /laws - language={language}, category_id={category_id}, " + f"status={law_status}, skip={skip}, limit={limit}" + ) + + try: + # Build query + query = select(Law).options(selectinload(Law.articles), selectinload(Law.category)) + + # Apply filters + if language: + query = query.where(Law.language == language) + + if category_id: + # Filter by category ID directly + query = query.where(Law.category_id == category_id) + + if law_status: + query = query.where(Law.status == law_status) + + # Apply pagination + query = query.offset(skip).limit(limit) + + # Execute query + result = await db.execute(query) + laws = result.scalars().all() + + logger.info(f"✅ Found {len(laws)} laws") + return laws + + except Exception as e: + logger.error(f"❌ Error fetching laws: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error fetching laws" + ) + + +@router.get("/laws/{law_id}", response_model=LawResponse) +async def get_law( + law_id: int, + db: AsyncSession = Depends(get_db), +): + """ + Get law detail by ID with articles. + + **Includes:** + - Law metadata + - All articles (ordered) + - Categories + - Auto-detection results (language, category suggestions) + + Args: + law_id: Law ID + + Returns: + Law detail with articles + + Raises: + 404: Law not found + """ + assert isinstance(law_id, int) and law_id > 0, "law_id must be a positive integer" + + logger.info(f"📄 GET /laws/{law_id}") + + try: + # Query law with relationships + query = ( + select(Law) + .options(selectinload(Law.articles), selectinload(Law.category)) + .where(Law.id == law_id) + ) + + result = await db.execute(query) + law = result.scalar_one_or_none() + + if not law: + logger.warning(f"⚠️ Law {law_id} not found") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f"Law with ID {law_id} not found" + ) + + logger.info(f"✅ Law {law_id} found: {law.title}") + return law + + except HTTPException: + raise + except Exception as e: + logger.error(f"❌ Error fetching law {law_id}: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error fetching law" + ) + + +@router.get("/laws/{law_id}/download") +async def download_law_file( + law_id: int, + db: AsyncSession = Depends(get_db), +): + """Download the original uploaded file for a law.""" + query = select(Law).where(Law.id == law_id) + result = await db.execute(query) + law = result.scalar_one_or_none() + + if not law: + raise HTTPException(status_code=404, detail="Law not found") + + if not law.file_id: + raise HTTPException(status_code=404, detail="No source file found for this law") + + from app.services.file_upload_service import get_upload_service + upload_service = get_upload_service() + + # Try to find file with extension + file_path = None + # Use stored filename or construct one + filename = law.original_filename or f"{law.reference}.pdf" + + # Check for PDF or DOCX using file_id + for ext in [".pdf", ".docx"]: + p = upload_service.storage_path / f"{law.file_id}{ext}" + if p.exists(): + file_path = p + break + + if not file_path: + # Fallback: maybe file_id IS the filename (legacy behavior?) + # Or maybe stored without extension? + p = upload_service.storage_path / law.file_id + if p.exists(): + file_path = p + else: + raise HTTPException(status_code=404, detail="File not found on server") + + return FileResponse( + path=file_path, + filename=law.original_filename or file_path.name, + media_type="application/pdf" if file_path.suffix == ".pdf" else "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + content_disposition_type="inline" + ) + + +@router.get("/laws/{law_id}/pdf-data") +async def get_law_pdf_data( + law_id: int, + db: AsyncSession = Depends(get_db), +): + """ + Return PDF as Base64-encoded JSON to bypass download manager interception. + This allows JavaScript to fetch the PDF without IDM/browser extensions intercepting the request. + """ + import base64 + + query = select(Law).where(Law.id == law_id) + result = await db.execute(query) + law = result.scalar_one_or_none() + + if not law: + raise HTTPException(status_code=404, detail="Law not found") + + if not law.file_id: + raise HTTPException(status_code=404, detail="No source file found for this law") + + from app.services.file_upload_service import get_upload_service + upload_service = get_upload_service() + + # Find the file + file_path = None + for ext in [".pdf", ".docx"]: + p = upload_service.storage_path / f"{law.file_id}{ext}" + if p.exists(): + file_path = p + break + + if not file_path: + p = upload_service.storage_path / law.file_id + if p.exists(): + file_path = p + else: + raise HTTPException(status_code=404, detail="File not found on server") + + # Read file and encode to Base64 + with open(file_path, "rb") as f: + file_bytes = f.read() + + base64_data = base64.b64encode(file_bytes).decode("utf-8") + + return { + "filename": law.original_filename or file_path.name, + "content_type": "application/pdf" if file_path.suffix == ".pdf" else "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "data": base64_data + } + + +@router.post("/laws/{law_id}/pdf-stream") +async def get_law_pdf_stream( + law_id: int, + db: AsyncSession = Depends(get_db), +): + """ + Stream PDF as binary data for efficient loading of large files. + + Uses POST method to bypass download managers (IDM) which typically + only intercept GET requests. Returns PDF as binary with special headers. + + This is more efficient than base64 encoding for large files + (saves ~33% bandwidth and avoids JavaScript decode overhead). + """ + from fastapi.responses import Response + + query = select(Law).where(Law.id == law_id) + result = await db.execute(query) + law = result.scalar_one_or_none() + + if not law: + raise HTTPException(status_code=404, detail="Law not found") + + if not law.file_id: + raise HTTPException(status_code=404, detail="No source file found for this law") + + from app.services.file_upload_service import get_upload_service + upload_service = get_upload_service() + + # Find the file + file_path = None + for ext in [".pdf", ".docx"]: + p = upload_service.storage_path / f"{law.file_id}{ext}" + if p.exists(): + file_path = p + break + + if not file_path: + p = upload_service.storage_path / law.file_id + if p.exists(): + file_path = p + else: + raise HTTPException(status_code=404, detail="File not found on server") + + # Read file content + with open(file_path, "rb") as f: + file_bytes = f.read() + + # Return as binary stream with headers that prevent IDM interception + # Using POST method + octet-stream prevents IDM from intercepting + return Response( + content=file_bytes, + media_type="application/octet-stream", # Not application/pdf - hides from IDM + headers={ + "Content-Disposition": "inline", + "X-Content-Type-Options": "nosniff", + "X-PDF-Content": "true", # Custom header to identify as PDF + "Cache-Control": "private, max-age=3600", + } + ) + + +@router.get("/laws/{law_id}/pdf-info") +async def get_law_pdf_info( + law_id: int, + db: AsyncSession = Depends(get_db), +): + """ + Get PDF information including page count. + Used by the frontend to know how many pages to request. + """ + from pypdf import PdfReader + + query = select(Law).where(Law.id == law_id) + result = await db.execute(query) + law = result.scalar_one_or_none() + + if not law: + raise HTTPException(status_code=404, detail="Law not found") + + if not law.file_id: + raise HTTPException(status_code=404, detail="No source file found for this law") + + from app.services.file_upload_service import get_upload_service + upload_service = get_upload_service() + + # Find the file + file_path = None + for ext in [".pdf", ".docx"]: + p = upload_service.storage_path / f"{law.file_id}{ext}" + if p.exists(): + file_path = p + break + + if not file_path: + p = upload_service.storage_path / law.file_id + if p.exists(): + file_path = p + else: + raise HTTPException(status_code=404, detail="File not found on server") + + # Get page count + try: + reader = PdfReader(str(file_path)) + page_count = len(reader.pages) + except Exception as e: + raise HTTPException(status_code=500, detail=f"Error reading PDF: {str(e)}") + + return { + "law_id": law_id, + "filename": law.original_filename or file_path.name, + "page_count": page_count + } + + +@router.get("/laws/{law_id}/page/{page_num}") +async def get_law_pdf_page_image( + law_id: int, + page_num: int, + dpi: int = Query(default=120, ge=72, le=200, description="DPI for rendering (72-200)"), + db: AsyncSession = Depends(get_db), +): + """ + Render a specific PDF page as a JPEG image. + + Uses Poppler (pdf2image) for reliable rendering of all image formats + including JPEG2000 which pdf.js cannot handle. + + Args: + law_id: ID of the law + page_num: Page number (1-indexed) + dpi: Resolution for rendering (default 120, max 200 for performance) + + Returns: + JPEG image of the specified page + """ + from io import BytesIO + from fastapi.responses import Response + + query = select(Law).where(Law.id == law_id) + result = await db.execute(query) + law = result.scalar_one_or_none() + + if not law: + raise HTTPException(status_code=404, detail="Law not found") + + if not law.file_id: + raise HTTPException(status_code=404, detail="No source file found for this law") + + from app.services.file_upload_service import get_upload_service + upload_service = get_upload_service() + + # Find the file + file_path = None + for ext in [".pdf", ".docx"]: + p = upload_service.storage_path / f"{law.file_id}{ext}" + if p.exists(): + file_path = p + break + + if not file_path: + p = upload_service.storage_path / law.file_id + if p.exists(): + file_path = p + else: + raise HTTPException(status_code=404, detail="File not found on server") + + # Convert page to image using Poppler + try: + from pdf2image import convert_from_path + + # Convert only the specific page (1-indexed) + images = convert_from_path( + str(file_path), + first_page=page_num, + last_page=page_num, + dpi=dpi, + fmt="jpeg", + ) + + if not images: + raise HTTPException(status_code=404, detail=f"Page {page_num} not found") + + # Convert to JPEG bytes + img_buffer = BytesIO() + images[0].save(img_buffer, format="JPEG", quality=85) + img_bytes = img_buffer.getvalue() + + return Response( + content=img_bytes, + media_type="image/jpeg", + headers={ + "Cache-Control": "public, max-age=86400", # Cache for 24 hours + "X-Page-Number": str(page_num), + } + ) + + except Exception as e: + logger.error(f"Error rendering page {page_num} for law {law_id}: {e}") + raise HTTPException(status_code=500, detail=f"Error rendering page: {str(e)}") + + +# ==================== INGESTION ENDPOINTS ==================== + + +@router.post("/admin/laws/ingest", response_model=LawResponse, status_code=status.HTTP_202_ACCEPTED) +async def ingest_law( + request: LawIngestRequest, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_admin_user), +): + """ + Ingest a new law from an uploaded file. + + 1. Creates a Law record (status=processing) + 2. Triggers Celery task to process the file + + Args: + request: Ingest request (file_id, title) + + Returns: + Created Law object (pending) + """ + assert request is not None, "LawIngestRequest must not be None" + assert isinstance(request.file_id, str) and len(request.file_id) > 0, "file_id must be a non-empty string" + + logger.info(f"📥 POST /admin/laws/ingest - File: {request.file_id}") + + try: + # Validate file_id + if not request.file_id or len(request.file_id) < 8: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid file_id: must be at least 8 characters", + ) + + # Check if law with this file_id already exists (idempotency) + ref_prefix = f"PENDING-{request.file_id[:8]}" + + # Check for existing pending law to avoid duplicates + query = select(Law).where(Law.reference == ref_prefix) + result = await db.execute(query) + existing_law = result.scalar_one_or_none() + + if existing_law and existing_law.status == "processing": + logger.info(f"ℹ️ Law already exists for file {request.file_id}: ID={existing_law.id}") + return existing_law + + # If existing law failed, create a new one with timestamp + if existing_law: + ref_prefix = f"{ref_prefix}-{int(time.time())}" + + # Use custom reference if provided, otherwise use PENDING prefix + law_reference = request.reference if request.reference else ref_prefix + + # Check if reference already exists (collision check) + existing_ref_query = select(Law).where(Law.reference == law_reference) + existing_ref_result = await db.execute(existing_ref_query) + existing_ref_law = existing_ref_result.scalar_one_or_none() + + if existing_ref_law: + logger.info(f"ℹ️ Law with reference '{law_reference}' already exists: ID={existing_ref_law.id}. Returning existing law.") + + # If it was stuck or previous attempt failed, we might want to trigger processing again + # For now, just return it so the frontend sees it's there + if existing_ref_law.status == "processing" or existing_ref_law.content == "Document en cours de traitement par le système.": + # Potentially restart failed processing here if needed + pass + + return existing_ref_law + + # Create initial Law record + new_law = Law( + reference=law_reference, + title=request.title or "Document en cours de traitement", + type="autre", # Changed from "unknown" to valid type + content="Document en cours de traitement par le système.", # Min 10 chars required + status="published", # Published so it appears in public list + category_id=request.category_id, # Category selected by admin + file_id=request.file_id, + original_filename=request.original_filename, + ) + + db.add(new_law) + await db.commit() + await db.refresh(new_law) + + # Process document synchronously (Windows-compatible) + # Celery on Windows has billiard issues, so we use direct synchronous processing + logger.info(f"🔄 Processing document synchronously for Law ID={new_law.id}") + + try: + import asyncio + + from app.tasks.process_law import process_law_sync + + # Run sync version in background thread to avoid blocking + result = await asyncio.to_thread(process_law_sync, new_law.id, request.file_id) + logger.info(f"✅ Synchronous processing completed: {result}") + + # Invalidate search cache to include new law in results + invalidate_search_cache() + except Exception as sync_error: + logger.error(f"❌ Synchronous processing failed: {sync_error}", exc_info=True) + logger.info(f"ℹ️ Law ID={new_law.id} created but processing failed") + + return new_law + + except HTTPException: + # Re-raise HTTPException as-is + raise + except Exception as e: + await db.rollback() + error_msg = f"Error starting ingestion: {type(e).__name__}: {str(e)}" + logger.error(f"❌ {error_msg}", exc_info=True) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error_msg) + + +# ==================== ADMIN ENDPOINTS ==================== + + +@router.post("/admin/laws", response_model=LawResponse, status_code=status.HTTP_201_CREATED) +async def create_law( + law: LawCreate, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_admin_user), +): + """ + Create a new law (admin only). + + **Auto-Detection (v2.1):** + - If `language` not provided, auto-detected from content + - Category suggestions provided based on content + + **Required Fields:** + - reference: Unique law reference + - title: Law title + - type: Law type (loi, décret, ordonnance, etc.) + - content: Full text content + + Args: + law: Law creation data + + Returns: + Created law with auto-detection results + + Raises: + 400: Validation error + 401: Unauthorized + 409: Law reference already exists + """ + assert law is not None, "LawCreate must not be None" + assert isinstance(law.reference, str) and len(law.reference) > 0, "Law reference must be a non-empty string" + + logger.info(f"➕ POST /admin/laws - Creating law: {law.title}") + + try: + # Check if reference already exists + existing_query = select(Law).where(Law.reference == law.reference) + existing_result = await db.execute(existing_query) + existing_law = existing_result.scalar_one_or_none() + + if existing_law: + logger.warning(f"⚠️ Law reference {law.reference} already exists") + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Law with reference '{law.reference}' already exists", + ) + + # Create law instance + new_law = Law(**law.model_dump()) + + # TODO: Integrate auto-detection services + # if not new_law.language: + # language_service = LanguageDetector() + # detection = language_service.detect(new_law.content) + # new_law.detected_language = detection["language"] + # new_law.language_confidence = detection["confidence"] + + # Add to database + db.add(new_law) + await db.commit() + await db.refresh(new_law) + + logger.info(f"✅ Law created: ID={new_law.id}, reference={new_law.reference}") + return new_law + + except HTTPException: + raise + except Exception as e: + await db.rollback() + logger.error(f"❌ Error creating law: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error creating law" + ) + + +@router.put("/admin/laws/{law_id}", response_model=LawResponse) +async def update_law( + law_id: int, + law_update: LawUpdate, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_admin_user), +): + """ + Update an existing law (admin only). + + **Partial Updates:** + - Only provided fields are updated + - Omitted fields remain unchanged + + Args: + law_id: Law ID to update + law_update: Fields to update + + Returns: + Updated law + + Raises: + 404: Law not found + 401: Unauthorized + """ + assert isinstance(law_id, int) and law_id > 0, "law_id must be a positive integer" + assert law_update is not None, "LawUpdate must not be None" + + logger.info(f"✏️ PUT /admin/laws/{law_id}") + + try: + # Fetch existing law + query = select(Law).where(Law.id == law_id) + result = await db.execute(query) + existing_law = result.scalar_one_or_none() + + if not existing_law: + logger.warning(f"⚠️ Law {law_id} not found") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f"Law with ID {law_id} not found" + ) + + # Update fields + update_data = law_update.model_dump(exclude_unset=True) + for field, value in update_data.items(): + setattr(existing_law, field, value) + + await db.commit() + await db.refresh(existing_law) + + logger.info(f"✅ Law {law_id} updated") + return existing_law + + except HTTPException: + raise + except Exception as e: + await db.rollback() + logger.error(f"❌ Error updating law {law_id}: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error updating law" + ) + + +@router.delete("/admin/laws/{law_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_law( + law_id: int, + db: AsyncSession = Depends(get_db), + current_user: dict = Depends(get_current_admin_user), +): + """ + Delete a law (admin only). + + **Cascade Delete:** + - Deletes law and all associated articles + - Removes from search index + + Args: + law_id: Law ID to delete + + Returns: + 204 No Content on success + + Raises: + 404: Law not found + 401: Unauthorized + """ + logger.info(f"🗑️ DELETE /admin/laws/{law_id}") + + try: + # Fetch law + query = select(Law).where(Law.id == law_id) + result = await db.execute(query) + law = result.scalar_one_or_none() + + if not law: + logger.warning(f"⚠️ Law {law_id} not found") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f"Law with ID {law_id} not found" + ) + + # Delete law (cascade deletes articles) + await db.delete(law) + await db.commit() + + # Remove from Meilisearch index + delete_from_meilisearch(law_id) + + # Invalidate search cache to prevent stale results + invalidate_search_cache() + + logger.info(f"✅ Law {law_id} deleted") + return None + + except HTTPException: + raise + except Exception as e: + await db.rollback() + logger.error(f"❌ Error deleting law {law_id}: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error deleting law" + ) diff --git a/app/api/routes/ocr.py b/app/api/routes/ocr.py new file mode 100644 index 0000000000000000000000000000000000000000..1fd0cf10bc81859c0e5e09969866e0e2f7164156 --- /dev/null +++ b/app/api/routes/ocr.py @@ -0,0 +1,206 @@ +""" +API endpoints for OCR operations. + +Provides REST endpoints for: +- PDF type detection (POST /api/v1/ocr/detect) +- OCR processing (POST /api/v1/ocr/process) +- Health check (GET /api/v1/ocr/health) + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +import logging +from pathlib import Path + +from fastapi import APIRouter, HTTPException, status +from fastapi.responses import JSONResponse + +from app.schemas.ocr import ( + OCRRequest, + OCRResult, + OCRServiceHealth, + PDFTypeDetectionRequest, + PDFTypeDetectionResult, +) +from app.services.ocr_service import OCRError, OCRService + +logger = logging.getLogger(__name__) + +# Create router +router = APIRouter(prefix="/api/v1/ocr", tags=["OCR"]) + +# Initialize service (singleton pattern) +_ocr_service: OCRService = None + + +def get_ocr_service() -> OCRService: + """ + Get or create OCRService instance. + + Returns: + OCRService instance + """ + global _ocr_service + if _ocr_service is None: + _ocr_service = OCRService( + default_language="fra+eng", + default_dpi=300, + ) + return _ocr_service + + +@router.post( + "/detect", + response_model=PDFTypeDetectionResult, + summary="Detect PDF type", + description=""" + Detect if a PDF is native (text-based), scanned (image-based), or hybrid. + + **Quick check** to determine if OCR processing is needed. + + **Returns:** + - PDF type classification + - Text coverage percentage + - OCR recommendation + """, +) +async def detect_pdf_type(request: PDFTypeDetectionRequest) -> PDFTypeDetectionResult: + """ + Detect PDF type without processing. + + Args: + request: Detection request with PDF path + + Returns: + PDFTypeDetectionResult with classification + + Raises: + 400: Invalid PDF path + 404: PDF file not found + 500: Detection error + """ + assert request is not None, "PDFTypeDetectionRequest must not be None" + assert isinstance(request.pdf_path, str) and len(request.pdf_path) > 0, "pdf_path must be a non-empty string" + + logger.info(f"🔍 PDF type detection request: {request.pdf_path}") + + pdf_path = Path(request.pdf_path) + + # Validate path + if not pdf_path.exists(): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"PDF file not found: {request.pdf_path}", + ) + + try: + service = get_ocr_service() + result = await service.detect_pdf_type(pdf_path) + + logger.info(f"✅ Detection complete: {result.pdf_type.value}") + return result + + except Exception as e: + logger.error(f"❌ Detection error: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"PDF type detection failed: {str(e)}", + ) + + +@router.post( + "/process", + response_model=OCRResult, + summary="Process PDF with OCR", + description=""" + Process a PDF document with OCR if needed. + + **Workflow:** + 1. Auto-detect PDF type + 2. Extract text (native or OCR) + 3. Clean and post-process text + + **Performance:** 5-30s depending on scan quality and page count. + + **Languages:** French (fra), English (eng), or both (fra+eng) + """, +) +async def process_pdf(request: OCRRequest) -> OCRResult: + """ + Process PDF with OCR. + + Args: + request: OCR request with PDF path and options + + Returns: + OCRResult with extracted text + + Raises: + 400: Invalid request + 404: PDF file not found + 500: Processing error + """ + assert request is not None, "OCRRequest must not be None" + assert isinstance(request.pdf_path, str) and len(request.pdf_path) > 0, "pdf_path must be a non-empty string" + + logger.info( + f"📄 OCR processing request: {request.pdf_path} " + f"(lang: {request.language}, dpi: {request.dpi})" + ) + + pdf_path = Path(request.pdf_path) + + # Validate path + if not pdf_path.exists(): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"PDF file not found: {request.pdf_path}", + ) + + try: + service = get_ocr_service() + result = await service.process_pdf( + pdf_path=pdf_path, + language=request.language, + dpi=request.dpi, + timeout_seconds=request.timeout_seconds, + ) + + logger.info( + f"✅ OCR complete: {result.text_length} chars " + f"({result.processing_time_ms}ms)" + ) + return result + + except OCRError as e: + logger.warning(f"⚠️ OCR error: {e}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(e), + ) + except Exception as e: + logger.error(f"❌ Processing error: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"OCR processing failed: {str(e)}", + ) + + +@router.get( + "/health", + response_model=OCRServiceHealth, + summary="Service health check", + description="Check the health status of the OCR service including Tesseract availability.", +) +async def health_check() -> OCRServiceHealth: + """ + Check service health. + + Returns: + Health status including Tesseract and Poppler availability + """ + service = get_ocr_service() + health = service.health_check() + + return OCRServiceHealth(**health) diff --git a/app/api/routes/personas.py b/app/api/routes/personas.py new file mode 100644 index 0000000000000000000000000000000000000000..2a585a5cb27531f5094ebeb8d1f08d701ca8460e --- /dev/null +++ b/app/api/routes/personas.py @@ -0,0 +1,708 @@ +""" +API Routes for Persona Management and Analytics (Service 9). + +Endpoints: +- GET /api/v1/personas - List all personas +- GET /api/v1/personas/{persona} - Get persona info +- GET /api/v1/personas/health - Health check +- GET /api/v1/personas/{persona}/stats - Get stats for persona +- GET /api/v1/personas/stats/all - Get stats for all personas +- GET /api/v1/personas/stats/usage - Get usage breakdown +- GET /api/v1/personas/{persona}/engagement - Get engagement metrics +- GET /api/v1/personas/{persona}/questions - Get popular questions +- GET /api/v1/personas/compare/{metric} - Compare personas +- GET /api/v1/personas/{persona}/trends/{metric} - Get trend data +- POST /api/v1/personas/feedback - Add message feedback +- GET /api/v1/personas/feedback/{message_id} - Get feedback +- GET /api/v1/personas/feedback/stats - Get feedback statistics + +Author: JuriX Team +Version: 2.1.0 +""" + +import logging +from typing import List, Dict, Optional, Any +from datetime import date +from fastapi import APIRouter, Depends, HTTPException, status, Query, Path +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.database import get_db +from app.services.persona_service import ( + PersonaService, + InvalidPersonaError, + MessageNotFoundError, + FeedbackAlreadyExistsError, + PersonaServiceError +) +from app.schemas.persona import ( + PersonaInfo, + PersonaStatsResponse, + MessageFeedbackCreate, + MessageFeedbackResponse, + FeedbackStats, + EngagementMetricsResponse, + PopularQuestionsResponse, + PopularQuestion, + PersonaComparisonResponse, + TrendsResponse, + TrendPoint, + UsageBreakdownResponse, + HealthCheckResponse, + ErrorResponse +) + +# Configure logger +logger = logging.getLogger(__name__) + +# Create router +router = APIRouter( + prefix="/api/v1/personas", + tags=["personas"], + responses={ + 400: {"description": "Bad Request - Invalid persona or validation error"}, + 404: {"description": "Not Found - Message or resource doesn't exist"}, + 409: {"description": "Conflict - Feedback already exists"}, + 500: {"description": "Internal Server Error"} + } +) + + +# ============================================================================ +# PERSONA INFORMATION ENDPOINTS +# ============================================================================ + +@router.get( + "", + response_model=List[PersonaInfo], + summary="List all available personas", + description="Get information about all 4 supported personas with metadata" +) +async def list_personas( + db: AsyncSession = Depends(get_db) +) -> List[PersonaInfo]: + """ + List all available personas. + + Returns metadata for all 4 personas: + - citoyen: Simplified responses for citizens + - avocat: Technical legal analysis for lawyers + - entrepreneur: Business-focused compliance advice + - étudiant: Pedagogical explanations for students + + **Returns:** + - List of persona info with name, display name, description, icon, tone, example questions + """ + try: + service = PersonaService(db) + personas = await service.list_personas() + logger.info(f"📋 Listed {len(personas)} personas") + return personas + except Exception as e: + logger.error(f"❌ Error listing personas: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) + + +@router.get( + "/{persona}", + response_model=PersonaInfo, + summary="Get info for specific persona", + responses={ + 404: {"model": ErrorResponse, "description": "Persona not found"} + } +) +async def get_persona_info( + persona: str = Path(..., description="Persona name (citoyen|avocat|entrepreneur|étudiant)"), + db: AsyncSession = Depends(get_db) +) -> PersonaInfo: + """ + Get information about a specific persona. + + **Parameters:** + - **persona**: Persona name (citoyen, avocat, entrepreneur, étudiant) + + **Returns:** + - Persona metadata including description, tone, and example questions + + **Raises:** + - 400: Invalid persona name + """ + assert isinstance(persona, str) and len(persona) > 0, "Persona must be a non-empty string" + + try: + service = PersonaService(db) + info = await service.get_persona_info(persona) + logger.info(f"📋 Retrieved info for persona: {persona}") + return info + except InvalidPersonaError as e: + logger.warning(f"❌ Invalid persona: {persona}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "error": "InvalidPersonaError", + "message": str(e), + "details": {"provided": persona, "valid": PersonaService.VALID_PERSONAS} + } + ) + except Exception as e: + logger.error(f"❌ Error getting persona info: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) + + +@router.get( + "/health", + response_model=HealthCheckResponse, + summary="Service health check" +) +def health_check( + db: AsyncSession = Depends(get_db) +) -> HealthCheckResponse: + """ + Check PersonaService health status. + + **Returns:** + - Service status, persona count, and timestamp + """ + try: + service = PersonaService(db) + health = service.health_check() + return health + except Exception as e: + logger.error(f"❌ Health check failed: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) + + +# ============================================================================ +# ANALYTICS - STATISTICS ENDPOINTS +# ============================================================================ + +@router.get( + "/{persona}/stats", + response_model=PersonaStatsResponse, + summary="Get statistics for a persona", + description="Retrieve usage, performance, engagement, and quality metrics for a specific persona" +) +async def get_persona_stats( + persona: str = Path(..., description="Persona name"), + start_date: Optional[str] = Query(None, description="Start date (YYYY-MM-DD), defaults to 30 days ago"), + end_date: Optional[str] = Query(None, description="End date (YYYY-MM-DD), defaults to today"), + db: AsyncSession = Depends(get_db) +) -> PersonaStatsResponse: + """ + Get statistics for a specific persona over a date range. + + Queries aggregated data from persona_stats table. + + **Parameters:** + - **persona**: Persona name + - **start_date**: Start date (ISO format), defaults to 30 days ago + - **end_date**: End date (ISO format), defaults to today + + **Returns:** + - Usage metrics (questions, conversations, sessions) + - Performance metrics (confidence, retrieval/generation time) + - Engagement metrics (messages per conversation, session duration) + - Quality metrics (helpful/unhelpful counts, satisfaction rate) + + **Raises:** + - 400: Invalid persona or date format + """ + assert isinstance(persona, str) and len(persona) > 0, "Persona must be a non-empty string" + assert start_date is None or isinstance(start_date, str), "Start date must be a string or None" + + try: + # Parse dates + start = date.fromisoformat(start_date) if start_date else None + end = date.fromisoformat(end_date) if end_date else None + + service = PersonaService(db) + stats = await service.get_persona_stats(persona, start, end) + logger.info(f"📊 Retrieved stats for {persona}") + return stats + except InvalidPersonaError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "InvalidPersonaError", "message": str(e)} + ) + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "ValidationError", "message": f"Invalid date format: {e}"} + ) + except Exception as e: + logger.error(f"❌ Error getting persona stats: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) + + +@router.get( + "/stats/all", + response_model=List[PersonaStatsResponse], + summary="Get statistics for all personas" +) +async def get_all_persona_stats( + start_date: Optional[str] = Query(None, description="Start date (YYYY-MM-DD)"), + end_date: Optional[str] = Query(None, description="End date (YYYY-MM-DD)"), + db: AsyncSession = Depends(get_db) +) -> List[PersonaStatsResponse]: + """ + Get statistics for all 4 personas. + + **Parameters:** + - **start_date**: Optional start date (ISO format) + - **end_date**: Optional end date (ISO format) + + **Returns:** + - List of stats for each persona + + **Raises:** + - 400: Invalid date format + """ + try: + start = date.fromisoformat(start_date) if start_date else None + end = date.fromisoformat(end_date) if end_date else None + + service = PersonaService(db) + all_stats = await service.get_all_persona_stats(start, end) + logger.info(f"📊 Retrieved stats for all personas") + return all_stats + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "ValidationError", "message": f"Invalid date format: {e}"} + ) + except Exception as e: + logger.error(f"❌ Error getting all persona stats: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) + + +@router.get( + "/stats/usage", + response_model=UsageBreakdownResponse, + summary="Get persona usage breakdown", + description="Get percentage breakdown of persona usage over last 30 days" +) +async def get_usage_breakdown( + db: AsyncSession = Depends(get_db) +) -> UsageBreakdownResponse: + """ + Get percentage breakdown of persona usage. + + Analyzes last 30 days of conversations. + + **Returns:** + - Total conversations across all personas + - Usage percentage for each persona + + """ + try: + service = PersonaService(db) + breakdown = await service.get_persona_usage_breakdown() + + # Calculate total from percentages (approximate) + total = sum(breakdown.values()) + + response = { + "total_conversations": int(total), + "percentages": breakdown + } + + logger.info(f"📊 Usage breakdown retrieved") + return response + except Exception as e: + logger.error(f"❌ Error getting usage breakdown: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) + + +# ============================================================================ +# ANALYTICS - ENGAGEMENT ENDPOINTS +# ============================================================================ + +@router.get( + "/{persona}/engagement", + response_model=EngagementMetricsResponse, + summary="Get engagement metrics for persona" +) +async def get_engagement_metrics( + persona: str = Path(..., description="Persona name"), + days: int = Query(30, ge=1, le=365, description="Number of days to analyze (1-365)"), + db: AsyncSession = Depends(get_db) +) -> EngagementMetricsResponse: + """ + Get engagement metrics for a persona over N days. + + **Parameters:** + - **persona**: Persona name + - **days**: Number of days to analyze (default 30, max 365) + + **Returns:** + - Total conversations and messages + - Average messages per conversation + - Average session duration + - Active days + - Conversations per active day + + **Raises:** + - 400: Invalid persona + """ + assert isinstance(persona, str) and len(persona) > 0, "Persona must be a non-empty string" + assert isinstance(days, int) and days > 0, "Days must be a positive integer" + + try: + service = PersonaService(db) + metrics = await service.get_engagement_metrics(persona, days) + logger.info(f"📊 Engagement metrics for {persona} over {days} days") + return metrics + except InvalidPersonaError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "InvalidPersonaError", "message": str(e)} + ) + except Exception as e: + logger.error(f"❌ Error getting engagement metrics: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) + + +@router.get( + "/{persona}/questions", + response_model=PopularQuestionsResponse, + summary="Get popular questions for persona" +) +async def get_popular_questions( + persona: str = Path(..., description="Persona name"), + limit: int = Query(10, ge=1, le=50, description="Max number of questions (1-50)"), + db: AsyncSession = Depends(get_db) +) -> PopularQuestionsResponse: + """ + Get most frequently asked questions for a persona. + + Analyzes last 90 days of questions. + + **Parameters:** + - **persona**: Persona name + - **limit**: Max number of questions to return (default 10, max 50) + + **Returns:** + - List of popular questions with frequency counts + + **Raises:** + - 400: Invalid persona + """ + assert isinstance(persona, str) and len(persona) > 0, "Persona must be a non-empty string" + assert isinstance(limit, int) and limit > 0, "Limit must be a positive integer" + + try: + service = PersonaService(db) + questions = await service.get_popular_questions(persona, limit) + + response = { + "persona": persona, + "questions": questions, + "total_count": len(questions) + } + + logger.info(f"📊 Popular questions for {persona}: {len(questions)} found") + return response + except InvalidPersonaError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "InvalidPersonaError", "message": str(e)} + ) + except Exception as e: + logger.error(f"❌ Error getting popular questions: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) + + +# ============================================================================ +# ANALYTICS - COMPARISON & TRENDS ENDPOINTS +# ============================================================================ + +@router.get( + "/compare/{metric}", + response_model=PersonaComparisonResponse, + summary="Compare personas by metric" +) +async def compare_personas( + metric: str = Path(..., description="Metric to compare (usage|confidence|satisfaction)"), + days: int = Query(30, ge=1, le=365, description="Number of days to analyze"), + db: AsyncSession = Depends(get_db) +) -> PersonaComparisonResponse: + """ + Compare all personas across a specific metric. + + **Parameters:** + - **metric**: Metric to compare (usage, confidence, satisfaction) + - **days**: Number of days to analyze (default 30) + + **Returns:** + - Comparison values for each persona + - Date range analyzed + + **Raises:** + - 400: Invalid metric + """ + try: + service = PersonaService(db) + comparison = await service.compare_personas(metric, days) + logger.info(f"📊 Compared personas by {metric} over {days} days") + return comparison + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "ValidationError", "message": str(e)} + ) + except Exception as e: + logger.error(f"❌ Error comparing personas: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) + + +@router.get( + "/{persona}/trends/{metric}", + response_model=TrendsResponse, + summary="Get trend data for persona" +) +async def get_trends( + persona: str = Path(..., description="Persona name"), + metric: str = Path(..., description="Metric to track (questions|confidence|satisfaction)"), + days: int = Query(30, ge=1, le=365, description="Number of days to analyze"), + db: AsyncSession = Depends(get_db) +) -> TrendsResponse: + """ + Get time-series trend data for a persona metric. + + **Parameters:** + - **persona**: Persona name + - **metric**: Metric to track (questions, confidence, satisfaction) + - **days**: Number of days to analyze (default 30) + + **Returns:** + - List of data points with date and value + - Metadata about persona, metric, and time range + + **Raises:** + - 400: Invalid persona or metric + """ + try: + service = PersonaService(db) + trends = await service.get_trends(persona, metric, days) + + response = { + "persona": persona, + "metric": metric, + "days": days, + "data_points": trends + } + + logger.info(f"📊 Trends for {persona} ({metric}) over {days} days: {len(trends)} points") + return response + except InvalidPersonaError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "InvalidPersonaError", "message": str(e)} + ) + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "ValidationError", "message": str(e)} + ) + except Exception as e: + logger.error(f"❌ Error getting trends: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) + + +# ============================================================================ +# FEEDBACK ENDPOINTS +# ============================================================================ + +@router.post( + "/feedback", + response_model=MessageFeedbackResponse, + status_code=status.HTTP_201_CREATED, + summary="Add message feedback", + responses={ + 404: {"model": ErrorResponse, "description": "Message not found"}, + 409: {"model": ErrorResponse, "description": "Feedback already exists"} + } +) +async def add_message_feedback( + feedback_data: MessageFeedbackCreate, + db: AsyncSession = Depends(get_db) +) -> MessageFeedbackResponse: + """ + Record user feedback on a message. + + Users can rate messages as helpful/unhelpful (👍/👎) with optional 1-5 star rating. + + **Parameters:** + - **message_id**: ID of message to rate + - **helpful**: Whether message was helpful (boolean) + - **rating**: Optional 1-5 star rating (integer) + - **comment**: Optional text comment (max 500 chars) + + **Returns:** + - Created feedback with ID and timestamp + + **Raises:** + - 404: Message not found + - 409: Feedback already exists for this message + - 400: Invalid rating (must be 1-5) + """ + try: + service = PersonaService(db) + feedback = await service.add_message_feedback( + message_id=feedback_data.message_id, + helpful=feedback_data.helpful, + rating=feedback_data.rating, + comment=feedback_data.comment + ) + logger.info(f"✅ Added feedback for message {feedback_data.message_id}") + return feedback + except MessageNotFoundError as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"error": "MessageNotFoundError", "message": str(e)} + ) + except FeedbackAlreadyExistsError as e: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={"error": "FeedbackAlreadyExistsError", "message": str(e)} + ) + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "ValidationError", "message": str(e)} + ) + except Exception as e: + logger.error(f"❌ Error adding feedback: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) + + +@router.get( + "/feedback/{message_id}", + response_model=Optional[MessageFeedbackResponse], + summary="Get feedback for message", + responses={ + 200: {"description": "Feedback found"}, + 404: {"description": "No feedback exists for this message"} + } +) +async def get_message_feedback( + message_id: int = Path(..., description="Message ID", gt=0), + db: AsyncSession = Depends(get_db) +) -> Optional[MessageFeedbackResponse]: + """ + Get feedback for a specific message. + + **Parameters:** + - **message_id**: ID of message + + **Returns:** + - Feedback data if exists, null otherwise + """ + try: + service = PersonaService(db) + feedback = await service.get_message_feedback(message_id) + + if not feedback: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"error": "NotFound", "message": f"No feedback found for message {message_id}"} + ) + + logger.info(f"📋 Retrieved feedback for message {message_id}") + return feedback + except HTTPException: + raise + except Exception as e: + logger.error(f"❌ Error getting feedback: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) + + +@router.get( + "/feedback/stats", + response_model=FeedbackStats, + summary="Get feedback statistics" +) +async def get_feedback_stats( + persona: Optional[str] = Query(None, description="Filter by persona"), + start_date: Optional[str] = Query(None, description="Start date (YYYY-MM-DD)"), + end_date: Optional[str] = Query(None, description="End date (YYYY-MM-DD)"), + db: AsyncSession = Depends(get_db) +) -> FeedbackStats: + """ + Get aggregated feedback statistics. + + **Parameters:** + - **persona**: Optional persona filter + - **start_date**: Optional start date (ISO format) + - **end_date**: Optional end date (ISO format) + + **Returns:** + - Total feedback count + - Helpful/unhelpful counts + - Satisfaction rate + - Average rating + - Breakdown by persona (if not filtered) + + **Raises:** + - 400: Invalid persona or date format + """ + try: + # Parse dates + start = date.fromisoformat(start_date) if start_date else None + end = date.fromisoformat(end_date) if end_date else None + + service = PersonaService(db) + stats = await service.get_feedback_stats(persona, start, end) + logger.info(f"📊 Feedback stats retrieved") + return stats + except InvalidPersonaError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "InvalidPersonaError", "message": str(e)} + ) + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "ValidationError", "message": f"Invalid date format: {e}"} + ) + except Exception as e: + logger.error(f"❌ Error getting feedback stats: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "InternalServerError", "message": str(e)} + ) diff --git a/app/api/routes/rag.py b/app/api/routes/rag.py new file mode 100644 index 0000000000000000000000000000000000000000..e630c102c4ecd3358dddd01cd61bf656c93bc1eb --- /dev/null +++ b/app/api/routes/rag.py @@ -0,0 +1,353 @@ +""" +API routes for RAG chatbot service. + +Provides endpoints for: +- ask: Standard RAG question answering +- ask/stream: Streaming RAG with SSE +- conversations: Conversation history management +- health: Service health check +""" + +import json +import logging + +from fastapi import APIRouter, Depends, HTTPException, status +from fastapi.responses import StreamingResponse +from sqlalchemy import desc, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import joinedload + +from app.core.database import get_db +from app.models.conversation import Conversation, Message +from app.schemas.rag import ( + ConversationResponse, + MessageResponse, + RAGRequest, + RAGResponse, +) +from app.services.rag_service import RAGService, RAGServiceError + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +def get_rag_service(db: AsyncSession = Depends(get_db)) -> RAGService: + """Dependency injection for RAGService.""" + return RAGService(db) + + +@router.post("/ask", response_model=RAGResponse, status_code=status.HTTP_200_OK) +async def ask( + request: RAGRequest, rag_service: RAGService = Depends(get_rag_service) +) -> RAGResponse: + """ + Ask a legal question and get RAG-powered answer. + + **Pipeline:** + 1. Retrieve top 5 relevant legal documents (<200ms) + 2. Load conversation history (last 5 messages) + 3. Generate persona-adapted answer with Mistral 7B (2-5s) + 4. Extract and validate source citations + 5. Calculate confidence score + 6. Save interaction to database + + **Performance:** <5s total (specification requirement) + + **Example Request:** + ```json + { + "question": "Quelle est la responsabilité des dirigeants de société?", + "persona": "avocat", + "language": "fr", + "session_id": "abc123", + "stream": false + } + ``` + + **Example Response:** + ```json + { + "answer": "Selon l'article 161 du Code OHADA...", + "confidence": 0.85, + "sources": [ + { + "law_id": 156, + "law_reference": "LOI-2024-001", + "law_title": "Code OHADA", + "article_number": "161", + "excerpt": "Les dirigeants sont responsables...", + "relevance_score": 0.92 + } + ], + "session_id": "abc123", + "retrieval_time_ms": 180, + "generation_time_ms": 2400, + "total_time_ms": 2600, + "persona": "avocat" + } + ``` + """ + assert request is not None, "RAGRequest must not be None" + assert isinstance(request.question, str) and len(request.question) > 0, "Question must be a non-empty string" + + try: + logger.info(f"📥 Ask request: persona={request.persona}") + + response = await rag_service.ask(request) + + logger.info(f"📤 Ask response: {response.total_time_ms}ms") + return response + + except RAGServiceError as e: + logger.error(f"❌ RAG error: {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + except Exception as e: + logger.error(f"❌ Unexpected error: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Erreur interne du serveur" + ) + + +@router.post("/ask/stream", status_code=status.HTTP_200_OK) +async def ask_stream(request: RAGRequest, rag_service: RAGService = Depends(get_rag_service)): + """ + Ask question with streaming response (SSE). + + **Benefits:** + - First token appears in <500ms (faster perceived response) + - User sees answer being generated in real-time + - Better UX for longer answers + + **Response Format:** Server-Sent Events (SSE) + + Each event is JSON: + ```json + {"chunk": "text fragment", "done": false} + {"chunk": "", "done": true, "sources": [...], "confidence": 0.85} + ``` + + **Example Usage (JavaScript):** + ```javascript + const eventSource = new EventSource('/api/v1/rag/ask/stream'); + eventSource.onmessage = (event) => { + const data = JSON.parse(event.data); + if (data.done) { + // Show sources and confidence + } else { + // Append chunk to answer + } + }; + ``` + """ + try: + # Force streaming mode + request.stream = True + + async def event_generator(): + try: + async for chunk_json in rag_service.ask_stream(request): + yield f"data: {chunk_json}\n\n" + except Exception as e: + logger.error(f"❌ Stream error: {e}") + error_json = json.dumps({"error": str(e), "done": True}) + yield f"data: {error_json}\n\n" + + return StreamingResponse( + event_generator(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", # Disable nginx buffering + }, + ) + + except Exception as e: + logger.error(f"❌ Streaming setup error: {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + +@router.get( + "/conversations/{session_id}", + response_model=ConversationResponse, + status_code=status.HTTP_200_OK, +) +async def get_conversation( + session_id: str, db: AsyncSession = Depends(get_db) +) -> ConversationResponse: + """ + Get conversation history by session ID. + + Returns all messages in chronological order with sources and metadata. + + **Example Response:** + ```json + { + "session_id": "abc123", + "persona": "citoyen", + "language": "fr", + "created_at": "2026-01-10T10:00:00", + "updated_at": "2026-01-10T10:05:00", + "messages": [ + { + "id": 1, + "role": "user", + "content": "Comment créer une entreprise?", + "sources": null, + "confidence": null, + "created_at": "2026-01-10T10:00:00" + }, + { + "id": 2, + "role": "assistant", + "content": "Pour créer une entreprise...", + "sources": [...], + "confidence": 0.85, + "created_at": "2026-01-10T10:00:05" + } + ] + } + ``` + """ + assert isinstance(session_id, str) and len(session_id) > 0, "session_id must be a non-empty string" + + try: + stmt = ( + select(Conversation) + .where(Conversation.session_id == session_id) + .options(joinedload(Conversation.messages)) + ) + result = await db.execute(stmt) + conversation = result.scalar_one_or_none() + + if not conversation: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f"Conversation {session_id} not found" + ) + + # Get messages in chronological order + msg_stmt = ( + select(Message) + .where(Message.conversation_id == conversation.id) + .order_by(Message.created_at) + ) + msg_result = await db.execute(msg_stmt) + messages = msg_result.scalars().all() + + return ConversationResponse( + session_id=conversation.session_id, + persona=conversation.persona, + language=conversation.language, + created_at=conversation.created_at, + updated_at=conversation.updated_at, + messages=[MessageResponse.model_validate(m) for m in messages], + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"❌ Error fetching conversation: {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + +@router.delete("/conversations/{session_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_conversation(session_id: str, db: AsyncSession = Depends(get_db)): + """ + Delete conversation and all associated messages. + + Useful for: + - User wants to start fresh + - Privacy/GDPR compliance + - Clear conversation context + + Returns 204 No Content on success. + """ + assert isinstance(session_id, str) and len(session_id) > 0, "session_id must be a non-empty string" + + try: + stmt = select(Conversation).where(Conversation.session_id == session_id) + result = await db.execute(stmt) + conversation = result.scalar_one_or_none() + + if not conversation: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f"Conversation {session_id} not found" + ) + + await db.delete(conversation) + await db.commit() + + logger.info(f"🗑️ Deleted conversation: {session_id}") + + except HTTPException: + raise + except Exception as e: + logger.error(f"❌ Error deleting conversation: {e}") + await db.rollback() + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + +@router.get("/health", status_code=status.HTTP_200_OK) +async def health_check(rag_service: RAGService = Depends(get_rag_service)) -> dict: + """ + Check health of RAG system components. + + Tests: + - LLM service availability (Gemini API) + - Database connectivity + - SearchService availability + + **Example Response:** + ```json + { + "status": "healthy", + "llm": "healthy", + "database": "connected", + "search_service": "healthy" + } + ``` + """ + health_status = { + "status": "healthy", + "llm": "not_configured", + "database": "unknown", + "search_service": "unknown", + } + + # Test LLM (Gemini - TODO: implement) + if rag_service.llm is not None: + try: + llm_health = await rag_service.llm.health_check() + health_status["llm"] = llm_health.get("status", "unknown") + if llm_health.get("status") != "healthy": + health_status["status"] = "degraded" + except Exception as e: + health_status["llm"] = f"error: {str(e)}" + health_status["status"] = "unhealthy" + else: + health_status["llm"] = "not_configured" + health_status["status"] = "degraded" + + # Test database + try: + await rag_service.db.execute(select(1)) + health_status["database"] = "connected" + except Exception as e: + health_status["database"] = f"error: {str(e)}" + health_status["status"] = "unhealthy" + + # Test SearchService + try: + # Simple test query + from app.schemas.search import SearchRequest + + test_response = await rag_service.search_service.search( + SearchRequest(query="test", mode="text", limit=1) + ) + health_status["search_service"] = "healthy" + except Exception as e: + health_status["search_service"] = f"error: {str(e)}" + health_status["status"] = "degraded" + + return health_status diff --git a/app/api/routes/search.py b/app/api/routes/search.py new file mode 100644 index 0000000000000000000000000000000000000000..db550869e9be168e397b79df3fac985c9b50a46f --- /dev/null +++ b/app/api/routes/search.py @@ -0,0 +1,668 @@ +""" +Routes API pour la recherche hybride. + +Endpoints: +- POST /api/v1/search - Recherche principale (text/semantic/hybrid) +- POST /api/v1/search/reindex - Réindexation complète (admin) +- GET /api/v1/search/stats - Statistiques recherche (admin) + +Author: JuriX Team +Version: 1.0.0 +""" + +import logging +import time +from typing import Optional + +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.database import get_db +from app.schemas.search import ( + ReindexResponse, + SearchRequest, + SearchResponse, + SearchStats, +) +from app.services.search_service import SearchService, SearchServiceError + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +# ==================== DEPENDENCIES ==================== + + +def get_search_service(db: AsyncSession = Depends(get_db)) -> SearchService: + """ + Dependency injection pour SearchService. + + Args: + db: Session database async + + Returns: + Instance SearchService + """ + return SearchService(db, use_cache=True) + + +# ==================== ENDPOINTS ==================== + + +@router.post("/", response_model=SearchResponse, status_code=status.HTTP_200_OK) +async def search( + request: SearchRequest, search_service: SearchService = Depends(get_search_service) +) -> SearchResponse: + """ + Point d'entrée principal pour la recherche. + + Effectue une recherche selon le mode spécifié: + - **text**: Recherche textuelle full-text (Meilisearch, <50ms) + - **semantic**: Recherche sémantique par similarité (pgvector, <200ms) + - **hybrid**: Combinaison optimale via RRF fusion (40/60, <200ms) + + **Filtres disponibles**: + - language: Code langue (fr/en) + - category_ids: Liste IDs catégories + - types: Types de lois + - status: Statut publication + - year_from/year_to: Plage années + + **Pagination**: + - limit: Nombre résultats (1-50, défaut: 15) + - offset: Offset pagination (défaut: 0) + + **Performance**: + - Text: <50ms + - Semantic: <200ms + - Hybrid: <200ms (spécification) + + Args: + request: SearchRequest avec query, mode, filters, pagination + search_service: Service injection + + Returns: + SearchResponse avec résultats et métadonnées + + Raises: + HTTPException 400: Si requête invalide + HTTPException 500: Si recherche échoue + + Example: + ```json + POST /api/v1/search + { + "query": "responsabilité dirigeants société", + "mode": "hybrid", + "filters": { + "language": "fr", + "category_ids": [1, 3] + }, + "limit": 15, + "offset": 0 + } + ``` + + Response: + ```json + { + "query": "responsabilité dirigeants société", + "mode": "hybrid", + "results": [ + { + "law_id": 156, + "reference": "LOI-2024-001", + "title": "Code des sociétés commerciales", + "relevance_score": 0.95, + "highlights": { + "title": "Code des sociétés commerciales", + "content": "...la responsabilité des dirigeants..." + } + } + ], + "total": 42, + "search_time_ms": 187, + "filters_applied": {"language": "fr", "category_ids": [1, 3]} + } + ``` + """ + assert request is not None, "SearchRequest must not be None" + assert isinstance(request.query, str) and len(request.query) > 0, "Query must be a non-empty string" + assert request.mode in ("text", "semantic", "hybrid"), f"Invalid search mode: {request.mode}" + + try: + logger.info(f'📥 Search request: mode={request.mode}, query="{request.query[:50]}"') + + # Exécution recherche + response = await search_service.search(request) + + logger.info(f"📤 Search response: {response.total} results in {response.search_time_ms}ms") + + return response + + except ValueError as e: + # Validation errors (query vide, mode invalide, etc.) + logger.warning(f"⚠️ Invalid search request: {e}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=f"Requête invalide: {str(e)}" + ) from e + + except SearchServiceError as e: + # Erreurs service (Meilisearch down, pgvector error, etc.) + logger.error(f"❌ Search service error: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Erreur lors de la recherche: {str(e)}", + ) from e + + except Exception as e: + # Erreurs inattendues + logger.error(f"❌ Unexpected error during search: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Erreur interne du serveur" + ) from e + + +@router.get("/suggest", status_code=status.HTTP_200_OK) +async def suggest( + q: str, + limit: int = 5, + search_service: SearchService = Depends(get_search_service) +) -> dict: + """ + Autocomplétion rapide pour suggestions de recherche. + + Utilise uniquement Meilisearch (mode text) pour des réponses ultra-rapides (<50ms). + Retourne titres et références de lois correspondant à la requête. + + Args: + q: Query de recherche (min 2 caractères) + limit: Nombre de suggestions (défaut: 5, max: 10) + + Returns: + Liste de suggestions avec titre, référence et ID + + Example: + GET /api/v1/search/suggest?q=const&limit=5 + + Response: + { + "suggestions": [ + {"id": 1, "title": "Constitution du Cameroun", "reference": "CONST-1996"}, + {"id": 42, "title": "Loi constitutionnelle", "reference": "LOI-2008-001"} + ], + "query": "const", + "search_time_ms": 12 + } + """ + assert isinstance(q, str), "Query must be a string" + assert isinstance(limit, int) and limit > 0, "Limit must be a positive integer" + + if len(q) < 2: + return {"suggestions": [], "query": q, "search_time_ms": 0} + + limit = min(limit, 10) # Max 10 suggestions + + try: + start_time = time.time() + + # Recherche Meilisearch directe (ultra-rapide) + index = search_service.meilisearch_client.index(SearchService.MEILISEARCH_INDEX) + + results = index.search( + q, + { + "limit": limit, + "attributesToRetrieve": ["id", "title", "reference"], + "attributesToHighlight": ["title"], + "highlightPreTag": "", + "highlightPostTag": "" + } + ) + + search_time_ms = int((time.time() - start_time) * 1000) + + suggestions = [ + { + "id": hit.get("id"), + "title": hit.get("_formatted", {}).get("title", hit.get("title", "")), + "reference": hit.get("reference", "") + } + for hit in results.get("hits", []) + ] + + logger.debug(f"📋 Suggest: {len(suggestions)} results for '{q}' in {search_time_ms}ms") + + return { + "suggestions": suggestions, + "query": q, + "search_time_ms": search_time_ms + } + + except Exception as e: + logger.warning(f"⚠️ Suggest error: {e}") + return {"suggestions": [], "query": q, "search_time_ms": 0, "error": str(e)} + + +def _parse_article_reference(query: str) -> tuple: + """ + Parse article number and document hint from a query string. + + Supports French (article, art.) and English (section) patterns. + Normalizes 'premier', '1er', 'first' etc. to '1'. + + Args: + query: Lowercased query string + + Returns: + (article_num: str | None, doc_hint: str | None) + """ + import re + + patterns = [ + # "article 5 de la constitution" + r"article\s+(premier|1er|1ère|\d+)\s+(?:de\s+)?(?:la\s+|le\s+|l[''']|du\s+|des\s+)?(.+)", + # "art. 12 de la loi" + r"art\.?\s*(\d+)\s+(?:de\s+)?(?:la\s+|le\s+|l[''']|du\s+)?(.+)", + # English: "section 5" + r"section\s+(\d+|one|first)\s+(?:of\s+)?(?:the\s+)?(.+)", + ] + + for pattern in patterns: + match = re.search(pattern, query, re.IGNORECASE) + if match: + article_num = match.group(1).strip() + doc_hint = match.group(2).strip() if match.group(2) else "" + + # Normalize + if article_num.lower() in ("premier", "1er", "1ère", "first", "one"): + article_num = "1" + return article_num, doc_hint + + return None, None + + +def _estimate_page_number(article) -> int: + """ + Estimate the PDF page number where an article appears. + + Uses stored page_number if available; otherwise estimates from + the article number (roughly 2 articles per page). + + Args: + article: Article ORM object with .page_number and .number + + Returns: + Estimated page number (>= 2) + """ + if article.page_number is not None: + return article.page_number + + try: + art_num = int(article.number.replace("er", "").replace("ère", "").strip()) + return max(2, 1 + (art_num // 2)) + except ValueError: + return 2 # Default to page 2 + + +@router.get("/article", status_code=status.HTTP_200_OK) +async def find_article( + q: str, + db: AsyncSession = Depends(get_db) +) -> dict: + """ + Recherche rapide d'article pour navigation directe (<2s). + + Détecte les références d'articles dans la query et retourne: + - law_id: ID du document contenant l'article + - article_num: Numéro de l'article trouvé + - direct_url: URL pour navigation directe + + Performance cible: <500ms + + Args: + q: Query contenant une référence d'article (ex: "article 5 de la constitution") + + Returns: + Résultat avec law_id et article_num pour navigation directe + """ + from sqlalchemy import select, or_ + from app.models.law import Law, Article + + assert isinstance(q, str) and len(q) > 0, "Query must be a non-empty string" + + start_time = time.time() + + # Parse article reference from query + article_num, doc_hint = _parse_article_reference(q.lower().strip()) + + if not article_num: + return { + "found": False, + "error": "Aucune référence d'article détectée", + "query": q, + "search_time_ms": int((time.time() - start_time) * 1000) + } + + try: + # Search for the law by title hint + law_query = select(Law).where(Law.status == "published") + + if doc_hint: + law_query = law_query.where( + or_( + Law.title.ilike(f"%{doc_hint}%"), + Law.reference.ilike(f"%{doc_hint}%") + ) + ) + + result = await db.execute(law_query.limit(5)) + laws = result.scalars().all() + + # Find article in these laws + for law in laws: + article_query = select(Article).where( + Article.law_id == law.id, + Article.number == article_num + ).limit(1) + + article_result = await db.execute(article_query) + article = article_result.scalar_one_or_none() + + if article: + search_time_ms = int((time.time() - start_time) * 1000) + logger.info(f"🎯 Found article {article.number} in law {law.id} in {search_time_ms}ms") + + page_number = _estimate_page_number(article) + + return { + "found": True, + "law_id": law.id, + "law_title": law.title, + "law_reference": law.reference, + "article_num": article.number, + "article_id": article.id, + "article_order": article.order, + "estimated_page": page_number, + "direct_url": f"/laws/{law.id}?article={article.number}&page={page_number}", + "query": q, + "search_time_ms": search_time_ms + } + + # Article not found + search_time_ms = int((time.time() - start_time) * 1000) + return { + "found": False, + "error": f"Article {article_num} non trouvé dans '{doc_hint or 'tous les documents'}'", + "query": q, + "search_time_ms": search_time_ms + } + + except Exception as e: + logger.error(f"❌ Article search error: {e}") + return { + "found": False, + "error": str(e), + "query": q, + "search_time_ms": int((time.time() - start_time) * 1000) + } + + + +@router.post( + "/reindex", + response_model=ReindexResponse, + status_code=status.HTTP_200_OK, + # TODO: Add admin authentication dependency + # dependencies=[Depends(require_admin)] +) +async def reindex_all( + background_tasks: BackgroundTasks, search_service: SearchService = Depends(get_search_service) +) -> ReindexResponse: + """ + Réindexe toutes les lois publiées dans Meilisearch (admin uniquement). + + **Opération longue** (~5s pour 100 lois): + - Récupère toutes les lois publiées depuis DB + - Supprime ancien index Meilisearch + - Recrée index avec configuration optimale + - Indexe documents en batch (1000 par batch) + + **Configuration index**: + - Searchable: title, reference, content + - Filterable: id, category_id, type, language, status, year + - Sortable: publication_year, created_at + - Typo tolerance: 1-2 typos selon longueur + - Ranking: words > typo > proximity > attribute > sort > exactness + + **Quand utiliser**: + - Après changements massifs de contenu + - Après mise à jour configuration Meilisearch + - Résolution corruption index + - Tests/développement + + **⚠️ Attention**: + - Recherche indisponible pendant réindexation + - Opération CPU/IO intensive + - Invalidation cache nécessaire + + Args: + background_tasks: FastAPI background tasks (future async support) + search_service: Service injection + + Returns: + ReindexResponse avec statistiques (total, indexed, failed, duration) + + Raises: + HTTPException 500: Si réindexation échoue + + Example: + ``` + POST /api/v1/search/reindex + Authorization: Bearer + ``` + + Response: + ```json + { + "status": "success", + "total_laws": 156, + "indexed_count": 156, + "failed_count": 0, + "duration_seconds": 4 + } + ``` + """ + try: + logger.info("🔄 Starting full reindex (admin operation)...") + + # Exécution réindexation (blocking pour l'instant) + # TODO: Implémenter via background_tasks pour async + response = await search_service.reindex_all_laws() + + logger.info( + f"✅ Reindex complete: {response.indexed_count}/{response.total_laws} " + f"in {response.duration_seconds}s" + ) + + return response + + except SearchServiceError as e: + logger.error(f"❌ Reindex failed: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Échec réindexation: {str(e)}", + ) from e + + except Exception as e: + logger.error(f"❌ Unexpected error during reindex: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Erreur interne du serveur" + ) from e + + +@router.get( + "/stats", + response_model=SearchStats, + status_code=status.HTTP_200_OK, + # TODO: Add admin authentication dependency + # dependencies=[Depends(require_admin)] +) +async def get_search_stats( + search_service: SearchService = Depends(get_search_service), +) -> SearchStats: + """ + Récupère les statistiques de recherche (admin uniquement). + + **Métriques fournies**: + - Nombre total de documents indexés + - Répartition par langue (fr/en) + - Répartition par catégorie + - Répartition par statut + - État de santé index Meilisearch + - État cache Redis + + **Cas d'usage**: + - Monitoring santé système recherche + - Analytics utilisation + - Debugging problèmes indexation + - Validation après réindexation + + Args: + search_service: Service injection + + Returns: + SearchStats avec métriques système + + Raises: + HTTPException 500: Si récupération stats échoue + + Example: + ``` + GET /api/v1/search/stats + Authorization: Bearer + ``` + + Response: + ```json + { + "total_documents": 156, + "by_language": {"fr": 142, "en": 14}, + "by_category": {"Droit Civil": 45, "Droit Pénal": 38, ...}, + "by_status": {"published": 156, "draft": 0}, + "index_health": "healthy", + "cache_status": "connected" + } + ``` + """ + try: + logger.info("📊 Fetching search stats (admin operation)...") + + # TODO: Implémenter récupération stats depuis Meilisearch + DB + # Pour l'instant retourne mock data + + # Récupération stats Meilisearch + index = search_service.meilisearch_client.index(SearchService.MEILISEARCH_INDEX) + + try: + index_stats = index.get_stats() + total_docs = index_stats.get("numberOfDocuments", 0) + except Exception: + total_docs = 0 + + # Mock stats (TODO: Vraies stats depuis DB) + stats = SearchStats( + total_documents=total_docs, + by_language={"fr": 0, "en": 0}, # TODO: Query DB + by_category={}, # TODO: Query DB + by_status={"published": total_docs}, + index_health="healthy" if total_docs > 0 else "empty", + cache_status=("connected" if search_service.redis_client else "disabled"), + ) + + logger.info(f"✅ Stats retrieved: {total_docs} documents indexed") + + return stats + + except Exception as e: + logger.error(f"❌ Failed to get stats: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Erreur lors de la récupération des statistiques", + ) from e + + +# ==================== HEALTH CHECK ==================== + + +@router.get("/health", status_code=status.HTTP_200_OK) +async def health_check(search_service: SearchService = Depends(get_search_service)) -> dict: + """ + Vérifie l'état de santé du service de recherche. + + Teste: + - Connexion Meilisearch + - Existence index + - Connexion Redis (si cache activé) + - Disponibilité EmbeddingService + + Returns: + Statut de santé avec détails composants + + Example: + ``` + GET /api/v1/search/health + ``` + + Response: + ```json + { + "status": "healthy", + "meilisearch": "connected", + "redis": "connected", + "embedding_service": "ready" + } + ``` + """ + health_status = { + "status": "healthy", + "meilisearch": "unknown", + "redis": "unknown", + "embedding_service": "unknown", + } + + # Test Meilisearch + try: + search_service.meilisearch_client.health() + health_status["meilisearch"] = "connected" + except Exception as e: + health_status["meilisearch"] = f"error: {str(e)}" + health_status["status"] = "degraded" + + # Test Redis + if search_service.redis_client: + try: + search_service.redis_client.ping() + health_status["redis"] = "connected" + except Exception as e: + health_status["redis"] = f"error: {str(e)}" + health_status["status"] = "degraded" + else: + health_status["redis"] = "disabled" + + # Test EmbeddingService + if search_service.embedding_service is not None: + try: + emb_health = search_service.embedding_service.health_check() + health_status["embedding_service"] = emb_health["status"] + if emb_health["status"] != "healthy": + health_status["status"] = "degraded" + except Exception as e: + health_status["embedding_service"] = f"error: {str(e)}" + health_status["status"] = "degraded" + else: + health_status["embedding_service"] = "unavailable (text search only)" + health_status["status"] = "degraded" + + return health_status diff --git a/app/api/routes/upload.py b/app/api/routes/upload.py new file mode 100644 index 0000000000000000000000000000000000000000..0888805bd3cd63db71077aa7d77a3b51d4ad74cb --- /dev/null +++ b/app/api/routes/upload.py @@ -0,0 +1,248 @@ +""" +API endpoints for file upload operations. + +Provides REST endpoints for: +- File upload (POST /api/v1/upload) +- Upload status (GET /api/v1/upload/{file_id}) +- File deletion (DELETE /api/v1/upload/{file_id}) +- Health check (GET /api/v1/upload/health) + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +import logging +from pathlib import Path +from typing import Dict + +from fastapi import APIRouter, File, HTTPException, UploadFile, status +from fastapi.responses import JSONResponse + +from app.schemas.file_upload import FileUploadResult, UploadServiceHealth +from app.services.file_upload_service import FileUploadError, FileUploadService + +logger = logging.getLogger(__name__) + +# Create router +router = APIRouter(prefix="/api/v1/upload", tags=["File Upload"]) + +# Initialize service (singleton pattern) +# In production, this should be dependency-injected +_upload_service: FileUploadService = None + + +def get_upload_service() -> FileUploadService: + """ + Get or create FileUploadService instance. + + Returns: + FileUploadService instance + """ + global _upload_service + if _upload_service is None: + # Ensure storage directory exists + storage_path = Path("./data/uploads") + storage_path.mkdir(parents=True, exist_ok=True) + + # TODO: Load from environment variables + _upload_service = FileUploadService( + storage_path="./data/uploads", + max_size_mb=250, + allowed_formats=("pdf", "docx"), + clamav_enabled=False, # Mock by default + cleanup_hours=24, + ) + return _upload_service + + +@router.post( + "", + response_model=FileUploadResult, + status_code=status.HTTP_201_CREATED, + summary="Upload a file", + description=""" + Upload a PDF or DOCX file for processing. + + **Validations:** + - Maximum file size: 50 MB + - Allowed formats: PDF, DOCX + - File structure validation + - Antivirus scan + + **Returns:** + - File ID for tracking + - Validation results + - Extracted metadata + - Scan results + """, +) +async def upload_file( + file: UploadFile = File(..., description="File to upload (PDF or DOCX, max 50 MB)"), +) -> FileUploadResult: + """ + Upload a file with validation and scanning. + + Args: + file: Uploaded file + + Returns: + FileUploadResult with complete upload information + + Raises: + 400: Invalid file format or validation failed + 413: File too large + 500: Internal server error + """ + assert file is not None, "UploadFile must not be None" + assert file.filename is not None and len(file.filename) > 0, "Filename must not be empty" + + logger.info(f"📤 Upload request: {file.filename} ({file.content_type})") + + try: + service = get_upload_service() + result = await service.upload_file(file) + + logger.info(f"✅ Upload successful: {result.file_id}") + return result + + except FileUploadError as e: + logger.warning(f"⚠️ Upload validation failed: {e}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(e), + ) + except Exception as e: + logger.error(f"❌ Upload error: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal server error during file upload", + ) + + +@router.get( + "/{file_id}", + response_model=Dict, + summary="Get upload status", + description="Retrieve information about an uploaded file by its ID.", +) +async def get_upload_status(file_id: str) -> Dict: + """ + Get status and metadata of an uploaded file. + + Args: + file_id: File identifier + + Returns: + File information + + Raises: + 404: File not found + """ + assert isinstance(file_id, str) and len(file_id) > 0, "file_id must be a non-empty string" + + service = get_upload_service() + + # Check if file exists + file_path = service.storage_path / f"{file_id}.pdf" + if not file_path.exists(): + file_path = service.storage_path / f"{file_id}.docx" + + if not file_path.exists(): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"File not found: {file_id}", + ) + + # Return basic file info + return { + "file_id": file_id, + "exists": True, + "file_size": file_path.stat().st_size, + "storage_path": str(file_path), + } + + +@router.delete( + "/{file_id}", + response_model=Dict, + summary="Delete uploaded file", + description="Delete an uploaded file from temporary storage.", +) +async def delete_upload(file_id: str) -> Dict: + """ + Delete an uploaded file. + + Args: + file_id: File identifier + + Returns: + Success confirmation + + Raises: + 404: File not found + """ + assert isinstance(file_id, str) and len(file_id) > 0, "file_id must be a non-empty string" + + service = get_upload_service() + + # Try to find and delete file + deleted = False + for ext in [".pdf", ".docx"]: + file_path = service.storage_path / f"{file_id}{ext}" + if file_path.exists(): + file_path.unlink() + deleted = True + logger.info(f"🗑️ Deleted file: {file_id}{ext}") + break + + if not deleted: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"File not found: {file_id}", + ) + + return {"success": True, "file_id": file_id, "message": "File deleted successfully"} + + +@router.get( + "/health", + response_model=UploadServiceHealth, + summary="Service health check", + description="Check the health status of the file upload service.", +) +async def health_check() -> UploadServiceHealth: + """ + Check service health. + + Returns: + Health status including storage and scanner information + """ + service = get_upload_service() + health = service.health_check() + + return UploadServiceHealth(**health) + + +@router.post( + "/cleanup", + response_model=Dict, + summary="Cleanup old files", + description="Remove files older than specified hours (default: 24h).", +) +async def cleanup_old_files(max_age_hours: int = 24) -> Dict: + """ + Cleanup old uploaded files. + + Args: + max_age_hours: Maximum age in hours (default: 24) + + Returns: + Cleanup statistics + """ + assert isinstance(max_age_hours, int) and max_age_hours > 0, "max_age_hours must be a positive integer" + + service = get_upload_service() + stats = await service.cleanup_old_files(max_age_hours) + + logger.info(f"🧹 Cleanup completed: {stats['deleted_count']} files deleted") + return stats diff --git a/app/core/__init__.py b/app/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/app/core/auth.py b/app/core/auth.py new file mode 100644 index 0000000000000000000000000000000000000000..ea769e4717a4517b9e386ae85bbf7757c05e1995 --- /dev/null +++ b/app/core/auth.py @@ -0,0 +1,188 @@ +""" +Authentication utilities for JuriX. + +Provides JWT token generation, password hashing, and user authentication. + +Author: JuriX Team +Date: 2026-01-12 +""" + +from datetime import datetime, timedelta +from typing import Optional + +from fastapi import Depends, HTTPException, status +from fastapi.security import OAuth2PasswordBearer +from jose import JWTError, jwt +from passlib.context import CryptContext +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import settings +from app.core.database import get_db +from app.models.user import User + +# Password hashing +pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") + +# OAuth2 scheme +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/v1/auth/login") + +# JWT settings +SECRET_KEY = ( + settings.SECRET_KEY + if hasattr(settings, "SECRET_KEY") + else "dev-secret-key-change-in-production" +) +ALGORITHM = "HS256" +ACCESS_TOKEN_EXPIRE_MINUTES = 60 * 24 * 7 # 7 days + + +def hash_password(password: str) -> str: + """ + Hash a password using bcrypt. + + Args: + password: Plain text password + + Returns: + Hashed password + """ + return pwd_context.hash(password) + + +def verify_password(plain_password: str, hashed_password: str) -> bool: + """ + Verify a password against its hash. + + Args: + plain_password: Plain text password to verify + hashed_password: Bcrypt hash to verify against + + Returns: + True if password matches, False otherwise + """ + return pwd_context.verify(plain_password, hashed_password) + + +def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -> str: + """ + Create a JWT access token. + + Args: + data: Data to encode in token (typically {"sub": user_email}) + expires_delta: Optional expiration time delta + + Returns: + Encoded JWT token + """ + to_encode = data.copy() + if expires_delta: + expire = datetime.utcnow() + expires_delta + else: + expire = datetime.utcnow() + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES) + + to_encode.update({"exp": expire}) + encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM) + return encoded_jwt + + +async def get_current_user( + token: str = Depends(oauth2_scheme), db: AsyncSession = Depends(get_db) +) -> User: + """ + Get current authenticated user from JWT token. + + Args: + token: JWT token from Authorization header + db: Database session + + Returns: + Authenticated User object + + Raises: + HTTPException: If token is invalid or user not found + """ + credentials_exception = HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Could not validate credentials", + headers={"WWW-Authenticate": "Bearer"}, + ) + + try: + payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) + email: str = payload.get("sub") + if email is None: + raise credentials_exception + except JWTError: + raise credentials_exception + + # Get user from database + query = select(User).where(User.email == email) + result = await db.execute(query) + user = result.scalar_one_or_none() + + if user is None: + raise credentials_exception + + if not user.is_active: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail="User account is inactive" + ) + + return user + + +async def get_current_active_user(current_user: User = Depends(get_current_user)) -> User: + """ + Get current active user. + + Args: + current_user: Current user from token + + Returns: + Active user + + Raises: + HTTPException: If user is inactive + """ + if not current_user.is_active: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Inactive user") + return current_user + + +async def get_current_admin_user(current_user: User = Depends(get_current_user)) -> User: + """ + Get current user with admin role. + + Args: + current_user: Current user from token + + Returns: + Admin user + + Raises: + HTTPException: If user doesn't have admin role + """ + if not current_user.is_admin(): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required") + return current_user + + +async def get_current_superadmin_user(current_user: User = Depends(get_current_user)) -> User: + """ + Get current user with superadmin role. + + Args: + current_user: Current user from token + + Returns: + Superadmin user + + Raises: + HTTPException: If user doesn't have superadmin role + """ + if not current_user.is_superadmin(): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail="Superadmin access required" + ) + return current_user diff --git a/app/core/celery_app.py b/app/core/celery_app.py new file mode 100644 index 0000000000000000000000000000000000000000..84d71936a635a46d22c8d3107c3b0c687342ef6e --- /dev/null +++ b/app/core/celery_app.py @@ -0,0 +1,50 @@ +""" +Celery application configuration for JuriX. + +Configures Celery with Redis broker for asynchronous task processing. + +Usage: + # Start worker + celery -A app.core.celery_app worker --loglevel=info + + # Monitor tasks + celery -A app.core.celery_app flower + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +import logging +import os + +from celery import Celery + +from app.core.config import settings + +os.environ.setdefault("FORKED_BY_MULTIPROCESSING", "1") + +logger = logging.getLogger(__name__) + +# Create Celery app with Redis broker +celery_app = Celery( + "jurix", + broker=settings.REDIS_URL, + backend=settings.REDIS_URL, +) + +# Celery configuration +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + task_track_started=True, + task_time_limit=3600, # 1 hour max + task_soft_time_limit=3000, # 50 minutes soft limit + worker_prefetch_multiplier=1, # One task at a time + worker_max_tasks_per_child=1000, # Restart worker after 1000 tasks + include=["app.tasks.process_law"], # Explicitly include task modules +) + +logger.info("✅ Celery app configured") diff --git a/app/core/config.py b/app/core/config.py new file mode 100644 index 0000000000000000000000000000000000000000..4a20ee115709a15de3d7c12762e5f490538b3d57 --- /dev/null +++ b/app/core/config.py @@ -0,0 +1,70 @@ +"""Configuration application - Toutes les variables d'environnement.""" + +from pydantic_settings import BaseSettings +from typing import Optional + + +class Settings(BaseSettings): + """Settings de l'application.""" + + # App + APP_NAME: str = "JuriX API" + VERSION: str = "2.1.0" + DEBUG: bool = True + ENVIRONMENT: str = "development" + + # Database + DATABASE_URL: str = "postgresql+asyncpg://jurix:jurix_dev_password_change_in_prod@localhost:5432/jurix_db" + + # Redis + REDIS_URL: str = "redis://:redis_dev_password_change_in_prod@localhost:6379/0" + + # Meilisearch + MEILISEARCH_URL: str = "http://localhost:7700" + MEILISEARCH_KEY: str = "meilisearch_master_key_dev_change_in_prod" + + # Gemini API (LLM for RAG) + GEMINI_API_KEY: str = "" + GEMINI_MODEL: str = "gemini-3-flash" + GEMINI_EMBEDDING_MODEL: str = "models/gemini-embedding-001" # native 3072-dim embeddings + + # LlamaParse (PDF extraction) + LLAMA_CLOUD_API_KEY: str = "" + + # CORS — comma-separated list of extra allowed origins for production + # Example: https://jurix.vercel.app,https://www.jurix.cm + ALLOWED_ORIGINS: str = "" + + # Security + SECRET_KEY: str = "dev_secret_key_change_in_production_with_openssl_rand_hex_32" + ALGORITHM: str = "HS256" + ACCESS_TOKEN_EXPIRE_MINUTES: int = 30 + + # Qodo (Tests) + QODO_API_KEY: Optional[str] = None + + # Zerostep (Tests E2E) + ZEROSTEP_API_KEY: Optional[str] = None + + # CORS + CORS_ORIGINS: list[str] = [ + "http://localhost:5173", + "http://127.0.0.1:5173", + "http://localhost:8000", + "http://127.0.0.1:8000", + "*" # Keep wildcard for flexibility if needed, but explicit ones usually take precedence + ] + + # Upload limits + MAX_UPLOAD_SIZE: int = 1073741824 # 1GB in bytes for batch upload + + # OCR + TESSERACT_PATH: str = r"C:\Program Files\Tesseract-OCR\tesseract.exe" + + class Config: + env_file = ".env" + case_sensitive = True + extra = "ignore" # Ignore extra fields from .env + + +settings = Settings() diff --git a/app/core/database.py b/app/core/database.py new file mode 100644 index 0000000000000000000000000000000000000000..a2861e5fb89bb1ebcb8c096e18abfecc8cd27938 --- /dev/null +++ b/app/core/database.py @@ -0,0 +1,160 @@ +""" +Gestion des sessions de base de données asynchrones. + +Ce module configure SQLAlchemy 2.0 avec sessions asynchrones pour PostgreSQL. +Utilise le pattern AsyncSession pour toutes les opérations DB. + +Usage dans routes FastAPI: + @router.get("/laws") + async def list_laws(db: AsyncSession = Depends(get_db)): + service = LawService(db) + return await service.list_laws() + +Author: JuriX Team +""" + +import logging +from typing import AsyncGenerator + +from sqlalchemy.ext.asyncio import ( + AsyncSession, + create_async_engine, + async_sessionmaker, +) +from sqlalchemy.orm import DeclarativeBase + +from app.core.config import settings + +logger = logging.getLogger(__name__) + + +# Base class for all SQLAlchemy models +class Base(DeclarativeBase): + """Base class pour tous les modèles SQLAlchemy.""" + pass + + +# Create async engine +engine = create_async_engine( + settings.DATABASE_URL, + echo=settings.DEBUG if hasattr(settings, 'DEBUG') else False, # Log SQL queries in debug mode + pool_size=20, # Connection pool size + max_overflow=10, # Additional connections beyond pool_size + pool_pre_ping=True, # Verify connections before using + pool_recycle=3600, # Recycle connections after 1 hour +) + +# Create async session factory +AsyncSessionLocal = async_sessionmaker( + engine, + class_=AsyncSession, + expire_on_commit=False, # Don't expire objects after commit + autoflush=False, # Manual flush control + autocommit=False, # Manual commit control +) + + +async def get_db() -> AsyncGenerator[AsyncSession, None]: + """ + Dependency pour obtenir une session DB asynchrone. + + Crée une nouvelle session pour chaque requête et garantit + sa fermeture propre même en cas d'erreur. + + Yields: + AsyncSession: Session de base de données asynchrone + + Example: + >>> @router.get("/laws") + >>> async def list_laws(db: AsyncSession = Depends(get_db)): + ... service = LawService(db) + ... return await service.list_laws() + """ + async with AsyncSessionLocal() as session: + try: + logger.debug("📦 Nouvelle session DB créée") + yield session + except Exception as e: + logger.error(f"❌ Erreur session DB: {e}") + await session.rollback() + raise + finally: + await session.close() + logger.debug("🗑️ Session DB fermée") + + +async def init_db() -> None: + """ + Initialise les tables de la base de données. + + Crée toutes les tables définies dans les modèles SQLAlchemy + si elles n'existent pas déjà. + + Note: + En production, utiliser Alembic pour les migrations. + Cette fonction est principalement pour les tests. + + Example: + >>> # Dans les tests + >>> await init_db() + """ + async with engine.begin() as conn: + logger.info("🚀 Initialisation des tables de la base de données...") + await conn.run_sync(Base.metadata.create_all) + logger.info("✅ Tables créées avec succès") + + +async def drop_db() -> None: + """ + Supprime toutes les tables de la base de données. + + ⚠️ ATTENTION: Cette fonction est destructive! + Utilisée uniquement pour les tests. + + Example: + >>> # Nettoyage après tests + >>> await drop_db() + """ + async with engine.begin() as conn: + logger.warning("⚠️ Suppression de toutes les tables...") + await conn.run_sync(Base.metadata.drop_all) + logger.info("✅ Tables supprimées") + + +async def close_db() -> None: + """ + Ferme le moteur de base de données. + + Ferme toutes les connexions actives et libère les ressources. + Doit être appelé lors de l'arrêt de l'application. + + Example: + >>> # Dans le shutdown event de FastAPI + >>> @app.on_event("shutdown") + >>> async def shutdown(): + ... await close_db() + """ + await engine.dispose() + logger.info("✅ Moteur de base de données fermé") + + +async def health_check_db() -> bool: + """ + Vérifie la santé de la connexion à la base de données. + + Returns: + bool: True si la connexion fonctionne, False sinon + + Example: + >>> is_healthy = await health_check_db() + >>> if not is_healthy: + ... logger.error("DB connection failed") + """ + try: + async with AsyncSessionLocal() as session: + await session.execute("SELECT 1") + logger.debug("✅ DB health check OK") + return True + except Exception as e: + logger.error(f"❌ DB health check failed: {e}") + return False diff --git a/app/core/dependencies.py b/app/core/dependencies.py new file mode 100644 index 0000000000000000000000000000000000000000..46d8c0e7243a44dcdcb201bf849a134ae56cd6b9 --- /dev/null +++ b/app/core/dependencies.py @@ -0,0 +1,151 @@ +""" +Dependency injection pour FastAPI. + +Ce module fournit des factory functions pour les services partagés. +Utilise le pattern singleton avec @lru_cache pour efficacité mémoire. + +Usage dans routes: + @router.post("/detect") + async def detect_language( + detector: LanguageDetector = Depends(get_language_detector) + ): + result = detector.detect(text) +""" + +import logging +from functools import lru_cache + +from app.services.document_classifier import DocumentClassifier +from app.services.embedding_service import EmbeddingService +from app.services.language_detector import LanguageDetector +from app.services.search_service import SearchService + +logger = logging.getLogger(__name__) + + +@lru_cache() +def get_language_detector() -> LanguageDetector: + """ + Factory pour LanguageDetector (singleton). + + Utilise @lru_cache pour créer une seule instance partagée + entre toutes les requêtes. Évite le rechargement des modèles + NLP (~200MB) à chaque requête. + + Returns: + Instance singleton de LanguageDetector + + Raises: + LanguageDetectionError: Si les modèles ne peuvent pas être chargés + + Example: + >>> from fastapi import Depends + >>> detector = Depends(get_language_detector) + """ + logger.info("📦 Création du singleton LanguageDetector") + return LanguageDetector() + + +def clear_detector_cache(): + """ + Efface le cache du detector (force rechargement). + + Utile pour: + - Tests (isolation entre tests) + - Mise à jour des modèles + - Debugging + + Example: + >>> clear_detector_cache() + >>> detector = get_language_detector() # Nouvelle instance + """ + get_language_detector.cache_clear() + logger.info("🗑️ Cache LanguageDetector effacé") + + +@lru_cache() +def get_document_classifier() -> DocumentClassifier: + """ + Factory pour DocumentClassifier (singleton). + + Utilise @lru_cache pour créer une seule instance partagée + entre toutes les requêtes. Évite le rechargement des modèles + ML à chaque requête (Phase 3). + + Returns: + Instance singleton de DocumentClassifier + + Example: + >>> from fastapi import Depends + >>> classifier = Depends(get_document_classifier) + """ + logger.info("📦 Création du singleton DocumentClassifier") + return DocumentClassifier() + + +def clear_classifier_cache(): + """ + Efface le cache du classifier (force rechargement). + + Utile pour: + - Tests (isolation entre tests) + - Mise à jour des modèles ML + - Debugging + + Example: + >>> clear_classifier_cache() + >>> classifier = get_document_classifier() # Nouvelle instance + """ + get_document_classifier.cache_clear() + logger.info("🗑️ Cache DocumentClassifier effacé") + + +@lru_cache() +def get_embedding_service() -> EmbeddingService: + """ + Factory pour EmbeddingService (singleton). + + Utilise @lru_cache pour créer une seule instance partagée + entre toutes les requêtes. Évite le rechargement du modèle + sentence-transformers (~120MB) à chaque requête. + + Returns: + Instance singleton de EmbeddingService + + Raises: + EmbeddingServiceError: Si le modèle ne peut pas être chargé + + Example: + >>> from fastapi import Depends + >>> service = Depends(get_embedding_service) + """ + logger.info("📦 Création du singleton EmbeddingService") + return EmbeddingService() + + +def clear_embedding_service_cache(): + """ + Efface le cache du service d'embeddings (force rechargement). + + Utile pour: + - Tests (isolation entre tests) + - Mise à jour des modèles + - Debugging + + Example: + >>> clear_embedding_service_cache() + >>> service = get_embedding_service() # Nouvelle instance + """ + get_embedding_service.cache_clear() + logger.info("🗑️ Cache EmbeddingService effacé") + + +# Exports for dependency injection +__all__ = [ + "get_language_detector", + "get_document_classifier", + "get_embedding_service", + "clear_detector_cache", + "clear_classifier_cache", + "clear_embedding_service_cache", +] diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000000000000000000000000000000000000..74fbf1596f99a8cc57e87cf7d4481c5da4ae3093 --- /dev/null +++ b/app/main.py @@ -0,0 +1,77 @@ +"""Point d'entrée FastAPI.""" + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from app.core.config import settings +from app.api.routes import language, classifier, articles, search, rag, categories, personas, upload, ocr, laws, analytics, admin, batch_upload + +app = FastAPI( + title=settings.APP_NAME, + version=settings.VERSION, + debug=settings.DEBUG +) + +# CORS — allow localhost in dev, Vercel domain in prod +# Set ALLOWED_ORIGINS in your env for production (comma-separated) +_extra_origins = [o.strip() for o in settings.ALLOWED_ORIGINS.split(",") if o.strip()] +_cors_origins = [ + "http://localhost:5173", + "http://localhost:4173", + "http://127.0.0.1:5173", +] + _extra_origins + +app.add_middleware( + CORSMiddleware, + allow_origins=_cors_origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/") +async def root(): + """Root endpoint.""" + return { + "message": "JuriX API v2.1", + "status": "running", + "version": settings.VERSION + } + + +@app.get("/health") +async def health(): + """Health check endpoint.""" + return {"status": "healthy"} + + +# Include routers +app.include_router(language.router) +app.include_router(classifier.router) +app.include_router(articles.router) +app.include_router( + search.router, + prefix="/api/v1/search", + tags=["search"] +) +app.include_router( + rag.router, + prefix="/api/v1/rag", + tags=["rag", "chatbot"] +) +app.include_router( + categories.router, + tags=["categories"] +) +app.include_router( + personas.router, + tags=["personas"] +) +app.include_router(upload.router) +app.include_router(ocr.router) +app.include_router(laws.router) +app.include_router(analytics.router) +app.include_router(admin.router) +app.include_router(batch_upload.router) # Batch upload with WebSocket +# Trigger reload + diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c8165c1ceac820450c2bd4911a7ba002b6cb3fda --- /dev/null +++ b/app/models/__init__.py @@ -0,0 +1,30 @@ +""" +Database models for JuriX application. + +Exports all SQLAlchemy models for easy import. +""" + +from app.models.conversation import ( + Conversation, + Message, + MessageFeedback, + PersonaInteraction, + PersonaStat, +) +from app.models.law import Article, Category, Law +from app.models.user import User + +__all__ = [ + # Law models + "Law", + "Article", + "Category", + # Conversation models + "Conversation", + "Message", + "PersonaStat", + "MessageFeedback", + "PersonaInteraction", + # User models + "User", +] diff --git a/app/models/conversation.py b/app/models/conversation.py new file mode 100644 index 0000000000000000000000000000000000000000..0e0f6c728ef01842863fca9b2bd5eacfed56df8d --- /dev/null +++ b/app/models/conversation.py @@ -0,0 +1,224 @@ +""" +Database models for RAG conversation tracking. + +Stores conversation sessions and individual messages with sources and metadata. +""" + +from datetime import date, datetime + +from sqlalchemy import ( + JSON, + Boolean, + CheckConstraint, + Column, + Date, + DateTime, + Float, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.orm import relationship + +from app.core.database import Base + + +class Conversation(Base): + """ + Conversation session tracking for RAG chatbot. + + Each conversation represents a session with a specific persona and language. + Conversations can span multiple questions and maintain context. + """ + + __tablename__ = "conversations" + + id = Column(Integer, primary_key=True, index=True) + session_id = Column(String(100), unique=True, index=True, nullable=False) + user_id = Column( + Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True + ) + persona = Column(String(50), nullable=False) # citoyen/avocat/entrepreneur/étudiant + language = Column(String(2), nullable=False, default="fr") # fr/en + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow) + + # Relationships + user = relationship("User", back_populates="conversations", lazy="selectin") + messages = relationship( + "Message", + back_populates="conversation", + cascade="all, delete-orphan", + order_by="Message.created_at", + ) + + def __repr__(self): + return f"" + + +class Message(Base): + """ + Individual messages in conversation (user questions and assistant responses). + + Stores message content, role, sources, confidence, and performance metrics. + """ + + __tablename__ = "messages" + + id = Column(Integer, primary_key=True, index=True) + conversation_id = Column( + Integer, ForeignKey("conversations.id", ondelete="CASCADE"), nullable=False, index=True + ) + role = Column(String(20), nullable=False) # "user" or "assistant" + content = Column(Text, nullable=False) + sources = Column(JSON, nullable=True) # [{"law_id": 1, "article_number": "5", ...}] + confidence = Column(Float, nullable=True) # 0.0-1.0 for assistant messages + retrieval_time_ms = Column(Integer, nullable=True) + generation_time_ms = Column(Integer, nullable=True) + created_at = Column(DateTime, nullable=False, default=datetime.utcnow, index=True) + + # Relationships + conversation = relationship("Conversation", back_populates="messages") + + def __repr__(self): + return f"" + + +# Create indexes for performance +Index("idx_conversations_session_id", Conversation.session_id) +Index("idx_messages_conversation_id", Message.conversation_id) +Index("idx_messages_created_at", Message.created_at.desc()) + + +class PersonaStat(Base): + """ + Aggregated daily statistics per persona. + + Generated by Celery task daily at midnight UTC. + Provides pre-aggregated metrics for fast analytics queries. + """ + + __tablename__ = "persona_stats" + + id = Column(Integer, primary_key=True, index=True) + persona = Column(String(50), nullable=False) # citoyen|avocat|entrepreneur|étudiant + date = Column(Date, nullable=False) + + # Usage Metrics + total_questions = Column(Integer, nullable=False, default=0) + total_conversations = Column(Integer, nullable=False, default=0) + unique_sessions = Column(Integer, nullable=False, default=0) + + # Performance Metrics + avg_confidence = Column(Float, nullable=False, default=0.0) + avg_retrieval_time_ms = Column(Integer, nullable=False, default=0) + avg_generation_time_ms = Column(Integer, nullable=False, default=0) + avg_total_time_ms = Column(Integer, nullable=False, default=0) + + # Engagement Metrics + avg_messages_per_conversation = Column(Float, nullable=False, default=0.0) + avg_session_duration_seconds = Column(Integer, nullable=False, default=0) + + # Quality Metrics + helpful_count = Column(Integer, nullable=False, default=0) + unhelpful_count = Column(Integer, nullable=False, default=0) + satisfaction_rate = Column(Float, nullable=False, default=0.0) + + # Metadata + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow) + + __table_args__ = (UniqueConstraint("persona", "date", name="uq_persona_date"),) + + def __repr__(self): + return f"" + + +class MessageFeedback(Base): + """ + User ratings and feedback on assistant messages. + + Allows users to rate messages with 👍/👎 and optional rating (1-5). + One feedback per message (unique constraint). + """ + + __tablename__ = "message_feedback" + + id = Column(Integer, primary_key=True, index=True) + message_id = Column( + Integer, ForeignKey("messages.id", ondelete="CASCADE"), nullable=False, unique=True + ) + helpful = Column(Boolean, nullable=False) + rating = Column(Integer, nullable=True) # 1-5 scale + comment = Column(Text, nullable=True) + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + + # Relationships + message = relationship("Message", backref="feedback") + + __table_args__ = (CheckConstraint("rating >= 1 AND rating <= 5", name="ck_rating_range"),) + + def __repr__(self): + return f"" + + +class PersonaInteraction(Base): + """ + Per-session analytics for persona usage. + + Tracks question/answer counts, confidence, response times, and persona switches + within a conversation session. + """ + + __tablename__ = "persona_interactions" + + id = Column(Integer, primary_key=True, index=True) + conversation_id = Column( + Integer, ForeignKey("conversations.id", ondelete="CASCADE"), nullable=False, index=True + ) + persona = Column(String(50), nullable=False) + language = Column(String(2), nullable=False) + + # Interaction Counts + question_count = Column(Integer, nullable=False, default=0) + answer_count = Column(Integer, nullable=False, default=0) + + # Performance + avg_confidence = Column(Float, nullable=False, default=0.0) + total_response_time_ms = Column(Integer, nullable=False, default=0) + + # Persona Behavior + personas_used = Column(JSON, nullable=True) # Array of all personas used + persona_switches = Column(Integer, nullable=False, default=0) + + # Metadata + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow) + + # Relationships + conversation = relationship("Conversation", backref="persona_interactions") + + def __repr__(self): + return ( + f"" + ) + + +# Create indexes for analytics tables +Index("idx_persona_stats_persona_date", PersonaStat.persona, PersonaStat.date.desc()) +Index("idx_persona_stats_date", PersonaStat.date.desc()) +Index( + "idx_message_feedback_message_id_helpful", MessageFeedback.message_id, MessageFeedback.helpful +) +Index("idx_message_feedback_created_at", MessageFeedback.created_at.desc()) +Index("idx_message_feedback_helpful", MessageFeedback.helpful) +Index( + "idx_persona_interactions_persona_created", + PersonaInteraction.persona, + PersonaInteraction.created_at.desc(), +) +Index("idx_persona_interactions_conversation_id", PersonaInteraction.conversation_id) +Index("idx_persona_interactions_created_at", PersonaInteraction.created_at.desc()) diff --git a/app/models/law.py b/app/models/law.py new file mode 100644 index 0000000000000000000000000000000000000000..d36dc7ea156e27ffb04a2b6653d9fa5778272c51 --- /dev/null +++ b/app/models/law.py @@ -0,0 +1,245 @@ +""" +Modèles SQLAlchemy pour les lois, articles et catégories. + +Ce module définit les modèles de base de données pour le système JuriX: +- Law: Documents juridiques (lois, décrets, ordonnances) +- Article: Articles individuels d'une loi +- Category: Catégories juridiques (12 catégories camerounaises) + +Features v2.1: +- Détection automatique de langue (language, language_confidence) +- Suggestions de catégories (suggested_categories, category_confidence) +- Support pgvector pour embeddings sémantiques + +Author: JuriX Team +""" + +from datetime import datetime, date +from typing import List, Optional + +from sqlalchemy import ( + Column, + Integer, + String, + Text, + Float, + Date, + DateTime, + ForeignKey, + Index, + JSON, +) +from sqlalchemy.dialects.postgresql import ARRAY +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func +from pgvector.sqlalchemy import Vector + +from app.core.database import Base + + +# Type adapter for suggested_categories - uses ARRAY for PostgreSQL, JSON for others +def get_array_type(): + """Get appropriate array type based on database dialect.""" + import os + db_url = os.getenv("DATABASE_URL", "") + if "postgresql" in db_url or "asyncpg" in db_url: + return ARRAY(Integer) + else: + # Use JSON for SQLite and other databases + return JSON + + +class Category(Base): + """ + Modèle pour les catégories juridiques. + + Représente les 12 catégories de droit camerounais: + 1. Droit Constitutionnel + 2. Droit Civil + 3. Droit Pénal + 4. Droit Commercial OHADA + 5. Droit du Travail + 6. Droit Fiscal + 7. Droit Administratif + 8. Droit Foncier + 9. Droit de la Famille + 10. Droit de l'Environnement + 11. Droit International + 12. Droit des Affaires + + Attributes: + id: Identifiant unique + name: Nom de la catégorie + description: Description détaillée + created_at: Date de création + laws: Relation vers les lois de cette catégorie + """ + + __tablename__ = "categories" + + # Primary key + id = Column(Integer, primary_key=True, index=True) + + # Fields + name = Column(String(100), nullable=False) + description = Column(Text, nullable=True) + icon = Column(String(10), nullable=True) # Emoji icon for category + display_order = Column(Integer, nullable=False, default=0) # Order for display + + # Metadata + created_at = Column(DateTime, server_default=func.now(), nullable=False) + + # Relationships + laws = relationship("Law", back_populates="category", lazy="selectin") + + def __repr__(self) -> str: + return f"" + + +class Law(Base): + """ + Modèle pour les documents juridiques (lois). + + Représente un document juridique complet (loi, décret, ordonnance, arrêté) + avec ses métadonnées, sa catégorisation et ses articles. + + Features v2.1: + - Détection automatique de langue + - Suggestions de catégories automatiques + - Scores de confiance pour détection/classification + + Attributes: + id: Identifiant unique + reference: Référence officielle (ex: "LOI-2024-001") + title: Titre de la loi + type: Type de document (loi, décret, ordonnance, arrêté) + content: Contenu complet de la loi + language: Langue détectée ou définie (fr, en) + language_confidence: Score de confiance de détection (0.0-1.0) + detected_language: Langue auto-détectée + suggested_categories: IDs des catégories suggérées (top 3) + category_confidence: Score de confiance catégorie principale + category_id: Catégorie assignée + status: Statut (draft, published, archived) + publication_date: Date de publication officielle + created_at: Date de création dans le système + updated_at: Date de dernière mise à jour + category: Relation vers la catégorie + articles: Relation vers les articles + """ + + __tablename__ = "laws" + + # Primary key + id = Column(Integer, primary_key=True, index=True) + + # Core fields + reference = Column(String(500), nullable=False, unique=True, index=True) + title = Column(String(500), nullable=False) + type = Column(String(50), nullable=False) # loi, décret, ordonnance, arrêté + content = Column(Text, nullable=False) + + # v2.1 Auto-detection fields + language = Column(String(2), nullable=True, index=True) # fr, en + language_confidence = Column(Float, nullable=True) + detected_language = Column(String(2), nullable=True) + suggested_categories = Column(JSON, nullable=True) # List[int] stored as JSON for compatibility + category_confidence = Column(Float, nullable=True) + + # File tracking + file_id = Column(String(100), nullable=True, index=True) # UUID of uploaded file + original_filename = Column(String(500), nullable=True) # Original filename + + # Relationships + category_id = Column(Integer, ForeignKey("categories.id"), nullable=True, index=True) + category = relationship("Category", back_populates="laws", lazy="selectin") + articles = relationship( + "Article", + back_populates="law", + cascade="all, delete-orphan", + lazy="selectin" + ) + + + # Status & metadata + status = Column(String(20), nullable=False, server_default="draft", index=True) # draft, published, archived, pending, processing, refused + processing_progress = Column(Integer, nullable=True) # 0-100 for batch upload tracking + processing_error = Column(Text, nullable=True) # Error message if processing failed + processing_started_at = Column(DateTime, nullable=True) # When processing started + publication_date = Column(Date, nullable=True) + created_at = Column(DateTime, server_default=func.now(), nullable=False) + updated_at = Column( + DateTime, + server_default=func.now(), + onupdate=func.now(), + nullable=False + ) + + def __repr__(self) -> str: + return f"" + + @property + def article_count(self) -> int: + """Nombre d'articles dans cette loi.""" + return len(self.articles) if self.articles else 0 + + +class Article(Base): + """ + Modèle pour les articles individuels d'une loi. + + Représente un article spécifique d'un document juridique, + avec son contenu et son embedding vectoriel pour la recherche sémantique. + + Attributes: + id: Identifiant unique + law_id: ID de la loi parent + number: Numéro de l'article (ex: "1er", "2", "3bis") + title: Titre optionnel de l'article + content: Contenu textuel de l'article + embedding: Vecteur d'embedding (3072 dimensions) pour recherche sémantique + order: Ordre dans la loi (pour tri) + created_at: Date de création + law: Relation vers la loi parent + """ + + __tablename__ = "articles" + + # Primary key + id = Column(Integer, primary_key=True, index=True) + + # Foreign key + law_id = Column( + Integer, + ForeignKey("laws.id", ondelete="CASCADE"), + nullable=False, + index=True + ) + + # Fields + number = Column(String(20), nullable=False) + title = Column(String(200), nullable=True) + section = Column(String(300), nullable=True) # TITRE/CHAPITRE section header + content = Column(Text, nullable=False) + + # pgvector embedding for semantic search (3072-dimensional vector from Gemini) + embedding = Column(Vector(3072), nullable=True) + + # Metadata + order = Column(Integer, nullable=False) # Position in law + page_number = Column(Integer, nullable=True) # PDF page number (1-indexed) for navigation + created_at = Column(DateTime, server_default=func.now(), nullable=False) + + # Relationships + law = relationship("Law", back_populates="articles") + + def __repr__(self) -> str: + return f"" + + +# Indexes for performance +Index("idx_laws_language", Law.language) +Index("idx_laws_category", Law.category_id) +Index("idx_laws_status", Law.status) +Index("idx_laws_reference", Law.reference) +Index("idx_articles_law", Article.law_id) diff --git a/app/models/user.py b/app/models/user.py new file mode 100644 index 0000000000000000000000000000000000000000..fb325f35e69124e64570470e72bbd7096796e19d --- /dev/null +++ b/app/models/user.py @@ -0,0 +1,107 @@ +""" +User model for authentication and authorization. + +Provides user management with role-based access control (RBAC). + +Author: JuriX Team +Date: 2026-01-12 +""" + +from datetime import datetime +from typing import Optional + +from sqlalchemy import Boolean, Column, DateTime, Integer, String +from sqlalchemy.orm import relationship + +from app.core.database import Base + + +class User(Base): + """ + User model for authentication and authorization. + + Attributes: + id: Unique user identifier + email: User email (unique, used for login) + username: Display username + hashed_password: Bcrypt hashed password + full_name: User's full name + role: User role (user, admin, superadmin) + is_active: Account active status + is_verified: Email verification status + created_at: Account creation timestamp + updated_at: Last update timestamp + last_login_at: Last login timestamp + + Relationships: + conversations: User's chat conversations + """ + + __tablename__ = "users" + + # Primary key + id = Column(Integer, primary_key=True, index=True) + + # Authentication + email = Column(String(255), unique=True, index=True, nullable=False) + username = Column(String(100), unique=True, index=True, nullable=False) + hashed_password = Column(String(255), nullable=False) + + # Profile + full_name = Column(String(255), nullable=True) + + # Authorization + role = Column( + String(50), + nullable=False, + default="user", + index=True, + comment="Role: user, admin, superadmin", + ) + + # Status flags + is_active = Column(Boolean, default=True, nullable=False) + is_verified = Column(Boolean, default=False, nullable=False) + + # Timestamps + created_at = Column(DateTime, default=datetime.utcnow, nullable=False) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False) + last_login_at = Column(DateTime, nullable=True) + + # Relationships + conversations = relationship( + "Conversation", back_populates="user", cascade="all, delete-orphan", lazy="selectin" + ) + + def __repr__(self) -> str: + return f"" + + def has_role(self, required_role: str) -> bool: + """ + Check if user has required role or higher. + + Role hierarchy: superadmin > admin > user + + Args: + required_role: Required role to check + + Returns: + True if user has required role or higher + + Example: + >>> user.role = "admin" + >>> user.has_role("user") # True + >>> user.has_role("superadmin") # False + """ + role_hierarchy = {"superadmin": 3, "admin": 2, "user": 1} + user_level = role_hierarchy.get(self.role, 0) + required_level = role_hierarchy.get(required_role, 0) + return user_level >= required_level + + def is_admin(self) -> bool: + """Check if user has admin or superadmin role.""" + return self.role in ("admin", "superadmin") + + def is_superadmin(self) -> bool: + """Check if user has superadmin role.""" + return self.role == "superadmin" diff --git a/app/schemas/file_upload.py b/app/schemas/file_upload.py new file mode 100644 index 0000000000000000000000000000000000000000..43b7430777350175b3c738529f67dfbc36688037 --- /dev/null +++ b/app/schemas/file_upload.py @@ -0,0 +1,238 @@ +""" +Pydantic schemas for file upload operations. + +Provides request/response models with validation for: +- File upload results +- File metadata (PDF, DOCX) +- Validation results +- Antivirus scan results + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel, Field, field_validator + + +# ============================================================================ +# File Upload Result Schemas +# ============================================================================ + + +class FileMetadata(BaseModel): + """ + Metadata extracted from uploaded files. + + Fields vary by file type: + - PDF: title, author, creation_date, page_count + - DOCX: title, author, creation_date, word_count + """ + + title: Optional[str] = Field(None, description="Document title") + author: Optional[str] = Field(None, description="Document author") + creation_date: Optional[datetime] = Field(None, description="Document creation date") + page_count: Optional[int] = Field(None, ge=0, description="Number of pages (PDF only)") + word_count: Optional[int] = Field(None, ge=0, description="Number of words (DOCX only)") + file_size: int = Field(..., ge=0, description="File size in bytes") + file_hash: str = Field(..., min_length=64, max_length=64, description="SHA-256 hash") + + class Config: + json_schema_extra = { + "example": { + "title": "Loi N°2024-015", + "author": "République du Cameroun", + "creation_date": "2024-01-15T10:30:00", + "page_count": 45, + "file_size": 2048576, + "file_hash": "a" * 64, + } + } + + +class ValidationResult(BaseModel): + """ + Result of file validation checks. + + Validates: + - File format (PDF, DOCX) + - File size (max 50 MB) + - File structure integrity + - MIME type + """ + + is_valid: bool = Field(..., description="Whether file passed all validation checks") + format: Optional[str] = Field(None, description="Detected file format (pdf or docx)") + mime_type: str = Field(..., description="Detected MIME type") + errors: List[str] = Field(default_factory=list, description="List of validation errors") + warnings: List[str] = Field(default_factory=list, description="List of validation warnings") + + class Config: + json_schema_extra = { + "example": { + "is_valid": True, + "format": "pdf", + "mime_type": "application/pdf", + "errors": [], + "warnings": [], + } + } + + +class ScanResult(BaseModel): + """ + Result of antivirus scan. + + Supports: + - ClamAV (production) + - Mock scanner (development) + """ + + is_clean: bool = Field(..., description="Whether file is clean (no threats detected)") + scanner: str = Field(..., description="Scanner used (clamav or mock)") + scan_time_ms: int = Field(..., ge=0, description="Scan duration in milliseconds") + threat_found: Optional[str] = Field(None, description="Name of threat if detected") + scanner_version: Optional[str] = Field(None, description="Scanner version") + + @field_validator("scanner") + @classmethod + def validate_scanner(cls, v: str) -> str: + """Validate scanner type.""" + allowed_scanners = {"clamav", "mock"} + if v.lower() not in allowed_scanners: + raise ValueError(f"Scanner must be one of: {', '.join(allowed_scanners)}. Got: {v}") + return v.lower() + + class Config: + json_schema_extra = { + "example": { + "is_clean": True, + "scanner": "mock", + "scan_time_ms": 150, + "threat_found": None, + "scanner_version": "1.0.0", + } + } + + +class FileUploadResult(BaseModel): + """ + Complete result of file upload operation. + + Includes: + - File identification + - Validation results + - Scan results + - Extracted metadata + - Storage information + """ + + file_id: str = Field(..., description="Unique file identifier (UUID)") + filename: str = Field(..., min_length=1, description="Original filename") + file_size: int = Field(..., ge=0, description="File size in bytes") + mime_type: str = Field(..., description="Detected MIME type") + file_hash: str = Field(..., min_length=64, max_length=64, description="SHA-256 hash") + + # Nested results + metadata: FileMetadata = Field(..., description="Extracted file metadata") + validation: ValidationResult = Field(..., description="Validation results") + scan_result: ScanResult = Field(..., description="Antivirus scan results") + + # Storage info + storage_path: str = Field(..., description="Relative path to stored file") + uploaded_at: datetime = Field(..., description="Upload timestamp") + expires_at: Optional[datetime] = Field(None, description="Expiration timestamp for temp files") + + class Config: + json_schema_extra = { + "example": { + "file_id": "550e8400-e29b-41d4-a716-446655440000", + "filename": "loi_2024_015.pdf", + "file_size": 2048576, + "mime_type": "application/pdf", + "file_hash": "a" * 64, + "metadata": { + "title": "Loi N°2024-015", + "author": "République du Cameroun", + "page_count": 45, + "file_size": 2048576, + "file_hash": "a" * 64, + }, + "validation": { + "is_valid": True, + "format": "pdf", + "mime_type": "application/pdf", + "errors": [], + "warnings": [], + }, + "scan_result": { + "is_clean": True, + "scanner": "mock", + "scan_time_ms": 150, + }, + "storage_path": "uploads/550e8400-e29b-41d4-a716-446655440000.pdf", + "uploaded_at": "2024-01-15T10:30:00", + "expires_at": "2024-01-16T10:30:00", + } + } + + +# ============================================================================ +# Request Schemas +# ============================================================================ + + +class FileUploadRequest(BaseModel): + """ + Request schema for file upload (for documentation). + + Note: Actual upload uses multipart/form-data, not JSON. + This schema is for OpenAPI documentation only. + """ + + file: str = Field(..., description="File to upload (multipart/form-data)") + + class Config: + json_schema_extra = { + "example": { + "file": "binary file data", + } + } + + +# ============================================================================ +# Health Check Schema +# ============================================================================ + + +class UploadServiceHealth(BaseModel): + """Health check response for upload service.""" + + service: str = Field(default="FileUploadService", description="Service name") + status: str = Field(..., description="Service status (healthy, degraded, unhealthy)") + storage: dict = Field(..., description="Storage status and statistics") + scanner: dict = Field(..., description="Antivirus scanner status") + timestamp: datetime = Field(..., description="Health check timestamp") + + class Config: + json_schema_extra = { + "example": { + "service": "FileUploadService", + "status": "healthy", + "storage": { + "path": "/app/data/uploads", + "exists": True, + "writable": True, + "total_files": 42, + "total_size_mb": 128.5, + }, + "scanner": { + "type": "mock", + "available": True, + "version": "1.0.0", + }, + "timestamp": "2024-01-15T10:30:00", + } + } diff --git a/app/schemas/law.py b/app/schemas/law.py new file mode 100644 index 0000000000000000000000000000000000000000..22ab5da7e812bd6a3b22f165390d94e16942ba86 --- /dev/null +++ b/app/schemas/law.py @@ -0,0 +1,467 @@ +""" +Pydantic schemas for Law CRUD operations. + +Provides request/response models with validation for: +- Law creation, update, and response +- Category operations +- Article management +- Filtering and pagination +- v2.1 features: language filtering, category suggestions, confidence scores + +Author: JuriX Development Team +Date: 2026-01-10 +""" + +from datetime import date, datetime +from typing import List, Optional + +from pydantic import BaseModel, Field, field_validator + +# ============================================================================ +# Category Schemas +# ============================================================================ + + +class CategoryBase(BaseModel): + """Base schema for Category with shared fields.""" + + name: str = Field(..., min_length=1, max_length=100, description="Category name") + description: Optional[str] = Field(None, description="Category description") + icon: Optional[str] = Field(None, max_length=10, description="Emoji icon for category") + + +class CategoryCreate(CategoryBase): + """Schema for creating a new category.""" + + pass + + +class CategoryUpdate(BaseModel): + """Schema for updating an existing category.""" + + name: Optional[str] = Field(None, min_length=1, max_length=100) + description: Optional[str] = None + + +class CategoryResponse(CategoryBase): + """Schema for category responses.""" + + id: int + created_at: datetime + law_count: int = Field(0, description="Number of laws in this category") + + class Config: + from_attributes = True # SQLAlchemy 2.0 (orm_mode replacement) + + +# ============================================================================ +# Article Schemas +# ============================================================================ + + +class ArticleBase(BaseModel): + """Base schema for Article with shared fields.""" + + number: str = Field( + ..., min_length=1, max_length=20, description="Article number (e.g., 'Art. 1', '42')" + ) + title: Optional[str] = Field(None, max_length=200, description="Article title (optional)") + content: str = Field(..., min_length=1, description="Article text content") + order: int = Field(..., ge=1, description="Display order within the law") + + +class ArticleCreate(ArticleBase): + """Schema for creating a new article.""" + + pass + + +class ArticleUpdate(BaseModel): + """Schema for updating an existing article.""" + + number: Optional[str] = Field(None, min_length=1, max_length=20) + title: Optional[str] = Field(None, max_length=200) + content: Optional[str] = Field(None, min_length=1) + order: Optional[int] = Field(None, ge=1) + + +class ArticleResponse(ArticleBase): + """Schema for article responses.""" + + id: int + law_id: int + created_at: datetime + has_embedding: bool = Field(False, description="Whether article has semantic embedding") + + class Config: + from_attributes = True + + +# ============================================================================ +# Law Schemas +# ============================================================================ + + +class LawBase(BaseModel): + """Base schema for Law with shared fields.""" + + reference: str = Field( + ..., + min_length=1, + max_length=500, + description="Unique law reference (e.g., 'LOI-2024-001', 'DECRET-2023-045')", + ) + title: str = Field(..., min_length=1, max_length=500, description="Law title") + type: str = Field(..., description="Law type (loi, décret, ordonnance, arrêté, etc.)") + content: str = Field(..., min_length=10, description="Full text content of the law") + language: Optional[str] = Field( + None, description="Language code (fr or en). Auto-detected if not provided." + ) + category_id: Optional[int] = Field( + None, ge=1, description="Category ID. Auto-suggested if not provided." + ) + status: Optional[str] = Field( + "draft", description="Publication status (draft, published, archived)" + ) + publication_date: Optional[date] = Field(None, description="Official publication date") + + @field_validator("type") + @classmethod + def validate_type(cls, v: str) -> str: + """Validate law type against allowed values.""" + allowed_types = { + "loi", + "décret", + "ordonnance", + "arrêté", + "circulaire", + "instruction", + "décision", + "autre", + "acte uniforme", # Added for OHADA acts + } + if v.lower() not in allowed_types: + raise ValueError(f"Type must be one of: {', '.join(allowed_types)}. Got: {v}") + return v.lower() + + @field_validator("language") + @classmethod + def validate_language(cls, v: Optional[str]) -> Optional[str]: + """Validate language code.""" + if v is None: + return v + if v.lower() not in {"fr", "en"}: + raise ValueError(f"Language must be 'fr' or 'en'. Got: {v}") + return v.lower() + + @field_validator("status") + @classmethod + def validate_status(cls, v: Optional[str]) -> str: + """Validate status against allowed values.""" + if v is None: + return "draft" + allowed_statuses = {"draft", "published", "archived"} + if v.lower() not in allowed_statuses: + raise ValueError(f"Status must be one of: {', '.join(allowed_statuses)}. Got: {v}") + return v.lower() + + +class LawCreate(LawBase): + """Schema for creating a new law.""" + + pass + + +class LawUpdate(BaseModel): + """ + Schema for updating an existing law. + + All fields are optional to support partial updates. + If content is updated, language and categories will be re-detected. + """ + + reference: Optional[str] = Field(None, min_length=1, max_length=500) + title: Optional[str] = Field(None, min_length=1, max_length=500) + type: Optional[str] = None + content: Optional[str] = Field(None, min_length=10) + language: Optional[str] = None + category_id: Optional[int] = Field(None, ge=1) + status: Optional[str] = None + publication_date: Optional[date] = None + + @field_validator("type") + @classmethod + def validate_type(cls, v: Optional[str]) -> Optional[str]: + """Validate law type if provided.""" + if v is None: + return v + allowed_types = { + "loi", + "décret", + "ordonnance", + "arrêté", + "circulaire", + "instruction", + "décision", + "autre", + } + if v.lower() not in allowed_types: + raise ValueError(f"Type must be one of: {', '.join(allowed_types)}. Got: {v}") + return v.lower() + + @field_validator("language") + @classmethod + def validate_language(cls, v: Optional[str]) -> Optional[str]: + """Validate language code if provided.""" + if v is None: + return v + if v.lower() not in {"fr", "en"}: + raise ValueError(f"Language must be 'fr' or 'en'. Got: {v}") + return v.lower() + + @field_validator("status") + @classmethod + def validate_status(cls, v: Optional[str]) -> Optional[str]: + """Validate status if provided.""" + if v is None: + return v + allowed_statuses = {"draft", "published", "archived"} + if v.lower() not in allowed_statuses: + raise ValueError(f"Status must be one of: {', '.join(allowed_statuses)}. Got: {v}") + return v.lower() + + +class LawResponse(LawBase): + """ + Schema for law responses with v2.1 auto-detection fields. + + Includes: + - Basic law metadata + - v2.1 language detection (detected_language, language_confidence) + - v2.1 category suggestions (suggested_categories, category_confidence) + - Article count + - Timestamps + """ + + id: int + + # v2.1 Auto-detection fields + detected_language: Optional[str] = Field(None, description="Auto-detected language (fr or en)") + language_confidence: Optional[float] = Field( + None, ge=0.0, le=1.0, description="Confidence score for language detection (0.0-1.0)" + ) + suggested_categories: Optional[List[int]] = Field( + None, description="Top 3 suggested category IDs from DocumentClassifier" + ) + category_confidence: Optional[float] = Field( + None, ge=0.0, le=1.0, description="Confidence score for top category suggestion (0.0-1.0)" + ) + + # File tracking + file_id: Optional[str] = Field(None, description="ID of the uploaded file") + original_filename: Optional[str] = Field(None, description="Original filename of the uploaded file") + + # Metadata + article_count: int = Field(0, description="Number of articles in this law") + created_at: datetime + updated_at: Optional[datetime] = None + + # Optional nested category + category: Optional[CategoryResponse] = None + + class Config: + from_attributes = True + + +class LawDetailResponse(LawResponse): + """ + Detailed law response including articles. + + Used for GET /laws/{id}?include_articles=true + """ + + articles: List[ArticleResponse] = Field(default_factory=list) + + +# ============================================================================ +# Filtering & Pagination Schemas +# ============================================================================ + + +class LawFilters(BaseModel): + """ + Query parameters for filtering and paginating law lists. + + Supports: + - Language filtering (v2.1 feature) + - Category filtering + - Status filtering + - Type filtering + - Date range filtering + - Pagination + """ + + # v2.1 Language filtering + language: Optional[str] = Field(None, description="Filter by language (fr or en)") + + # Category filtering + category_id: Optional[int] = Field(None, ge=1, description="Filter by category ID") + + # Status filtering + status: Optional[str] = Field(None, description="Filter by status (draft, published, archived)") + + # Type filtering + type: Optional[str] = Field(None, description="Filter by law type") + + # Date range filtering + year_from: Optional[int] = Field( + None, ge=1900, le=2100, description="Filter laws published from this year" + ) + year_to: Optional[int] = Field( + None, ge=1900, le=2100, description="Filter laws published up to this year" + ) + + # Pagination + page: int = Field(1, ge=1, description="Page number (1-indexed)") + per_page: int = Field(20, ge=1, le=100, description="Items per page (max 100)") + + # Sorting + sort_by: Optional[str] = Field( + "created_at", description="Sort field (created_at, publication_date, title, reference)" + ) + sort_order: Optional[str] = Field("desc", description="Sort order (asc or desc)") + + @field_validator("language") + @classmethod + def validate_language(cls, v: Optional[str]) -> Optional[str]: + """Validate language filter.""" + if v is None: + return v + if v.lower() not in {"fr", "en"}: + raise ValueError(f"Language must be 'fr' or 'en'. Got: {v}") + return v.lower() + + @field_validator("status") + @classmethod + def validate_status(cls, v: Optional[str]) -> Optional[str]: + """Validate status filter.""" + if v is None: + return v + allowed_statuses = {"draft", "published", "archived"} + if v.lower() not in allowed_statuses: + raise ValueError(f"Status must be one of: {', '.join(allowed_statuses)}. Got: {v}") + return v.lower() + + @field_validator("sort_by") + @classmethod + def validate_sort_by(cls, v: Optional[str]) -> str: + """Validate sort field.""" + if v is None: + return "created_at" + allowed_fields = {"created_at", "publication_date", "title", "reference", "updated_at"} + if v.lower() not in allowed_fields: + raise ValueError(f"sort_by must be one of: {', '.join(allowed_fields)}. Got: {v}") + return v.lower() + + @field_validator("sort_order") + @classmethod + def validate_sort_order(cls, v: Optional[str]) -> str: + """Validate sort order.""" + if v is None: + return "desc" + if v.lower() not in {"asc", "desc"}: + raise ValueError(f"sort_order must be 'asc' or 'desc'. Got: {v}") + return v.lower() + + @field_validator("year_to") + @classmethod + def validate_year_range(cls, v: Optional[int], info) -> Optional[int]: + """Ensure year_to >= year_from if both provided.""" + if v is None: + return v + year_from = info.data.get("year_from") + if year_from is not None and v < year_from: + raise ValueError(f"year_to ({v}) must be >= year_from ({year_from})") + return v + + +class LawListResponse(BaseModel): + """ + Paginated list of laws with metadata. + + Used for GET /laws with filtering and pagination. + """ + + items: List[LawResponse] = Field(default_factory=list) + total: int = Field(0, description="Total number of laws matching filters") + page: int = Field(1, description="Current page number") + per_page: int = Field(20, description="Items per page") + pages: int = Field(0, description="Total number of pages") + + # Optional filter summary + filters_applied: Optional[dict] = Field(None, description="Summary of applied filters") + + +# ============================================================================ +# Search Schemas +# ============================================================================ + + +class SearchQuery(BaseModel): + """Schema for search operations.""" + + query: str = Field(..., min_length=1, max_length=500, description="Search query text") + filters: Optional[LawFilters] = Field(None, description="Additional filters to apply") + + +class SearchResult(LawResponse): + """Search result with relevance score.""" + + relevance_score: float = Field( + 0.0, ge=0.0, le=1.0, description="Search relevance score (0.0-1.0)" + ) + + +class SearchResponse(BaseModel): + """Response for search operations.""" + + query: str + results: List[SearchResult] = Field(default_factory=list) + total: int = Field(0, description="Total number of results") + search_time_ms: float = Field(0.0, description="Search execution time in milliseconds") + + +# ============================================================================ +# Statistics Schemas +# ============================================================================ + + +class LanguageStats(BaseModel): + """Statistics on law distribution by language (v2.1 feature).""" + + french: int = Field(0, description="Number of French laws") + english: int = Field(0, description="Number of English laws") + unknown: int = Field(0, description="Number of laws with unknown language") + total: int = Field(0, description="Total number of laws") + + +class CategoryStats(BaseModel): + """Statistics on law distribution by category.""" + + category_id: int + category_name: str + law_count: int + percentage: float = Field(0.0, ge=0.0, le=100.0) + + +class LawStats(BaseModel): + """Overall statistics for the law database.""" + + total_laws: int = Field(0) + total_articles: int = Field(0) + by_language: LanguageStats + by_status: dict = Field(default_factory=dict) # {"draft": 10, "published": 50, ...} + by_type: dict = Field(default_factory=dict) # {"loi": 30, "décret": 20, ...} + top_categories: List[CategoryStats] = Field(default_factory=list) + avg_articles_per_law: float = Field(0.0) + latest_publication: Optional[date] = None diff --git a/app/schemas/ocr.py b/app/schemas/ocr.py new file mode 100644 index 0000000000000000000000000000000000000000..0165452eaf0a21ab20abdde6553d831b5b5aab5c --- /dev/null +++ b/app/schemas/ocr.py @@ -0,0 +1,213 @@ +""" +Pydantic schemas for OCR operations. + +Provides request/response models with validation for: +- PDF type detection (native, scanned, hybrid) +- OCR processing results +- OCR configuration +- Health check + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +from enum import Enum +from typing import Optional + +from pydantic import BaseModel, Field, field_validator + + +# ============================================================================ +# Enums +# ============================================================================ + + +class PDFType(str, Enum): + """Type of PDF document.""" + + NATIVE = "native" # Text-based PDF with embedded text + SCANNED = "scanned" # Image-based PDF requiring OCR + HYBRID = "hybrid" # Mixed content (text + images) + UNKNOWN = "unknown" # Unable to determine + + +# ============================================================================ +# OCR Result Schemas +# ============================================================================ + + +class OCRResult(BaseModel): + """ + Result of OCR processing on a PDF. + + Contains: + - PDF type classification + - Extracted text (cleaned) + - Processing metadata + - Confidence scores + """ + + pdf_type: PDFType = Field(..., description="Detected PDF type") + text: str = Field(..., description="Extracted and cleaned text") + page_count: int = Field(..., ge=1, description="Number of pages processed") + processing_time_ms: int = Field(..., ge=0, description="Processing time in milliseconds") + language: str = Field(..., description="OCR language used (e.g., 'fra+eng')") + confidence: Optional[float] = Field( + None, ge=0.0, le=1.0, description="OCR confidence score (0-1)" + ) + ocr_engine: str = Field(default="tesseract", description="OCR engine used") + text_length: int = Field(..., ge=0, description="Length of extracted text") + + class Config: + json_schema_extra = { + "example": { + "pdf_type": "scanned", + "text": "Article 1. La présente loi régit...", + "page_count": 45, + "processing_time_ms": 15000, + "language": "fra+eng", + "confidence": 0.92, + "ocr_engine": "tesseract", + "text_length": 25000, + } + } + + +class PDFTypeDetectionResult(BaseModel): + """ + Result of PDF type detection only (no OCR). + + Quick check to determine if OCR is needed. + """ + + pdf_type: PDFType = Field(..., description="Detected PDF type") + has_text: bool = Field(..., description="Whether PDF has embedded text") + has_images: bool = Field(..., description="Whether PDF contains images") + page_count: int = Field(..., ge=1, description="Number of pages") + text_coverage: float = Field( + ..., ge=0.0, le=1.0, description="Percentage of pages with text (0-1)" + ) + needs_ocr: bool = Field(..., description="Whether OCR processing is recommended") + + class Config: + json_schema_extra = { + "example": { + "pdf_type": "scanned", + "has_text": False, + "has_images": True, + "page_count": 45, + "text_coverage": 0.0, + "needs_ocr": True, + } + } + + +# ============================================================================ +# Request Schemas +# ============================================================================ + + +class OCRRequest(BaseModel): + """ + Request schema for OCR processing. + """ + + pdf_path: str = Field(..., min_length=1, description="Path to PDF file") + language: str = Field( + default="fra+eng", + description="Tesseract language codes (e.g., 'fra', 'eng', 'fra+eng')", + ) + dpi: int = Field( + default=300, ge=72, le=600, description="DPI for PDF to image conversion" + ) + timeout_seconds: int = Field( + default=60, ge=5, le=300, description="Maximum processing time per page" + ) + + @field_validator("language") + @classmethod + def validate_language(cls, v: str) -> str: + """Validate language codes.""" + allowed_languages = {"fra", "eng", "fra+eng", "eng+fra"} + if v.lower() not in allowed_languages: + raise ValueError( + f"Language must be one of: {', '.join(allowed_languages)}. Got: {v}" + ) + return v.lower() + + class Config: + json_schema_extra = { + "example": { + "pdf_path": "/path/to/document.pdf", + "language": "fra+eng", + "dpi": 300, + "timeout_seconds": 60, + } + } + + +class PDFTypeDetectionRequest(BaseModel): + """Request schema for PDF type detection only.""" + + pdf_path: str = Field(..., min_length=1, description="Path to PDF file") + + class Config: + json_schema_extra = {"example": {"pdf_path": "/path/to/document.pdf"}} + + +# ============================================================================ +# Health Check Schema +# ============================================================================ + + +class OCRServiceHealth(BaseModel): + """Health check response for OCR service.""" + + service: str = Field(default="OCRService", description="Service name") + status: str = Field(..., description="Service status (healthy, degraded, unhealthy)") + tesseract: dict = Field(..., description="Tesseract OCR status") + poppler: dict = Field(..., description="Poppler (pdf2image) status") + supported_languages: list = Field( + default_factory=list, description="Available Tesseract languages" + ) + + class Config: + json_schema_extra = { + "example": { + "service": "OCRService", + "status": "healthy", + "tesseract": { + "available": True, + "version": "5.3.0", + "path": "/usr/bin/tesseract", + }, + "poppler": {"available": True, "version": "22.12.0"}, + "supported_languages": ["fra", "eng", "osd"], + } + } + + +# ============================================================================ +# Statistics Schema +# ============================================================================ + + +class OCRStatistics(BaseModel): + """Statistics for OCR processing.""" + + total_pages_processed: int = Field(0, ge=0) + total_processing_time_ms: int = Field(0, ge=0) + average_time_per_page_ms: float = Field(0.0, ge=0.0) + total_text_extracted: int = Field(0, ge=0, description="Total characters extracted") + success_rate: float = Field(0.0, ge=0.0, le=1.0, description="Success rate (0-1)") + + class Config: + json_schema_extra = { + "example": { + "total_pages_processed": 450, + "total_processing_time_ms": 180000, + "average_time_per_page_ms": 400, + "total_text_extracted": 1250000, + "success_rate": 0.98, + } + } diff --git a/app/schemas/persona.py b/app/schemas/persona.py new file mode 100644 index 0000000000000000000000000000000000000000..78fa5857c9831471c59326015364975ad2a9e1eb --- /dev/null +++ b/app/schemas/persona.py @@ -0,0 +1,428 @@ +""" +Pydantic schemas for PersonaService API. + +Request/response models for persona management and analytics endpoints. +""" + +from datetime import date, datetime +from typing import Dict, List, Optional, Any +from pydantic import BaseModel, Field, field_validator + + +# ============================================================================ +# REQUEST SCHEMAS +# ============================================================================ + +class MessageFeedbackCreate(BaseModel): + """ + Schema for creating message feedback. + + Users can rate messages as helpful/unhelpful with optional 1-5 rating. + """ + message_id: int = Field(..., description="ID of the message to rate", gt=0) + helpful: bool = Field(..., description="Whether the message was helpful (👍/👎)") + rating: Optional[int] = Field(None, description="Optional 1-5 star rating", ge=1, le=5) + comment: Optional[str] = Field(None, description="Optional comment", max_length=500) + + model_config = { + "json_schema_extra": { + "examples": [ + { + "message_id": 123, + "helpful": True, + "rating": 5, + "comment": "Très claire et utile!" + } + ] + } + } + + +# ============================================================================ +# RESPONSE SCHEMAS +# ============================================================================ + +class PersonaInfo(BaseModel): + """ + Information about a specific persona. + + Includes name, display name, description, icon, tone, and example questions. + """ + name: str = Field(..., description="Persona identifier (citoyen|avocat|entrepreneur|étudiant)") + display_name: str = Field(..., description="Display name (e.g., 'Citoyen', 'Avocat')") + description: str = Field(..., description="Description of persona's approach") + icon: str = Field(..., description="Emoji icon representing persona") + tone: str = Field(..., description="Communication tone of persona") + example_questions: List[str] = Field(..., description="Example questions for this persona") + + model_config = { + "json_schema_extra": { + "examples": [ + { + "name": "citoyen", + "display_name": "Citoyen", + "description": "Assistant bienveillant qui aide les citoyens à comprendre leurs droits", + "icon": "👤", + "tone": "Empathique et rassurant", + "example_questions": [ + "Quels sont mes droits en tant que locataire?", + "Comment porter plainte au commissariat?" + ] + } + ] + } + } + + +class UsageMetrics(BaseModel): + """Usage statistics metrics.""" + total_questions: int = Field(..., description="Total number of questions asked") + total_conversations: int = Field(..., description="Total number of conversations") + unique_sessions: int = Field(..., description="Number of unique session IDs") + + +class PerformanceMetrics(BaseModel): + """Performance statistics metrics.""" + avg_confidence: float = Field(..., description="Average confidence score (0.0-1.0)") + avg_retrieval_time_ms: int = Field(..., description="Average retrieval time in milliseconds") + avg_generation_time_ms: int = Field(..., description="Average generation time in milliseconds") + avg_total_time_ms: int = Field(..., description="Average total response time in milliseconds") + + +class EngagementMetrics(BaseModel): + """Engagement statistics metrics.""" + avg_messages_per_conversation: float = Field(..., description="Average messages per conversation") + avg_session_duration_seconds: int = Field(..., description="Average session duration in seconds") + + +class QualityMetrics(BaseModel): + """Quality statistics metrics from user feedback.""" + helpful_count: int = Field(..., description="Number of helpful ratings (👍)") + unhelpful_count: int = Field(..., description="Number of unhelpful ratings (👎)") + satisfaction_rate: float = Field(..., description="Satisfaction rate (0.0-1.0)") + + +class DateRange(BaseModel): + """Date range for statistics.""" + start: str = Field(..., description="Start date (ISO format)") + end: str = Field(..., description="End date (ISO format)") + + +class PersonaStatsResponse(BaseModel): + """ + Comprehensive statistics for a persona over a date range. + + Aggregated from persona_stats table. + """ + persona: str = Field(..., description="Persona name") + date_range: DateRange = Field(..., description="Date range for statistics") + usage: UsageMetrics = Field(..., description="Usage metrics") + performance: PerformanceMetrics = Field(..., description="Performance metrics") + engagement: EngagementMetrics = Field(..., description="Engagement metrics") + quality: QualityMetrics = Field(..., description="Quality metrics from feedback") + + model_config = { + "json_schema_extra": { + "examples": [ + { + "persona": "citoyen", + "date_range": {"start": "2024-01-01", "end": "2024-01-31"}, + "usage": { + "total_questions": 150, + "total_conversations": 80, + "unique_sessions": 75 + }, + "performance": { + "avg_confidence": 0.85, + "avg_retrieval_time_ms": 450, + "avg_generation_time_ms": 1200, + "avg_total_time_ms": 1650 + }, + "engagement": { + "avg_messages_per_conversation": 1.88, + "avg_session_duration_seconds": 180 + }, + "quality": { + "helpful_count": 45, + "unhelpful_count": 5, + "satisfaction_rate": 0.9 + } + } + ] + } + } + + +class MessageFeedbackResponse(BaseModel): + """Response schema for message feedback.""" + id: int = Field(..., description="Feedback ID") + message_id: int = Field(..., description="Message ID") + helpful: bool = Field(..., description="Whether message was helpful") + rating: Optional[int] = Field(None, description="Optional 1-5 rating") + comment: Optional[str] = Field(None, description="Optional comment") + created_at: str = Field(..., description="Creation timestamp (ISO format)") + + model_config = { + "json_schema_extra": { + "examples": [ + { + "id": 1, + "message_id": 123, + "helpful": True, + "rating": 5, + "comment": "Très utile!", + "created_at": "2024-01-15T10:30:00" + } + ] + } + } + + +class PersonaFeedbackBreakdown(BaseModel): + """Feedback breakdown for a single persona.""" + total: int = Field(..., description="Total feedback count") + helpful: int = Field(..., description="Helpful count") + unhelpful: int = Field(..., description="Unhelpful count") + + +class FeedbackStats(BaseModel): + """ + Aggregated feedback statistics. + + Can be filtered by persona and date range. + """ + total_feedback: int = Field(..., description="Total number of feedback entries") + helpful_count: int = Field(..., description="Number of helpful ratings") + unhelpful_count: int = Field(..., description="Number of unhelpful ratings") + satisfaction_rate: float = Field(..., description="Satisfaction rate (0.0-1.0)") + avg_rating: Optional[float] = Field(None, description="Average star rating (1-5)") + by_persona: Dict[str, PersonaFeedbackBreakdown] = Field( + default_factory=dict, + description="Breakdown by persona (empty if filtered by persona)" + ) + + model_config = { + "json_schema_extra": { + "examples": [ + { + "total_feedback": 100, + "helpful_count": 85, + "unhelpful_count": 15, + "satisfaction_rate": 0.85, + "avg_rating": 4.2, + "by_persona": { + "citoyen": {"total": 50, "helpful": 45, "unhelpful": 5}, + "avocat": {"total": 30, "helpful": 25, "unhelpful": 5}, + "entrepreneur": {"total": 15, "helpful": 12, "unhelpful": 3}, + "étudiant": {"total": 5, "helpful": 3, "unhelpful": 2} + } + } + ] + } + } + + +class DetailedEngagementMetrics(BaseModel): + """Detailed engagement metrics for a persona.""" + total_conversations: int = Field(..., description="Total conversations in period") + total_messages: int = Field(..., description="Total messages exchanged") + avg_messages_per_conversation: float = Field(..., description="Average messages per conversation") + avg_session_duration_seconds: int = Field(..., description="Average session duration") + active_days: int = Field(..., description="Number of days with activity") + conversations_per_active_day: float = Field(..., description="Average conversations per active day") + + +class EngagementMetricsResponse(BaseModel): + """ + Response schema for engagement metrics endpoint. + """ + persona: str = Field(..., description="Persona name") + days: int = Field(..., description="Number of days analyzed") + metrics: DetailedEngagementMetrics = Field(..., description="Engagement metrics") + + model_config = { + "json_schema_extra": { + "examples": [ + { + "persona": "citoyen", + "days": 30, + "metrics": { + "total_conversations": 120, + "total_messages": 240, + "avg_messages_per_conversation": 2.0, + "avg_session_duration_seconds": 180, + "active_days": 25, + "conversations_per_active_day": 4.8 + } + } + ] + } + } + + +class PopularQuestion(BaseModel): + """A popular question with its frequency count.""" + question: str = Field(..., description="Question text") + count: int = Field(..., description="Number of times asked") + + +class PopularQuestionsResponse(BaseModel): + """Response schema for popular questions endpoint.""" + persona: str = Field(..., description="Persona name") + questions: List[PopularQuestion] = Field(..., description="List of popular questions") + total_count: int = Field(..., description="Total number of questions found") + + model_config = { + "json_schema_extra": { + "examples": [ + { + "persona": "citoyen", + "questions": [ + {"question": "Quels sont mes droits en tant que locataire?", "count": 15}, + {"question": "Comment porter plainte?", "count": 12} + ], + "total_count": 2 + } + ] + } + } + + +class PersonaValue(BaseModel): + """Value and display name for a persona in comparison.""" + value: float = Field(..., description="Metric value") + display_name: str = Field(..., description="Display name of persona") + + +class PersonaComparisonResponse(BaseModel): + """ + Response schema for persona comparison endpoint. + """ + metric: str = Field(..., description="Metric being compared (usage|confidence|satisfaction)") + days: int = Field(..., description="Number of days analyzed") + date_range: DateRange = Field(..., description="Date range for comparison") + personas: Dict[str, PersonaValue] = Field(..., description="Comparison values per persona") + + model_config = { + "json_schema_extra": { + "examples": [ + { + "metric": "usage", + "days": 30, + "date_range": {"start": "2024-01-01", "end": "2024-01-31"}, + "personas": { + "citoyen": {"value": 150, "display_name": "Citoyen"}, + "avocat": {"value": 80, "display_name": "Avocat"}, + "entrepreneur": {"value": 45, "display_name": "Entrepreneur"}, + "étudiant": {"value": 25, "display_name": "Étudiant"} + } + } + ] + } + } + + +class TrendPoint(BaseModel): + """A single point in a trend time series.""" + date: str = Field(..., description="Date (ISO format)") + value: float = Field(..., description="Metric value for this date") + + +class TrendsResponse(BaseModel): + """Response schema for trends endpoint.""" + persona: str = Field(..., description="Persona name") + metric: str = Field(..., description="Metric tracked (questions|confidence|satisfaction)") + days: int = Field(..., description="Number of days analyzed") + data_points: List[TrendPoint] = Field(..., description="Time series data points") + + model_config = { + "json_schema_extra": { + "examples": [ + { + "persona": "citoyen", + "metric": "questions", + "days": 7, + "data_points": [ + {"date": "2024-01-01", "value": 5}, + {"date": "2024-01-02", "value": 8}, + {"date": "2024-01-03", "value": 12} + ] + } + ] + } + } + + +class UsageBreakdownResponse(BaseModel): + """Response schema for persona usage breakdown.""" + total_conversations: int = Field(..., description="Total conversations across all personas") + percentages: Dict[str, float] = Field(..., description="Usage percentage per persona") + + model_config = { + "json_schema_extra": { + "examples": [ + { + "total_conversations": 300, + "percentages": { + "citoyen": 50.0, + "avocat": 26.67, + "entrepreneur": 15.0, + "étudiant": 8.33 + } + } + ] + } + } + + +class HealthCheckResponse(BaseModel): + """Response schema for health check endpoint.""" + service: str = Field(..., description="Service name") + status: str = Field(..., description="Service status") + personas_count: int = Field(..., description="Number of supported personas") + personas: List[str] = Field(..., description="List of persona names") + timestamp: str = Field(..., description="Timestamp (ISO format)") + + model_config = { + "json_schema_extra": { + "examples": [ + { + "service": "PersonaService", + "status": "healthy", + "personas_count": 4, + "personas": ["citoyen", "avocat", "entrepreneur", "étudiant"], + "timestamp": "2024-01-15T10:30:00" + } + ] + } + } + + +# ============================================================================ +# ERROR RESPONSE SCHEMAS +# ============================================================================ + +class ErrorDetail(BaseModel): + """Detailed error information.""" + error: str = Field(..., description="Error type") + message: str = Field(..., description="Human-readable error message") + details: Optional[Dict[str, Any]] = Field(None, description="Additional error details") + + +class ErrorResponse(BaseModel): + """Standard error response schema.""" + detail: ErrorDetail = Field(..., description="Error details") + + model_config = { + "json_schema_extra": { + "examples": [ + { + "detail": { + "error": "InvalidPersonaError", + "message": "Invalid persona 'unknown'. Must be one of: citoyen, avocat, entrepreneur, étudiant", + "details": {"provided": "unknown", "valid": ["citoyen", "avocat", "entrepreneur", "étudiant"]} + } + } + ] + } + } diff --git a/app/schemas/rag.py b/app/schemas/rag.py new file mode 100644 index 0000000000000000000000000000000000000000..67747984a31efccfee7b6adc174ca2c17cddc82c --- /dev/null +++ b/app/schemas/rag.py @@ -0,0 +1,185 @@ +""" +Pydantic schemas for RAG chatbot service. + +Defines request/response models for RAG endpoints, citations, and conversation management. +""" + +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel, Field, field_validator + + +class RAGRequest(BaseModel): + """Request schema for RAG ask endpoint.""" + + question: str = Field( + ..., + min_length=5, + max_length=1000, + description="User question", + examples=["Quelle est la responsabilité des dirigeants de société?"] + ) + persona: str = Field( + "citoyen", + description="User persona (citoyen/avocat/entrepreneur/étudiant)" + ) + language: str = Field( + "fr", + description="Response language (fr/en)" + ) + session_id: Optional[str] = Field( + None, + description="Session ID for conversation continuity" + ) + stream: bool = Field( + False, + description="Enable streaming response" + ) + law_id: Optional[int] = Field( + None, + description="Current law ID being viewed (prioritizes this document in search)" + ) + top_k: Optional[int] = Field( + 5, + ge=1, + le=20, + description="Number of results to retrieve" + ) + + @field_validator("persona") + @classmethod + def validate_persona(cls, v: str) -> str: + allowed = {"citoyen", "avocat", "entrepreneur", "étudiant"} + if v not in allowed: + raise ValueError(f"Persona must be one of: {', '.join(allowed)}") + return v + + @field_validator("language") + @classmethod + def validate_language(cls, v: str) -> str: + if v not in {"fr", "en"}: + raise ValueError("Language must be 'fr' or 'en'") + return v + + model_config = { + "json_schema_extra": { + "example": { + "question": "Quelle est la responsabilité des dirigeants de société?", + "persona": "avocat", + "language": "fr", + "session_id": "abc123", + "stream": False + } + } + } + + +class Citation(BaseModel): + """Source citation with structured data for frontend linking.""" + + law_id: int = Field(..., description="Database ID of law") + law_reference: str = Field(..., description="Law reference (e.g., LOI-2024-001)") + law_title: str = Field(..., description="Law title") + article_number: Optional[str] = Field(None, description="Article number if specific") + excerpt: str = Field(..., max_length=300, description="Relevant excerpt") + relevance_score: float = Field(..., ge=0.0, le=1.0, description="Relevance score") + + model_config = { + "json_schema_extra": { + "example": { + "law_id": 156, + "law_reference": "LOI-2024-001", + "law_title": "Code OHADA", + "article_number": "161", + "excerpt": "Les dirigeants sont responsables...", + "relevance_score": 0.92 + } + } + } + + +class RAGResponse(BaseModel): + """Response schema for RAG ask endpoint.""" + + answer: str = Field(..., description="Generated answer") + confidence: float = Field(..., ge=0.0, le=1.0, description="Confidence score") + sources: List[Citation] = Field(default_factory=list, description="Source citations") + session_id: str = Field(..., description="Session ID for continuation") + retrieval_time_ms: int = Field(..., description="Time for document retrieval") + generation_time_ms: int = Field(..., description="Time for answer generation") + total_time_ms: int = Field(..., description="Total processing time") + persona: str = Field(..., description="Persona used for response") + + model_config = { + "json_schema_extra": { + "example": { + "answer": "La responsabilité des dirigeants de société...", + "confidence": 0.85, + "sources": [ + { + "law_id": 156, + "law_reference": "LOI-2024-001", + "law_title": "Code OHADA", + "article_number": "161", + "excerpt": "Les dirigeants sont responsables...", + "relevance_score": 0.92 + } + ], + "session_id": "abc123", + "retrieval_time_ms": 180, + "generation_time_ms": 2400, + "total_time_ms": 2600, + "persona": "citoyen" + } + } + } + + +class RAGStreamChunk(BaseModel): + """Streaming response chunk for SSE.""" + + chunk: str = Field(..., description="Response text chunk") + done: bool = Field(False, description="Whether generation is complete") + sources: Optional[List[Citation]] = Field( + None, + description="Sources (only in final chunk)" + ) + confidence: Optional[float] = Field( + None, + description="Confidence (only in final chunk)" + ) + session_id: Optional[str] = Field( + None, + description="Session ID (only in final chunk)" + ) + + +class MessageResponse(BaseModel): + """Single message in conversation.""" + + id: int + role: str # "user" or "assistant" + content: str + sources: Optional[List[Citation]] = None + confidence: Optional[float] = None + created_at: datetime + + model_config = { + "from_attributes": True + } + + +class ConversationResponse(BaseModel): + """Conversation history response.""" + + session_id: str + persona: str + language: str + created_at: datetime + updated_at: datetime + messages: List[MessageResponse] + + model_config = { + "from_attributes": True + } diff --git a/app/schemas/search.py b/app/schemas/search.py new file mode 100644 index 0000000000000000000000000000000000000000..9d21fcac231ed19f6924b8cc6d64c78857ba2ef0 --- /dev/null +++ b/app/schemas/search.py @@ -0,0 +1,377 @@ +""" +Pydantic schemas for Search operations. + +Provides request/response models for hybrid search (text + semantic + RRF): +- SearchFilters: Filter parameters for search queries +- SearchRequest: Main search request with query, mode, filters +- ArticleMatch: Matched article with snippet and relevance +- SearchResult: Single search result with highlights +- SearchResponse: Complete search response with metadata +- SearchStats: Admin statistics for search operations + +Author: JuriX Development Team +Date: 2026-01-10 +""" + +from datetime import date +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field, field_validator + + +# ============================================================================ +# Filter Schemas +# ============================================================================ + +class SearchFilters(BaseModel): + """ + Filter parameters for search operations. + + Supports filtering by: + - Language (fr/en) + - Categories (multiple IDs) + - Document types + - Status (draft, published, archived) + - Publication date range + """ + + language: Optional[str] = Field( + None, + description="Filter by language code (fr or en)" + ) + category_ids: Optional[List[int]] = Field( + None, + description="Filter by category IDs" + ) + types: Optional[List[str]] = Field( + None, + description="Filter by law types (loi, décret, ordonnance, arrêté)" + ) + status: Optional[str] = Field( + None, + description="Filter by status (draft, published, archived)" + ) + year_from: Optional[int] = Field( + None, + ge=1900, + le=2100, + description="Filter laws from this publication year" + ) + year_to: Optional[int] = Field( + None, + ge=1900, + le=2100, + description="Filter laws up to this publication year" + ) + + @field_validator("language") + @classmethod + def validate_language(cls, v: Optional[str]) -> Optional[str]: + """Validate language code.""" + if v is None: + return v + if v.lower() not in {"fr", "en"}: + raise ValueError(f"Language must be 'fr' or 'en'. Got: {v}") + return v.lower() + + @field_validator("status") + @classmethod + def validate_status(cls, v: Optional[str]) -> Optional[str]: + """Validate status.""" + if v is None: + return v + allowed = {"draft", "published", "archived"} + if v.lower() not in allowed: + raise ValueError(f"Status must be one of: {', '.join(allowed)}. Got: {v}") + return v.lower() + + @field_validator("year_to") + @classmethod + def validate_year_range(cls, v: Optional[int], info) -> Optional[int]: + """Ensure year_to >= year_from.""" + if v is None: + return v + year_from = info.data.get("year_from") + if year_from is not None and v < year_from: + raise ValueError(f"year_to ({v}) must be >= year_from ({year_from})") + return v + + +# ============================================================================ +# Request Schemas +# ============================================================================ + +class SearchRequest(BaseModel): + """ + Request schema for search endpoint. + + Supports three search modes: + - text: Full-text search only (Meilisearch) + - semantic: Vector search only (pgvector) + - hybrid: Combined search with RRF fusion (default, recommended) + """ + + query: str = Field( + ..., + min_length=1, + max_length=500, + description="Search query text" + ) + mode: str = Field( + "hybrid", + description="Search mode: text, semantic, or hybrid" + ) + filters: Optional[SearchFilters] = Field( + None, + description="Optional filters to apply" + ) + limit: int = Field( + 15, + ge=1, + le=50, + description="Maximum number of results to return (1-50)" + ) + offset: int = Field( + 0, + ge=0, + description="Number of results to skip (for pagination)" + ) + + @field_validator("mode") + @classmethod + def validate_mode(cls, v: str) -> str: + """Validate search mode.""" + allowed_modes = {"text", "semantic", "hybrid"} + if v.lower() not in allowed_modes: + raise ValueError( + f"Mode must be one of: {', '.join(allowed_modes)}. Got: {v}" + ) + return v.lower() + + class Config: + json_schema_extra = { + "example": { + "query": "responsabilité dirigeants société", + "mode": "hybrid", + "filters": { + "category_ids": [3, 7], + "types": ["loi", "décret"], + "language": "fr", + "status": "published", + "year_from": 2020 + }, + "limit": 15, + "offset": 0 + } + } + + +# ============================================================================ +# Result Schemas +# ============================================================================ + +class ArticleMatch(BaseModel): + """ + Matched article within a law with highlighted snippet. + + Represents an article that matches the search query + with its relevance score and content snippet. + """ + + article_id: int = Field(..., description="Article ID") + number: str = Field(..., description="Article number (e.g., '1er', '42')") + title: Optional[str] = Field(None, description="Article title (optional)") + content_snippet: str = Field(..., description="Highlighted content snippet") + relevance_score: float = Field( + ..., + ge=0.0, + le=1.0, + description="Relevance score (0.0-1.0)" + ) + + +class SearchResult(BaseModel): + """ + Single search result with law information and highlights. + + Contains complete law metadata, relevance score, + matched articles, and highlighted text snippets. + """ + + law_id: int = Field(..., description="Law ID") + reference: str = Field(..., description="Law reference (e.g., LOI-2024-001)") + title: str = Field(..., description="Law title") + type: str = Field(..., description="Law type") + language: Optional[str] = Field(None, description="Language code (fr/en)") + status: str = Field(..., description="Status") + category_id: Optional[int] = Field(None, description="Category ID") + category_name: Optional[str] = Field(None, description="Category name") + publication_date: Optional[date] = Field(None, description="Publication date") + relevance_score: float = Field( + ..., + ge=0.0, + le=1.0, + description="Overall relevance score (0.0-1.0)" + ) + matched_articles: List[ArticleMatch] = Field( + default_factory=list, + description="List of matched articles within this law" + ) + highlights: Dict[str, str] = Field( + default_factory=dict, + description="Highlighted text snippets (title, content)" + ) + content: Optional[str] = Field( + None, + description="Full article content for RAG context" + ) + + class Config: + from_attributes = True + + +class SearchResponse(BaseModel): + """ + Complete search response with results and metadata. + + Contains search results, total count, execution time, + and applied filters for client display. + """ + + query: str = Field(..., description="Original search query") + mode: str = Field(..., description="Search mode used") + results: List[SearchResult] = Field( + default_factory=list, + description="List of search results" + ) + total: int = Field( + ..., + ge=0, + description="Total number of results found" + ) + search_time_ms: int = Field( + ..., + ge=0, + description="Search execution time in milliseconds" + ) + filters_applied: Optional[Dict] = Field( + None, + description="Filters that were applied to this search" + ) + # Article-specific navigation fields + target_article: Optional[str] = Field( + None, + description="Article number/reference to scroll to (e.g., '5', 'PREMIER')" + ) + direct_navigation: bool = Field( + False, + description="If true, client should redirect directly to the document" + ) + + class Config: + json_schema_extra = { + "example": { + "query": "responsabilité dirigeants", + "mode": "hybrid", + "results": [ + { + "law_id": 156, + "reference": "Code OHADA", + "title": "Acte uniforme relatif au droit des sociétés", + "type": "code", + "language": "fr", + "status": "published", + "category_id": 3, + "category_name": "Droit Commercial OHADA", + "relevance_score": 0.92, + "matched_articles": [], + "highlights": { + "title": "Acte uniforme relatif au droit des sociétés", + "content": "...la responsabilité des dirigeants..." + } + } + ], + "total": 15, + "search_time_ms": 187, + "filters_applied": { + "language": "fr", + "status": "published" + } + } + } + + +# ============================================================================ +# Admin Statistics Schemas +# ============================================================================ + +class SearchStats(BaseModel): + """ + Statistics about search index (admin only). + + Provides insights into indexed documents, + distribution by language/category/status, + and system health. + """ + + total_documents: int = Field( + 0, + ge=0, + description="Total documents indexed in Meilisearch" + ) + by_language: Dict[str, int] = Field( + default_factory=dict, + description="Document count by language" + ) + by_category: Dict[str, int] = Field( + default_factory=dict, + description="Document count by category" + ) + by_status: Dict[str, int] = Field( + default_factory=dict, + description="Document count by status" + ) + index_health: str = Field( + "unknown", + description="Index health status" + ) + cache_status: str = Field( + "unknown", + description="Cache connection status" + ) + + class Config: + json_schema_extra = { + "example": { + "total_documents": 156, + "by_language": { + "fr": 142, + "en": 14 + }, + "by_category": { + "Droit Civil": 45, + "Droit Pénal": 38, + "Droit Commercial": 32 + }, + "by_status": { + "published": 156, + "draft": 0 + }, + "index_health": "healthy", + "cache_status": "connected" + } + } + + +# ============================================================================ +# Reindex Response Schema +# ============================================================================ + +class ReindexResponse(BaseModel): + """Response for reindex operations.""" + + status: str = Field(..., description="Reindex status (success/failed)") + total_laws: int = Field(..., description="Total laws processed") + indexed_count: int = Field(..., description="Number of laws successfully indexed") + failed_count: int = Field(0, description="Number of laws that failed to index") + duration_seconds: int = Field(..., description="Time elapsed in seconds") diff --git a/app/schemas/user.py b/app/schemas/user.py new file mode 100644 index 0000000000000000000000000000000000000000..364c1aa49789127f65adf47bc0b61d4c2c4e1cc5 --- /dev/null +++ b/app/schemas/user.py @@ -0,0 +1,105 @@ +""" +Pydantic schemas for User model. + +Request/response schemas for user authentication and management. + +Author: JuriX Team +Date: 2026-01-12 +""" + +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel, EmailStr, Field, field_validator + +# ==================== Base Schemas ==================== + + +class UserBase(BaseModel): + """Base user schema with common fields.""" + + email: EmailStr + username: str = Field(..., min_length=3, max_length=100) + full_name: Optional[str] = Field(None, max_length=255) + role: str = Field(default="user", pattern="^(user|admin|superadmin)$") + + @field_validator("username") + @classmethod + def validate_username(cls, v: str) -> str: + """Validate username format.""" + if not v.isalnum() and "_" not in v and "-" not in v: + raise ValueError("Username must be alphanumeric (with _ or - allowed)") + return v.lower() + + +# ==================== Request Schemas ==================== + + +class UserCreate(UserBase): + """Schema for creating a new user.""" + + password: str = Field(..., min_length=8, max_length=100) + + @field_validator("password") + @classmethod + def validate_password(cls, v: str) -> str: + """Validate password strength.""" + if not any(c.isupper() for c in v): + raise ValueError("Password must contain at least one uppercase letter") + if not any(c.islower() for c in v): + raise ValueError("Password must contain at least one lowercase letter") + if not any(c.isdigit() for c in v): + raise ValueError("Password must contain at least one digit") + return v + + +class UserUpdate(BaseModel): + """Schema for updating user information.""" + + email: Optional[EmailStr] = None + username: Optional[str] = Field(None, min_length=3, max_length=100) + full_name: Optional[str] = Field(None, max_length=255) + password: Optional[str] = Field(None, min_length=8, max_length=100) + is_active: Optional[bool] = None + + +class UserLogin(BaseModel): + """Schema for user login.""" + + email: EmailStr + password: str + + +# ==================== Response Schemas ==================== + + +class UserResponse(UserBase): + """Schema for user response (public fields).""" + + id: int + is_active: bool + is_verified: bool + created_at: datetime + last_login_at: Optional[datetime] = None + + class Config: + from_attributes = True + + +class UserWithToken(UserResponse): + """Schema for user response with authentication token.""" + + access_token: str + token_type: str = "bearer" + + +# ==================== Admin Schemas ==================== + + +class UserAdminResponse(UserResponse): + """Schema for admin view of user (includes all fields).""" + + updated_at: datetime + + class Config: + from_attributes = True diff --git a/app/scripts/reindex_search.py b/app/scripts/reindex_search.py new file mode 100644 index 0000000000000000000000000000000000000000..e668c48fc81b2ee2a3677fe85200a8d979ff3664 --- /dev/null +++ b/app/scripts/reindex_search.py @@ -0,0 +1,115 @@ +""" +Script pour réindexer tous les documents dans Meilisearch. + +Usage: + python -m app.scripts.reindex_search +""" + +import asyncio +import logging +import sys +from pathlib import Path + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from sqlalchemy import select +from sqlalchemy.orm import joinedload + +from app.core.config import settings +from app.core.database import AsyncSessionLocal +from app.models.law import Law + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +async def reindex_all(): + """Reindex all published laws in Meilisearch.""" + import meilisearch + + # Initialize Meilisearch client + client = meilisearch.Client(settings.MEILISEARCH_URL, settings.MEILISEARCH_KEY) + + # Delete old index if exists + try: + client.delete_index("laws_index") + logger.info("🗑️ Deleted old index") + except Exception: + logger.info("ℹ️ No existing index to delete") + + # Create new index + client.create_index("laws_index", {"primaryKey": "id"}) + index = client.index("laws_index") + + # Configure index settings + index.update_settings({ + "searchableAttributes": ["title", "reference", "content"], + "filterableAttributes": [ + "id", "category_id", "type", "language", + "status", "publication_year" + ], + "sortableAttributes": ["publication_year", "created_at_timestamp"], + "rankingRules": [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness" + ], + "typoTolerance": { + "enabled": True, + "minWordSizeForTypos": { + "oneTypo": 4, + "twoTypos": 8 + } + } + }) + + logger.info("✅ Index created and configured") + + # Fetch all published laws + async with AsyncSessionLocal() as db: + stmt = ( + select(Law) + .where(Law.status == "published") + .options(joinedload(Law.category)) + ) + result = await db.execute(stmt) + laws = result.scalars().all() + + logger.info(f"📚 Found {len(laws)} published laws") + + # Prepare documents + documents = [] + for law in laws: + doc = { + "id": law.id, + "reference": law.reference, + "title": law.title, + "content": law.content[:10000], # Truncate to 10k chars + "type": law.type, + "language": law.language or "unknown", + "status": law.status, + "category_id": law.category_id, + "category_name": law.category.name if law.category else None, + "publication_year": ( + law.publication_date.year if law.publication_date else None + ), + "created_at_timestamp": int(law.created_at.timestamp()) + } + documents.append(doc) + + # Index documents in batches + batch_size = 100 + for i in range(0, len(documents), batch_size): + batch = documents[i:i + batch_size] + index.add_documents(batch) + logger.info(f"📦 Indexed batch {i//batch_size + 1}: {i + len(batch)}/{len(documents)}") + + logger.info(f"✅ Reindexing complete! {len(documents)} documents indexed") + + +if __name__ == "__main__": + asyncio.run(reindex_all()) diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/app/services/category_service.py b/app/services/category_service.py new file mode 100644 index 0000000000000000000000000000000000000000..ed5c0972e0efe1a82221ec87faf29e5b04790ed8 --- /dev/null +++ b/app/services/category_service.py @@ -0,0 +1,602 @@ +""" +Service 8: CategoryService - Gestion des catégories juridiques. + +Features: +- CRUD catégories (create, read, update, delete, list) +- Mapping ID ↔ Name (bidirectional lookups) +- Statistiques par catégorie (law counts, distributions) +- Health check endpoint + +Author: JuriX Team +Version: 2.1.0 +""" + +import logging +from typing import Dict, List, Optional, Any +from datetime import datetime + +from sqlalchemy import select, func, and_ +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import joinedload + +from app.models.law import Category, Law +from app.schemas.law import CategoryCreate, CategoryUpdate, CategoryResponse, CategoryStats + +# Configure logger +logger = logging.getLogger(__name__) + + +# ============================================================================ +# CUSTOM EXCEPTIONS +# ============================================================================ + +class CategoryServiceError(Exception): + """Base exception for CategoryService operations.""" + pass + + +class CategoryNotFoundError(CategoryServiceError): + """Raised when a category is not found.""" + pass + + +class DuplicateCategoryNameError(CategoryServiceError): + """Raised when trying to create/update a category with duplicate name.""" + pass + + +class CategoryInUseError(CategoryServiceError): + """Raised when trying to delete a category that has associated laws.""" + pass + + +# ============================================================================ +# CATEGORY SERVICE +# ============================================================================ + +class CategoryService: + """ + Service for managing legal categories. + + Provides CRUD operations, mapping utilities, and statistics + for the 12 Cameroonian legal categories. + + Features: + - Create, read, update, delete, list categories + - Bidirectional ID ↔ Name mapping + - Statistics: law counts and distribution percentages + - Smart delete: prevents deletion of categories with laws + - Health check endpoint + + Usage: + service = CategoryService(db_session) + categories = await service.list_categories() + mapping = await service.get_category_mapping() + stats = await service.get_all_category_stats() + """ + + def __init__(self, db: AsyncSession): + """ + Initialize CategoryService. + + Args: + db: Async database session + """ + self.db = db + logger.info("📦 CategoryService initialized") + + # ======================================================================== + # CRUD OPERATIONS + # ======================================================================== + + async def create_category(self, category_data: CategoryCreate) -> Category: + """ + Create a new category. + + Args: + category_data: Category creation data + + Returns: + Created category + + Raises: + DuplicateCategoryNameError: If category name already exists + CategoryServiceError: On database errors + """ + try: + logger.info(f"📝 Creating category: {category_data.name}") + + # Validate unique name + await self._validate_unique_name(category_data.name) + + # Create category + category = Category( + name=category_data.name, + description=category_data.description + ) + + self.db.add(category) + await self.db.commit() + await self.db.refresh(category) + + logger.info(f"✅ Category created: {category.name} (ID: {category.id})") + return category + + except DuplicateCategoryNameError: + await self.db.rollback() + raise + + except IntegrityError as e: + await self.db.rollback() + if "unique constraint" in str(e).lower() or "duplicate" in str(e).lower(): + raise DuplicateCategoryNameError( + f"Category with name '{category_data.name}' already exists" + ) + raise CategoryServiceError(f"Database integrity error: {e}") from e + + except Exception as e: + await self.db.rollback() + logger.error(f"❌ Error creating category: {e}") + raise CategoryServiceError(f"Failed to create category: {e}") from e + + async def get_category(self, category_id: int) -> Category: + """ + Get a category by ID. + + Args: + category_id: Category ID + + Returns: + Category entity + + Raises: + CategoryNotFoundError: If category doesn't exist + """ + try: + logger.debug(f"🔍 Fetching category ID: {category_id}") + + stmt = select(Category).where(Category.id == category_id) + result = await self.db.execute(stmt) + category = result.scalar_one_or_none() + + if not category: + raise CategoryNotFoundError(f"Category with ID {category_id} not found") + + logger.debug(f"✅ Category found: {category.name}") + return category + + except CategoryNotFoundError: + raise + except Exception as e: + logger.error(f"❌ Error fetching category: {e}") + raise CategoryServiceError(f"Failed to fetch category: {e}") from e + + async def list_categories( + self, + skip: int = 0, + limit: int = 100 + ) -> List[CategoryResponse]: + """ + List all categories with law counts. + + Args: + skip: Number of records to skip (pagination) + limit: Maximum number of records to return + + Returns: + List of categories with law counts + """ + try: + logger.info(f"📋 Listing categories (skip={skip}, limit={limit})") + + # Query categories with law counts + stmt = ( + select( + Category.id, + Category.name, + Category.description, + Category.icon, + Category.display_order, + Category.created_at, + func.count(Law.id).label("law_count") + ) + .outerjoin(Law, Category.id == Law.category_id) + .group_by(Category.id, Category.name, Category.description, Category.icon, Category.display_order, Category.created_at) + .order_by(Category.display_order) + .offset(skip) + .limit(limit) + ) + + result = await self.db.execute(stmt) + rows = result.all() + + # Build response + categories = [ + CategoryResponse( + id=row.id, + name=row.name, + description=row.description, + icon=row.icon, + created_at=row.created_at, + law_count=row.law_count + ) + for row in rows + ] + + logger.info(f"✅ Found {len(categories)} categories") + return categories + + except Exception as e: + logger.error(f"❌ Error listing categories: {e}") + raise CategoryServiceError(f"Failed to list categories: {e}") from e + + async def update_category( + self, + category_id: int, + category_data: CategoryUpdate + ) -> Category: + """ + Update a category. + + Args: + category_id: Category ID + category_data: Update data + + Returns: + Updated category + + Raises: + CategoryNotFoundError: If category doesn't exist + DuplicateCategoryNameError: If new name conflicts + """ + try: + logger.info(f"✏️ Updating category ID: {category_id}") + + # Fetch category + category = await self.get_category(category_id) + + # Validate unique name if changing + if category_data.name and category_data.name != category.name: + await self._validate_unique_name(category_data.name, exclude_id=category_id) + category.name = category_data.name + + # Update description + if category_data.description is not None: + category.description = category_data.description + + await self.db.commit() + await self.db.refresh(category) + + logger.info(f"✅ Category updated: {category.name}") + return category + + except (CategoryNotFoundError, DuplicateCategoryNameError): + await self.db.rollback() + raise + + except IntegrityError as e: + await self.db.rollback() + if "unique constraint" in str(e).lower(): + raise DuplicateCategoryNameError( + f"Category with name '{category_data.name}' already exists" + ) + raise CategoryServiceError(f"Database integrity error: {e}") from e + + except Exception as e: + await self.db.rollback() + logger.error(f"❌ Error updating category: {e}") + raise CategoryServiceError(f"Failed to update category: {e}") from e + + async def delete_category(self, category_id: int, force: bool = False) -> bool: + """ + Delete a category. + + Args: + category_id: Category ID + force: If True, allows deletion even if laws are associated + (laws will have category_id set to NULL) + + Returns: + True if deleted successfully + + Raises: + CategoryNotFoundError: If category doesn't exist + CategoryInUseError: If category has laws and force=False + """ + try: + logger.info(f"🗑️ Deleting category ID: {category_id} (force={force})") + + # Fetch category + category = await self.get_category(category_id) + + # Check if category has associated laws + law_count = await self._get_law_count_for_category(category_id) + + if law_count > 0 and not force: + raise CategoryInUseError( + f"Cannot delete category '{category.name}': " + f"{law_count} law(s) are associated. Use force=True to delete anyway." + ) + + # If force=True and laws exist, set their category_id to NULL + if law_count > 0 and force: + logger.warning( + f"⚠️ Force deleting category with {law_count} law(s). " + f"Setting category_id to NULL for affected laws." + ) + stmt = ( + select(Law) + .where(Law.category_id == category_id) + ) + result = await self.db.execute(stmt) + laws = result.scalars().all() + + for law in laws: + law.category_id = None + + # Delete category + await self.db.delete(category) + await self.db.commit() + + logger.info(f"✅ Category deleted: {category.name}") + return True + + except (CategoryNotFoundError, CategoryInUseError): + await self.db.rollback() + raise + + except Exception as e: + await self.db.rollback() + logger.error(f"❌ Error deleting category: {e}") + raise CategoryServiceError(f"Failed to delete category: {e}") from e + + # ======================================================================== + # MAPPING UTILITIES (ID ↔ Name) + # ======================================================================== + + async def get_category_mapping(self) -> Dict[int, str]: + """ + Get ID → Name mapping for all categories. + + Returns: + Dictionary mapping category IDs to names + + Example: + {1: "Droit Civil", 2: "Droit Pénal", ...} + """ + try: + logger.debug("🗺️ Fetching category ID → Name mapping") + + stmt = select(Category.id, Category.name).order_by(Category.name) + result = await self.db.execute(stmt) + rows = result.all() + + mapping = {row.id: row.name for row in rows} + logger.debug(f"✅ Mapping fetched: {len(mapping)} categories") + return mapping + + except Exception as e: + logger.error(f"❌ Error fetching mapping: {e}") + raise CategoryServiceError(f"Failed to fetch category mapping: {e}") from e + + async def get_reverse_category_mapping(self) -> Dict[str, int]: + """ + Get Name → ID mapping for all categories. + + Returns: + Dictionary mapping category names to IDs + + Example: + {"Droit Civil": 1, "Droit Pénal": 2, ...} + """ + try: + logger.debug("🗺️ Fetching category Name → ID mapping") + + stmt = select(Category.name, Category.id).order_by(Category.name) + result = await self.db.execute(stmt) + rows = result.all() + + mapping = {row.name: row.id for row in rows} + logger.debug(f"✅ Reverse mapping fetched: {len(mapping)} categories") + return mapping + + except Exception as e: + logger.error(f"❌ Error fetching reverse mapping: {e}") + raise CategoryServiceError(f"Failed to fetch reverse category mapping: {e}") from e + + async def get_category_by_name(self, name: str) -> Optional[Category]: + """ + Get a category by name (case-insensitive). + + Args: + name: Category name + + Returns: + Category entity or None if not found + """ + try: + logger.debug(f"🔍 Fetching category by name: {name}") + + stmt = select(Category).where(func.lower(Category.name) == name.lower()) + result = await self.db.execute(stmt) + category = result.scalar_one_or_none() + + if category: + logger.debug(f"✅ Category found: {category.name}") + else: + logger.debug(f"⚠️ Category not found: {name}") + + return category + + except Exception as e: + logger.error(f"❌ Error fetching category by name: {e}") + raise CategoryServiceError(f"Failed to fetch category by name: {e}") from e + + # ======================================================================== + # STATISTICS + # ======================================================================== + + async def get_category_stats(self, category_id: int) -> CategoryStats: + """ + Get statistics for a single category. + + Args: + category_id: Category ID + + Returns: + Category statistics with law count and percentage + + Raises: + CategoryNotFoundError: If category doesn't exist + """ + try: + logger.debug(f"📊 Fetching stats for category ID: {category_id}") + + # Fetch category + category = await self.get_category(category_id) + + # Get total laws + total_laws_stmt = select(func.count(Law.id)) + total_laws = (await self.db.execute(total_laws_stmt)).scalar_one() + + # Get law count for this category + law_count = await self._get_law_count_for_category(category_id) + + # Calculate percentage + percentage = (law_count / total_laws * 100) if total_laws > 0 else 0.0 + + stats = CategoryStats( + category_id=category.id, + category_name=category.name, + law_count=law_count, + percentage=percentage + ) + + logger.debug(f"✅ Stats fetched: {category.name} - {law_count} laws ({percentage:.2f}%)") + return stats + + except CategoryNotFoundError: + raise + except Exception as e: + logger.error(f"❌ Error fetching category stats: {e}") + raise CategoryServiceError(f"Failed to fetch category stats: {e}") from e + + async def get_all_category_stats(self) -> List[CategoryStats]: + """ + Get statistics for all categories. + + Returns: + List of category statistics ordered by law count (descending) + """ + try: + logger.info("📊 Fetching stats for all categories") + + # Get total laws + total_laws_stmt = select(func.count(Law.id)) + total_laws = (await self.db.execute(total_laws_stmt)).scalar_one() + + # Get law counts per category + stmt = ( + select( + Category.id, + Category.name, + func.count(Law.id).label("law_count") + ) + .outerjoin(Law, Category.id == Law.category_id) + .group_by(Category.id, Category.name) + .order_by(func.count(Law.id).desc(), Category.name) + ) + + result = await self.db.execute(stmt) + rows = result.all() + + # Build stats list + stats_list = [ + CategoryStats( + category_id=row.id, + category_name=row.name, + law_count=row.law_count, + percentage=(row.law_count / total_laws * 100) if total_laws > 0 else 0.0 + ) + for row in rows + ] + + logger.info(f"✅ Stats fetched for {len(stats_list)} categories") + return stats_list + + except Exception as e: + logger.error(f"❌ Error fetching all category stats: {e}") + raise CategoryServiceError(f"Failed to fetch all category stats: {e}") from e + + # ======================================================================== + # HEALTH CHECK + # ======================================================================== + + def health_check(self) -> Dict[str, Any]: + """ + Check service health. + + Returns: + Health status dictionary + """ + return { + "service": "CategoryService", + "status": "healthy", + "version": "2.1.0", + "timestamp": datetime.utcnow().isoformat() + } + + # ======================================================================== + # PRIVATE HELPERS + # ======================================================================== + + async def _validate_unique_name( + self, + name: str, + exclude_id: Optional[int] = None + ) -> None: + """ + Validate that category name is unique. + + Args: + name: Category name to validate + exclude_id: Category ID to exclude from check (for updates) + + Raises: + DuplicateCategoryNameError: If name already exists + """ + conditions = [func.lower(Category.name) == name.lower()] + if exclude_id is not None: + conditions.append(Category.id != exclude_id) + + stmt = select(Category).where(and_(*conditions)) + result = await self.db.execute(stmt) + existing = result.scalar_one_or_none() + + if existing: + raise DuplicateCategoryNameError( + f"Category with name '{name}' already exists" + ) + + async def _get_law_count_for_category(self, category_id: int) -> int: + """ + Get number of laws associated with a category. + + Args: + category_id: Category ID + + Returns: + Number of associated laws + """ + stmt = select(func.count(Law.id)).where(Law.category_id == category_id) + count = (await self.db.execute(stmt)).scalar_one() + return count + + +# ============================================================================ +# MODULE-LEVEL UTILITIES +# ============================================================================ + +# Note: CategoryService is stateful (requires db session per request) +# No singleton pattern needed - instances created per request via dependency injection diff --git a/app/services/document_classifier.py b/app/services/document_classifier.py new file mode 100644 index 0000000000000000000000000000000000000000..1eee2087af028a588147f7eb8d66e613b1a3a005 --- /dev/null +++ b/app/services/document_classifier.py @@ -0,0 +1,496 @@ +""" +Service de catégorisation automatique de documents juridiques. + +Ce service utilise une approche hybride pour classifier les documents: +- Keywords matching (40%): Rapide, transparent, basé sur dictionnaires +- ML (60%): TF-IDF + Logistic Regression (Phase 3) + +Phase 1 (MVP): Keywords-only (70-75% précision) +Phase 3: Hybrid 40/60 (87% top-1, 96% top-3) + +12 catégories juridiques camerounaises: +1. Droit Constitutionnel +2. Droit Civil +3. Droit Pénal +4. Droit Commercial OHADA +5. Droit du Travail +6. Droit Fiscal +7. Droit Administratif +8. Droit Foncier +9. Droit de la Famille +10. Droit de l'Environnement +11. Droit International +12. Droit des Affaires + +Usage: + classifier = DocumentClassifier() + result = classifier.classify("Article 1. La SARL est une société commerciale...") + # [(4, 0.85, 'keyword'), (12, 0.62, 'keyword'), (2, 0.45, 'keyword')] +""" + +import logging +import re +import time +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +logger = logging.getLogger(__name__) + + +class DocumentClassificationError(Exception): + """Exception levée lors d'erreurs de classification.""" + + pass + + +class DocumentClassifier: + """ + Service de catégorisation automatique de documents juridiques. + + Approche hybrid: + - Phase 1 (MVP): Keywords-only (70-75% précision) + - Phase 3: 40% Keywords + 60% ML (87% top-1, 96% top-3) + + Performance cible: <2s (MVP: <500ms) + """ + + # Dictionnaires de mots-clés par catégorie + # Primary keywords: poids 1.0 + # Secondary keywords: poids 0.5 + CATEGORY_KEYWORDS = { + 1: { # Droit Constitutionnel + 'primary': [ + 'constitution', 'parlement', 'président', + 'assemblée nationale', 'sénat', 'référendum' + ], + 'secondary': [ + 'pouvoir exécutif', 'pouvoir législatif', 'institutions' + ] + }, + 2: { # Droit Civil + 'primary': [ + 'contrat', 'obligations', 'responsabilité civile', + 'dommages', 'code civil' + ], + 'secondary': [ + 'débiteur', 'créancier', 'prescription' + ] + }, + 3: { # Droit Pénal + 'primary': [ + 'crime', 'délit', 'peine', 'emprisonnement', + 'code pénal', 'infraction' + ], + 'secondary': [ + 'criminel', 'tribunal correctionnel', 'procès' + ] + }, + 4: { # Droit Commercial OHADA + 'primary': [ + 'ohada', 'sarl', 'sa', 'capital social', + 'parts sociales', 'acte uniforme' + ], + 'secondary': [ + 'associés', 'actionnaires', 'dirigeants', 'société commerciale' + ] + }, + 5: { # Droit du Travail + 'primary': [ + 'salarié', 'employeur', 'licenciement', + 'contrat de travail', 'code du travail' + ], + 'secondary': [ + 'salaire', 'congés', 'inspection du travail' + ] + }, + 6: { # Droit Fiscal + 'primary': [ + 'impôt', 'taxe', 'tva', 'is', 'irpp', 'fiscalité' + ], + 'secondary': [ + 'contribuable', 'recouvrement', 'administration fiscale' + ] + }, + 7: { # Droit Administratif + 'primary': [ + 'fonction publique', 'service public', 'administration', + 'contentieux administratif' + ], + 'secondary': [ + 'agent public', 'autorité administrative' + ] + }, + 8: { # Droit Foncier + 'primary': [ + 'terrain', 'titre foncier', 'cadastre', + 'domaine', 'immatriculation' + ], + 'secondary': [ + 'propriété foncière', 'lotissement' + ] + }, + 9: { # Droit de la Famille + 'primary': [ + 'mariage', 'divorce', 'succession', + 'filiation', 'autorité parentale' + ], + 'secondary': [ + 'conjoint', 'héritier', 'tutelle' + ] + }, + 10: { # Droit de l'Environnement + 'primary': [ + 'environnement', 'pollution', 'écologie', + 'protection', 'ressources naturelles' + ], + 'secondary': [ + 'développement durable', 'faune', 'flore' + ] + }, + 11: { # Droit International + 'primary': [ + 'traité', 'convention', 'nations unies', + 'international', 'ratification' + ], + 'secondary': [ + 'accord bilatéral', 'souveraineté' + ] + }, + 12: { # Droit des Affaires + 'primary': [ + 'contrat commercial', 'franchise', + 'bail commercial', 'concurrence' + ], + 'secondary': [ + 'commerce', 'registre du commerce' + ] + } + } + + # Noms des catégories (pour référence) + CATEGORY_NAMES = { + 1: "Droit Constitutionnel", + 2: "Droit Civil", + 3: "Droit Pénal", + 4: "Droit Commercial OHADA", + 5: "Droit du Travail", + 6: "Droit Fiscal", + 7: "Droit Administratif", + 8: "Droit Foncier", + 9: "Droit de la Famille", + 10: "Droit de l'Environnement", + 11: "Droit International", + 12: "Droit des Affaires" + } + + # Limites de texte + MIN_TEXT_LENGTH = 50 + MAX_TEXT_LENGTH = 250000 # Increased from 50000 to handle large legal documents + + def __init__(self, model_path: Optional[Path] = None): + """ + Initialise le classifier. + + Phase 1 (MVP): Keywords-only + Phase 3: Charge les modèles ML si disponibles + + Args: + model_path: Chemin vers les modèles ML (optionnel) + """ + self.ml_model = None + self.vectorizer = None + + # Tentative de chargement des modèles ML (Phase 3) + if model_path is None: + model_path = Path(__file__).parent.parent.parent / "models" + + self._load_ml_models(model_path) + + # Log du mode de fonctionnement + if self.ml_model and self.vectorizer: + logger.info("✅ DocumentClassifier initialisé (mode hybrid)") + else: + logger.info("✅ DocumentClassifier initialisé (mode keywords-only)") + + def _load_ml_models(self, model_path: Path): + """ + Charge les modèles ML s'ils existent (Phase 3). + + Args: + model_path: Répertoire contenant les modèles + """ + try: + import joblib + + vectorizer_path = model_path / "category_vectorizer.pkl" + classifier_path = model_path / "category_classifier.pkl" + + if vectorizer_path.exists() and classifier_path.exists(): + self.vectorizer = joblib.load(vectorizer_path) + self.ml_model = joblib.load(classifier_path) + logger.info(f"✅ Modèles ML chargés depuis {model_path}") + else: + logger.info(f"ℹ️ Modèles ML non trouvés dans {model_path} (mode keywords-only)") + + except ImportError: + logger.warning("⚠️ joblib non disponible, mode keywords-only uniquement") + except Exception as e: + logger.error(f"❌ Erreur chargement modèles ML: {e}") + + def classify( + self, + text: str, + language: str = "fr", + top_k: int = 3, + min_confidence: float = 0.0 + ) -> List[Tuple[int, float, str]]: + """ + Classifie un document juridique. + + Args: + text: Texte du document à classifier + language: Langue du texte ('fr' ou 'en') - pour Phase 3 + top_k: Nombre de suggestions à retourner (1-5) + min_confidence: Seuil de confiance minimum (0.0-1.0) + + Returns: + Liste de tuples (category_id, confidence, method) + Exemple: [(4, 0.92, 'hybrid'), (2, 0.78, 'keyword'), (6, 0.54, 'keyword')] + + Raises: + ValueError: Si texte invalide + DocumentClassificationError: Si erreur de classification + """ + assert isinstance(text, str) and len(text) > 0, "text must be a non-empty string" + assert isinstance(top_k, int) and 1 <= top_k <= 5, "top_k must be between 1 and 5" + + start_time = time.time() + + # Validation entrée + self._validate_input(text, top_k) + + # Preprocessing + processed_text = self._preprocess_text(text) + + # Méthode 1: Keywords (toujours exécutée) + keyword_scores = self._classify_keywords(processed_text) + + # Méthode 2: ML (si disponible, Phase 3) + if self.ml_model and self.vectorizer: + ml_scores = self._classify_ml(processed_text) + + # Fusion: 40% keywords + 60% ML + final_scores = {} + all_categories = set(keyword_scores.keys()) | set(ml_scores.keys()) + + for cat_id in all_categories: + kw_score = keyword_scores.get(cat_id, 0.0) + ml_score = ml_scores.get(cat_id, 0.0) + final_scores[cat_id] = 0.4 * kw_score + 0.6 * ml_score + + method = 'hybrid' + else: + # Phase 1: Keywords-only + final_scores = keyword_scores + method = 'keyword' + + # Filtrer par seuil de confiance + final_scores = { + cat_id: score + for cat_id, score in final_scores.items() + if score >= min_confidence + } + + # Trier et retourner top-k + sorted_results = sorted( + final_scores.items(), + key=lambda x: x[1], + reverse=True + )[:top_k] + + # Formatter résultats + results = [ + (cat_id, float(score), method) + for cat_id, score in sorted_results + ] + + processing_time = (time.time() - start_time) * 1000 + + logger.info( + f"Classification terminée: {len(results)} suggestions, " + f"top={results[0] if results else 'N/A'}, " + f"méthode={method}, temps={processing_time:.0f}ms" + ) + + return results + + def _validate_input(self, text: str, top_k: int): + """ + Valide les paramètres d'entrée. + + Args: + text: Texte à valider + top_k: Nombre de suggestions + + Raises: + ValueError: Si paramètres invalides + """ + if not text or not text.strip(): + raise ValueError("Le texte ne peut pas être vide") + + if len(text) < self.MIN_TEXT_LENGTH: + raise ValueError( + f"Texte trop court: {len(text)} caractères " + f"(minimum {self.MIN_TEXT_LENGTH})" + ) + + if len(text) > self.MAX_TEXT_LENGTH: + raise ValueError( + f"Texte trop long: {len(text)} caractères " + f"(maximum {self.MAX_TEXT_LENGTH})" + ) + + if top_k < 1 or top_k > 12: + raise ValueError(f"top_k doit être entre 1 et 12 (reçu: {top_k})") + + def _preprocess_text(self, text: str) -> str: + """ + Prétraite le texte pour classification. + + - Convertit en minuscules + - Supprime espaces excessifs + - Supprime numéros d'articles + - Préserve accents français + + Args: + text: Texte brut + + Returns: + Texte prétraité + """ + # Conversion minuscules + text = text.lower() + + # Suppression numéros d'articles (Art. 1, Article 2, etc.) + text = re.sub(r'\bart\.?\s*\d+', '', text) + text = re.sub(r'\barticle\s*\d+', '', text) + + # Suppression espaces multiples + text = re.sub(r'\s+', ' ', text) + + # Strip + text = text.strip() + + return text + + def _classify_keywords(self, text: str) -> Dict[int, float]: + """ + Classification par keywords avec scores pondérés. + + Logique: + - Compte occurrences de chaque keyword (regex word boundaries) + - Score = Σ(primary_matches * 1.0) + Σ(secondary_matches * 0.5) + - Normalise: min(score / 10, 1.0) → [0, 1] + + Args: + text: Texte prétraité + + Returns: + Dictionnaire {category_id: confidence_score} + """ + scores = {} + + for cat_id, keywords in self.CATEGORY_KEYWORDS.items(): + score = 0.0 + + # Primary keywords (poids 1.0) + for keyword in keywords['primary']: + # Utilise word boundaries pour éviter matches partiels + pattern = r'\b' + re.escape(keyword) + r'\b' + matches = len(re.findall(pattern, text)) + score += matches * 1.0 + + # Secondary keywords (poids 0.5) + for keyword in keywords['secondary']: + pattern = r'\b' + re.escape(keyword) + r'\b' + matches = len(re.findall(pattern, text)) + score += matches * 0.5 + + # Normalisation: score / 10, max 1.0 + # Score de 10+ = 100% confiance + normalized_score = min(score / 10.0, 1.0) + + if normalized_score > 0: + scores[cat_id] = normalized_score + + return scores + + def _classify_ml(self, text: str) -> Dict[int, float]: + """ + Classification par ML (TF-IDF + Logistic Regression). + + Phase 3 uniquement. + + Args: + text: Texte prétraité + + Returns: + Dictionnaire {category_id: probability} + """ + try: + # Vectorisation TF-IDF + X = self.vectorizer.transform([text]) + + # Prédiction probabilités + probabilities = self.ml_model.predict_proba(X)[0] + + # Map vers category IDs + scores = { + int(cat_id): float(prob) + for cat_id, prob in zip(self.ml_model.classes_, probabilities) + } + + return scores + + except Exception as e: + logger.error(f"❌ Erreur classification ML: {e}") + return {} + + def health_check(self) -> Dict[str, Any]: + """ + Vérifie l'état de santé du service. + + Returns: + Dictionnaire avec état du service + """ + return { + "service": "DocumentClassifier", + "status": "healthy", + "mode": "hybrid" if (self.ml_model and self.vectorizer) else "keywords-only", + "ml_model_loaded": self.ml_model is not None, + "vectorizer_loaded": self.vectorizer is not None, + "categories_count": len(self.CATEGORY_KEYWORDS), + "version": "1.0.0-mvp" + } + + def get_category_name(self, category_id: int) -> Optional[str]: + """ + Retourne le nom d'une catégorie. + + Args: + category_id: ID de la catégorie (1-12) + + Returns: + Nom de la catégorie ou None si invalide + """ + return self.CATEGORY_NAMES.get(category_id) + + def get_all_categories(self) -> Dict[int, str]: + """ + Retourne toutes les catégories disponibles. + + Returns: + Dictionnaire {id: nom} + """ + return self.CATEGORY_NAMES.copy() diff --git a/app/services/embedding_service.py b/app/services/embedding_service.py new file mode 100644 index 0000000000000000000000000000000000000000..fa38f97e2e6f3e286c1388c74b6162a29c22b264 --- /dev/null +++ b/app/services/embedding_service.py @@ -0,0 +1,607 @@ +""" +Service de génération d'embeddings vectoriels pour recherche sémantique. + +Ce service utilise Gemini API pour générer des embeddings 3072-dim +multilingues (FR/EN) avec cache Redis pour optimiser les performances. + +Architecture: +- Model: models/gemini-embedding-001 (Gemini API) +- Dimensions: 3072 (compatible pgvector) +- Cache: Redis avec TTL 7 jours +- Performance: <300ms single, <2s batch(10) + +Usage: + service = EmbeddingService() + + # Single embedding + emb = service.generate_embedding("Article 1er du Code civil") + # np.ndarray shape (3072,) + + # Batch embeddings + embs = service.generate_batch_embeddings([ + "Article 1er...", + "Article 2..." + ]) + # List[np.ndarray] + + # Similarity + score = service.similarity(emb1, emb2) # 0.0 to 1.0 + +Performance cible: +- Single: <300ms +- Batch (10): <2s +- Batch (32): <6s +- Cache hit: <10ms + +Author: JuriX Team +Version: 2.0.0 (Gemini API) +""" + +import hashlib +import json +import logging +import time +from typing import Any, Dict, List, Optional + +import numpy as np +import redis +import google.generativeai as genai +from app.core.config import settings + +logger = logging.getLogger(__name__) + + +class EmbeddingServiceError(Exception): + """Exception levée lors d'erreurs de génération d'embeddings.""" + pass + + +class EmbeddingService: + """ + Service de génération d'embeddings vectoriels pour recherche sémantique. + + Utilise Gemini API avec modèle gemini-embedding-001 optimisé pour: + - Recherche sémantique dans textes juridiques + - Support français et anglais + - Embeddings de dimension 3072 (compatible pgvector) + - Cache Redis pour performances optimales + + Caractéristiques: + - Model: models/gemini-embedding-001 (Gemini) + - Normalisation L2 automatique (cosine similarity) + - Batch processing cache-aware + - Retry logic pour robustesse + - Graceful degradation si Redis indisponible + + Performance: + - Single embedding: <300ms + - Batch (10 embeddings): <2s + - Cache hit: <10ms + + Attributes: + EMBEDDING_MODEL: Nom du modèle Gemini + EMBEDDING_DIM: Dimension des embeddings (3072) + CACHE_TTL_SECONDS: Durée de vie cache (7 jours) + redis_client: Client Redis (optionnel) + use_cache: Flag activation cache + """ + + # Configuration constantes + EMBEDDING_MODEL = settings.GEMINI_EMBEDDING_MODEL + EMBEDDING_DIM = 3072 + CACHE_TTL_SECONDS = 7 * 24 * 60 * 60 # 7 jours + MAX_TEXT_LENGTH = 10000 # Limite sécurité + MAX_RETRIES = 3 + RETRY_DELAY = 1 # seconds + + def __init__( + self, + redis_url: Optional[str] = None, + use_cache: bool = True, + api_key: Optional[str] = None + ): + """ + Initialise le service d'embeddings. + + Configure Gemini API et initialise optionnellement la connexion + Redis pour le cache. + + Args: + redis_url: URL connexion Redis (ex: "redis://localhost:6379/0"). + Si None, utilise settings.REDIS_URL par défaut. + use_cache: Active/désactive le cache Redis + api_key: Clé API Gemini (si None, utilise settings.GEMINI_API_KEY) + + Raises: + EmbeddingServiceError: Si Gemini API ne peut pas être initialisée + + Example: + >>> # Avec cache (défaut) + >>> service = EmbeddingService() + >>> + >>> # Sans cache + >>> service = EmbeddingService(use_cache=False) + >>> + >>> # Redis custom + >>> service = EmbeddingService( + ... redis_url="redis://:password@localhost:6379/1" + ... ) + """ + logger.info("🚀 Initialisation EmbeddingService (Gemini API)...") + + # Initialisation Gemini API + try: + api_key = api_key or settings.GEMINI_API_KEY + if not api_key: + raise EmbeddingServiceError("GEMINI_API_KEY non configurée") + + genai.configure(api_key=api_key) + logger.info(f"✅ Gemini API configurée (model: {self.EMBEDDING_MODEL})") + except Exception as e: + logger.error(f"❌ Échec configuration Gemini API: {e}") + raise EmbeddingServiceError( + f"Impossible de configurer Gemini API: {e}" + ) from e + + # Initialisation cache Redis + self.use_cache = use_cache + self.redis_client = None + + if use_cache: + try: + redis_url = redis_url or settings.REDIS_URL + self.redis_client = redis.from_url( + redis_url, + decode_responses=False, # Binary pour np.array + socket_connect_timeout=2, + socket_timeout=2 + ) + # Test connexion + self.redis_client.ping() + logger.info(f"✅ Cache Redis connecté: {redis_url}") + except Exception as e: + logger.warning( + f"⚠️ Redis non disponible, cache désactivé: {e}" + ) + self.redis_client = None + self.use_cache = False + else: + logger.info("ℹ️ Cache Redis désactivé") + + logger.info( + f"✅ EmbeddingService initialisé " + f"(dim={self.EMBEDDING_DIM}, cache={self.use_cache}, provider=Gemini)" + ) + + def generate_embedding( + self, + text: str, + normalize: bool = True + ) -> np.ndarray: + """ + Génère l'embedding pour un texte unique via Gemini API. + + Vérifie d'abord le cache Redis, génère l'embedding si nécessaire, + puis le stocke dans le cache pour utilisation future. + + Args: + text: Texte à encoder (article de loi, question, etc.) + normalize: Applique normalisation L2 (True recommandé pour cosine) + + Returns: + Embedding numpy array de dimension (3072,), normalisé si demandé + + Raises: + ValueError: Si texte vide ou trop long (>10000 chars) + EmbeddingServiceError: Si génération échoue + + Performance: + - Cache hit: <10ms + - Cache miss: <300ms (Gemini API) + + Example: + >>> service = EmbeddingService() + >>> emb = service.generate_embedding( + ... "Article 1er du Code civil camerounais" + ... ) + >>> emb.shape + (3072,) + >>> np.linalg.norm(emb) # Normalized + 1.0 + """ + assert isinstance(text, str), "text must be a string" + assert isinstance(normalize, bool), "normalize must be a boolean" + + start_time = time.time() + + # Validation + self._validate_text(text) + + # Prétraitement pour cache key + cache_key = self._get_cache_key(text) + + # Tentative récupération cache + if self.use_cache and self.redis_client: + cached_embedding = self._get_from_cache(cache_key) + if cached_embedding is not None: + elapsed = (time.time() - start_time) * 1000 + logger.debug(f"🔍 Cache HIT ({elapsed:.1f}ms): {cache_key[:20]}...") + return cached_embedding + + # Génération embedding via Gemini API + for attempt in range(self.MAX_RETRIES): + try: + result = genai.embed_content( + model=self.EMBEDDING_MODEL, + content=text, + task_type="retrieval_document" + ) + + embedding = np.array(result['embedding'], dtype=np.float32) + + # Validation dimension + if embedding.shape[0] != self.EMBEDDING_DIM: + raise EmbeddingServiceError( + f"Dimension incorrecte: {embedding.shape[0]} != {self.EMBEDDING_DIM}" + ) + + # Normalisation L2 si demandée + if normalize: + norm = np.linalg.norm(embedding) + if norm > 0: + embedding = embedding / norm + + # Stockage cache + if self.use_cache and self.redis_client: + self._store_in_cache(cache_key, embedding) + + elapsed = (time.time() - start_time) * 1000 + logger.debug( + f"⏱️ Embedding généré en {elapsed:.1f}ms " + f"(dim={embedding.shape[0]}, norm={normalize})" + ) + + return embedding + + except Exception as e: + if attempt < self.MAX_RETRIES - 1: + logger.warning(f"⚠️ Tentative {attempt + 1} échouée, retry: {e}") + time.sleep(self.RETRY_DELAY * (attempt + 1)) + else: + logger.error(f"❌ Erreur génération embedding après {self.MAX_RETRIES} tentatives: {e}") + raise EmbeddingServiceError( + f"Échec génération embedding: {e}" + ) from e + + def generate_batch_embeddings( + self, + texts: List[str], + batch_size: int = 20, + normalize: bool = True, + max_retries: int = 3, + retry_delay: float = 5.0 + ) -> List[np.ndarray]: + """ + Génère les embeddings pour un batch de textes (cache-aware). + + Pipeline: validate → check cache → generate missing → reconstitute. + + Args: + texts: Liste de textes à encoder + batch_size: Taille max des chunks pour l'API + normalize: Applique normalisation L2 + max_retries: Nombre max de tentatives par chunk + retry_delay: Délai entre tentatives + + Returns: + Liste d'embeddings numpy array (ordre préservé) + + Raises: + ValueError: Si liste vide ou textes invalides + EmbeddingServiceError: Si génération échoue + """ + assert texts, "La liste de textes ne peut pas être vide" + start_time = time.time() + + # Validate all texts + for idx, text in enumerate(texts): + try: + self._validate_text(text) + except ValueError as e: + raise ValueError(f"Texte invalide à l'index {idx}: {e}") from e + + logger.info(f"🔄 Génération batch: {len(texts)} textes...") + + # Phase 1: Check cache + embeddings_dict, texts_to_generate, indices_to_generate = self._check_batch_cache(texts) + cache_hits = len(texts) - len(texts_to_generate) + logger.info(f"📊 Cache: {cache_hits} hits, {len(texts_to_generate)} misses") + + # Phase 2: Generate missing embeddings in chunks + if texts_to_generate: + total = len(texts_to_generate) + chunk_count = (total + batch_size - 1) // batch_size + logger.info(f"📦 Processing {total} texts in {chunk_count} chunks of max {batch_size}") + + for chunk_idx in range(0, total, batch_size): + chunk_texts = texts_to_generate[chunk_idx:chunk_idx + batch_size] + chunk_indices = indices_to_generate[chunk_idx:chunk_idx + batch_size] + current_chunk = (chunk_idx // batch_size) + 1 + + self._generate_chunk_with_retry( + chunk_texts, chunk_indices, embeddings_dict, + current_chunk, chunk_count, normalize, max_retries, retry_delay + ) + + # Rate limit delay between chunks + if chunk_idx + batch_size < total: + time.sleep(0.5) + + # Phase 3: Reconstitute original order + embeddings_list = [embeddings_dict[i] for i in range(len(texts))] + + elapsed = (time.time() - start_time) * 1000 + logger.info( + f"✅ Batch généré en {elapsed:.0f}ms " + f"({len(texts)} embeddings, {cache_hits} from cache)" + ) + return embeddings_list + + def _check_batch_cache(self, texts: List[str]): + """Check cache for all texts, return dict of hits and lists of misses.""" + embeddings_dict = {} + texts_to_generate = [] + indices_to_generate = [] + + for idx, text in enumerate(texts): + cache_key = self._get_cache_key(text) + if self.use_cache and self.redis_client: + cached_emb = self._get_from_cache(cache_key) + if cached_emb is not None: + embeddings_dict[idx] = cached_emb + continue + texts_to_generate.append(text) + indices_to_generate.append(idx) + + return embeddings_dict, texts_to_generate, indices_to_generate + + def _generate_chunk_with_retry( + self, chunk_texts, chunk_indices, embeddings_dict, + current_chunk, chunk_count, normalize, max_retries, retry_delay + ): + """Generate embeddings for a single chunk with retry logic.""" + logger.info(f" 📝 Chunk {current_chunk}/{chunk_count}: {len(chunk_texts)} texts...") + + for attempt in range(max_retries): + try: + result = genai.embed_content( + model=self.EMBEDDING_MODEL, + content=chunk_texts, + task_type="retrieval_document" + ) + new_embeddings = [ + np.array(emb, dtype=np.float32) for emb in result['embedding'] + ] + + if normalize: + new_embeddings = [ + emb / np.linalg.norm(emb) if np.linalg.norm(emb) > 0 else emb + for emb in new_embeddings + ] + + # Store in dict and cache + for idx, text, embedding in zip(chunk_indices, chunk_texts, new_embeddings): + embeddings_dict[idx] = embedding + if self.use_cache and self.redis_client: + cache_key = self._get_cache_key(text) + self._store_in_cache(cache_key, embedding) + + logger.info(f" ✅ Chunk {current_chunk}/{chunk_count} completed") + return # Success + + except Exception as e: + if attempt < max_retries - 1: + wait_time = retry_delay * (attempt + 1) + logger.warning( + f" ⚠️ Chunk {current_chunk} failed (attempt {attempt + 1}/{max_retries}): {e}. " + f"Retrying in {wait_time}s..." + ) + time.sleep(wait_time) + else: + logger.error(f"❌ Chunk {current_chunk} failed after {max_retries} attempts: {e}") + raise EmbeddingServiceError( + f"Échec génération batch embeddings après {max_retries} tentatives: {e}" + ) from e + + def similarity( + self, + emb1: np.ndarray, + emb2: np.ndarray + ) -> float: + """ + Calcule la similarité cosine entre deux embeddings. + + Args: + emb1: Premier embedding (3072,) + emb2: Deuxième embedding (3072,) + + Returns: + Score de similarité entre 0.0 et 1.0 + (1.0 = identiques, 0.0 = orthogonaux) + + Raises: + ValueError: Si dimensions incorrectes ou non-matching + + Example: + >>> service = EmbeddingService() + >>> emb1 = service.generate_embedding("Article 1er") + >>> emb2 = service.generate_embedding("Article premier") + >>> score = service.similarity(emb1, emb2) + >>> score > 0.8 # Très similaires + True + """ + # Validation dimensions + if emb1.shape != (self.EMBEDDING_DIM,): + raise ValueError( + f"emb1 dimension incorrecte: {emb1.shape} != ({self.EMBEDDING_DIM},)" + ) + if emb2.shape != (self.EMBEDDING_DIM,): + raise ValueError( + f"emb2 dimension incorrecte: {emb2.shape} != ({self.EMBEDDING_DIM},)" + ) + + # Cosine similarity (dot product si normalisés) + similarity_score = float(np.dot(emb1, emb2)) + + # Clamp entre 0 et 1 (erreurs arrondis) + return max(0.0, min(1.0, similarity_score)) + + def health_check(self) -> Dict[str, Any]: + """ + Vérifie l'état de santé du service. + + Teste: + - Disponibilité de Gemini API (génération test) + - Connectivité Redis si cache activé + - Configuration actuelle + + Returns: + Dictionnaire avec status et métadonnées + + Example: + >>> service = EmbeddingService() + >>> health = service.health_check() + >>> health['status'] + 'healthy' + >>> health['dimensions'] + 3072 + """ + status_info = { + "service": "EmbeddingService", + "provider": "Gemini API", + "model": self.EMBEDDING_MODEL, + "dimensions": self.EMBEDDING_DIM, + "cache_enabled": self.use_cache, + "status": "healthy" + } + + # Test Gemini API + try: + test_emb = self.generate_embedding("test", normalize=False) + if test_emb.shape[0] != self.EMBEDDING_DIM: + status_info["status"] = "degraded" + status_info["api_error"] = f"Dimension incorrecte: {test_emb.shape[0]}" + except Exception as e: + status_info["status"] = "unhealthy" + status_info["api_error"] = str(e) + + # Test cache Redis + if self.use_cache and self.redis_client: + try: + self.redis_client.ping() + status_info["cache_status"] = "connected" + except Exception as e: + status_info["cache_status"] = "disconnected" + status_info["cache_error"] = str(e) + else: + status_info["cache_status"] = "disabled" + + return status_info + + # ==================== MÉTHODES PRIVÉES ==================== + + def _validate_text(self, text: str) -> None: + """ + Valide le texte d'entrée. + + Args: + text: Texte à valider + + Raises: + ValueError: Si texte vide ou trop long + """ + if not text or not text.strip(): + raise ValueError("Le texte ne peut pas être vide") + + if len(text) > self.MAX_TEXT_LENGTH: + raise ValueError( + f"Texte trop long: {len(text)} chars > {self.MAX_TEXT_LENGTH} max" + ) + + def _preprocess_text(self, text: str) -> str: + """ + Prétraite le texte pour normalisation cache. + + Args: + text: Texte brut + + Returns: + Texte normalisé (strip whitespace) + """ + return text.strip() + + def _get_cache_key(self, text: str) -> str: + """ + Génère la clé de cache pour un texte. + + Utilise MD5 hash du texte normalisé pour: + - Clés compactes (32 chars vs texte complet) + - Déterminisme (même texte = même clé) + - Risque collision négligeable (<1M textes) + + Args: + text: Texte à hasher + + Returns: + Clé cache format "embedding:gemini:{md5_hash}" + """ + normalized = self._preprocess_text(text).lower() + text_hash = hashlib.md5(normalized.encode('utf-8')).hexdigest() + return f"embedding:gemini:{text_hash}" + + def _get_from_cache(self, cache_key: str) -> Optional[np.ndarray]: + """ + Récupère un embedding du cache Redis. + + Args: + cache_key: Clé de cache + + Returns: + Embedding si trouvé, None sinon + """ + try: + cached_data = self.redis_client.get(cache_key) + if cached_data: + # Désérialisation JSON -> numpy array + embedding_list = json.loads(cached_data) + return np.array(embedding_list, dtype=np.float32) + except Exception as e: + logger.warning(f"⚠️ Erreur lecture cache {cache_key[:20]}: {e}") + + return None + + def _store_in_cache( + self, + cache_key: str, + embedding: np.ndarray + ) -> None: + """ + Stocke un embedding dans le cache Redis. + + Args: + cache_key: Clé de cache + embedding: Embedding à stocker + """ + try: + # Sérialisation numpy -> JSON + embedding_list = embedding.tolist() + cached_data = json.dumps(embedding_list) + + # Stockage avec TTL + self.redis_client.setex( + cache_key, + self.CACHE_TTL_SECONDS, + cached_data + ) + except Exception as e: + logger.warning(f"⚠️ Erreur écriture cache {cache_key[:20]}: {e}") diff --git a/app/services/file_upload_service.py b/app/services/file_upload_service.py new file mode 100644 index 0000000000000000000000000000000000000000..bd0a623c4e068888ebd427a19a71d431fb3b2b0a --- /dev/null +++ b/app/services/file_upload_service.py @@ -0,0 +1,665 @@ +""" +Service de téléchargement sécurisé de fichiers pour documents juridiques. + +Ce service gère: +- Validation format/taille (PDF, DOCX, max 50 MB) +- Scan antivirus (ClamAV avec fallback mock) +- Stockage temporaire sécurisé +- Extraction métadonnées (titre, auteur, pages, etc.) + +Objectif: Upload sécurisé avec validation complète en <2s. + +Usage: + service = FileUploadService(storage_path="/app/data/uploads") + result = await service.upload_file(file) + # {'file_id': 'uuid', 'validation': {...}, 'scan_result': {...}, ...} +""" + +import logging +import time +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Dict, Optional + +from fastapi import UploadFile + +from app.schemas.file_upload import ( + FileMetadata, + FileUploadResult, + ScanResult, + ValidationResult, +) +from app.utils.file_utils import ( + detect_file_type, + ensure_storage_directory, + generate_unique_filename, + get_file_hash, + get_file_size_mb, + get_mime_type, + is_file_size_valid, + is_valid_docx, + is_valid_pdf, +) + +logger = logging.getLogger(__name__) + + +class FileUploadError(Exception): + """Exception levée lors d'erreurs d'upload.""" + + pass + + +class FileUploadService: + """ + Service de téléchargement sécurisé de fichiers. + + Fonctionnalités: + - Validation stricte (format, taille, structure) + - Scan antivirus (ClamAV ou mock) + - Stockage temporaire avec TTL + - Extraction métadonnées complètes + + Attributes: + storage_path: Chemin du répertoire de stockage + max_size_mb: Taille maximale autorisée (MB) + allowed_formats: Formats autorisés + clamav_enabled: Si ClamAV est activé + cleanup_hours: Durée de vie des fichiers (heures) + """ + + def __init__( + self, + storage_path: str = "./data/uploads", + max_size_mb: int = 250, + allowed_formats: tuple = ("pdf", "docx"), + clamav_enabled: bool = False, + cleanup_hours: int = 24, + ): + """ + Initialise le service d'upload. + + Args: + storage_path: Chemin du répertoire de stockage + max_size_mb: Taille maximale en MB (défaut: 50) + allowed_formats: Formats autorisés (défaut: pdf, docx) + clamav_enabled: Activer ClamAV (défaut: False, utilise mock) + cleanup_hours: TTL des fichiers en heures (défaut: 24) + + Raises: + FileUploadError: Si le répertoire de stockage ne peut être créé + """ + logger.info("🚀 Initialisation du FileUploadService...") + + self.storage_path = Path(storage_path) + self.max_size_mb = max_size_mb + self.allowed_formats = allowed_formats + self.clamav_enabled = clamav_enabled + self.cleanup_hours = cleanup_hours + + # Créer le répertoire de stockage + try: + ensure_storage_directory(self.storage_path) + logger.info(f" ✅ Répertoire de stockage: {self.storage_path}") + except Exception as e: + error_msg = f"Échec de création du répertoire de stockage: {e}" + logger.error(f"❌ {error_msg}") + raise FileUploadError(error_msg) from e + + # Initialiser le scanner antivirus + if self.clamav_enabled: + try: + import clamd + + self.clamav_client = clamd.ClamdUnixSocket() + self.clamav_client.ping() + logger.info(" ✅ ClamAV connecté") + except Exception as e: + logger.warning(f" ⚠️ ClamAV non disponible, utilisation du mock: {e}") + self.clamav_enabled = False + self.clamav_client = None + else: + self.clamav_client = None + logger.info(" ℹ️ Mode mock pour antivirus (développement)") + + logger.info( + f"✅ FileUploadService initialisé " + f"(max: {self.max_size_mb}MB, formats: {', '.join(self.allowed_formats)})" + ) + + async def upload_file(self, file: UploadFile) -> FileUploadResult: + """ + Upload et traite un fichier complet. + + Workflow: + 1. Sauvegarde temporaire + 2. Validation (format, taille, structure) + 3. Scan antivirus + 4. Extraction métadonnées + 5. Stockage permanent + + Args: + file: Fichier uploadé (FastAPI UploadFile) + + Returns: + FileUploadResult avec toutes les informations + + Raises: + FileUploadError: Si validation ou scan échoue + + Example: + >>> result = await service.upload_file(file) + >>> result.file_id + '550e8400-e29b-41d4-a716-446655440000' + """ + assert file is not None, "File must not be None" + assert hasattr(file, "filename"), "File must have a filename attribute" + + start_time = time.time() + logger.info(f"📤 Upload de fichier: {file.filename}") + + # Générer un ID unique + file_id = generate_unique_filename(file.filename or "unknown") + temp_path = self.storage_path / f"temp_{file_id}" + + try: + # 1. Sauvegarder temporairement + await self._save_temp_file(file, temp_path) + logger.debug(f" ✅ Fichier temporaire sauvegardé: {temp_path}") + + # 2. Valider le fichier + validation = await self.validate_file(temp_path, file.filename or "unknown") + if not validation.is_valid: + raise FileUploadError(f"Validation échouée: {', '.join(validation.errors)}") + logger.debug(f" ✅ Validation réussie: {validation.format}") + + # 3. Scanner antivirus + scan_result = await self.scan_virus(temp_path) + if not scan_result.is_clean: + raise FileUploadError( + f"Fichier infecté détecté: {scan_result.threat_found}" + ) + logger.debug(f" ✅ Scan antivirus: clean ({scan_result.scanner})") + + # 4. Extraire métadonnées + metadata = await self.extract_metadata(temp_path, validation.format) + logger.debug(f" ✅ Métadonnées extraites: {metadata.title or 'N/A'}") + + # 5. Déplacer vers stockage permanent + final_path = self.storage_path / file_id + temp_path.rename(final_path) + logger.debug(f" ✅ Fichier déplacé: {final_path}") + + # Calculer temps de traitement + processing_time = int((time.time() - start_time) * 1000) + logger.info( + f"✅ Upload terminé: {file.filename} " + f"({metadata.file_size / 1024 / 1024:.2f} MB, {processing_time}ms)" + ) + + # Construire le résultat + return FileUploadResult( + file_id=file_id.split(".")[0], # UUID sans extension + filename=file.filename or "unknown", + file_size=metadata.file_size, + mime_type=validation.mime_type, + file_hash=metadata.file_hash, + metadata=metadata, + validation=validation, + scan_result=scan_result, + storage_path=str(final_path.relative_to(self.storage_path.parent)), + uploaded_at=datetime.utcnow(), + expires_at=datetime.utcnow() + timedelta(hours=self.cleanup_hours), + ) + + except FileUploadError: + # Nettoyer le fichier temporaire en cas d'erreur + if temp_path.exists(): + temp_path.unlink() + raise + except Exception as e: + # Nettoyer et relancer + if temp_path.exists(): + temp_path.unlink() + error_msg = f"Erreur lors de l'upload: {str(e)}" + logger.error(f"❌ {error_msg}") + raise FileUploadError(error_msg) from e + + async def _save_temp_file(self, file: UploadFile, temp_path: Path) -> None: + """ + Sauvegarde un fichier uploadé temporairement. + + Args: + file: Fichier uploadé + temp_path: Chemin temporaire + + Raises: + FileUploadError: Si sauvegarde échoue + """ + try: + content = await file.read() + with open(temp_path, "wb") as f: + f.write(content) + except Exception as e: + raise FileUploadError(f"Échec de sauvegarde temporaire: {e}") from e + + async def validate_file( + self, file_path: Path, original_filename: str + ) -> ValidationResult: + """ + Valide un fichier uploadé. + + Validations: + - Taille (max 50 MB) + - Format (PDF ou DOCX via magic bytes) + - Structure (intégrité du fichier) + + Args: + file_path: Chemin du fichier + original_filename: Nom original du fichier + + Returns: + ValidationResult avec détails + + Example: + >>> result = await service.validate_file(path, "doc.pdf") + >>> result.is_valid + True + """ + errors = [] + warnings = [] + + # 1. Vérifier la taille + is_size_valid, size_error = is_file_size_valid(file_path, self.max_size_mb) + if not is_size_valid: + errors.append(size_error) + + # 2. Détecter le type de fichier + detected_type = detect_file_type(file_path) + if detected_type is None: + errors.append("Format de fichier non reconnu") + return ValidationResult( + is_valid=False, + format=None, + mime_type="application/octet-stream", + errors=errors, + warnings=warnings, + ) + + # 3. Vérifier que le format est autorisé + if detected_type not in self.allowed_formats: + errors.append( + f"Format '{detected_type}' non autorisé. " + f"Formats acceptés: {', '.join(self.allowed_formats)}" + ) + + # 4. Valider la structure selon le type + if detected_type == "pdf": + is_valid, error = is_valid_pdf(file_path) + if not is_valid: + errors.append(f"PDF invalide: {error}") + elif detected_type == "docx": + is_valid, error = is_valid_docx(file_path) + if not is_valid: + errors.append(f"DOCX invalide: {error}") + + # 5. Vérifier cohérence extension/type + file_extension = Path(original_filename).suffix.lower().lstrip(".") + if file_extension != detected_type: + warnings.append( + f"Extension '{file_extension}' ne correspond pas au type détecté '{detected_type}'" + ) + + # Résultat final + is_valid = len(errors) == 0 + mime_type = get_mime_type(detected_type) if detected_type else "application/octet-stream" + + return ValidationResult( + is_valid=is_valid, + format=detected_type, + mime_type=mime_type, + errors=errors, + warnings=warnings, + ) + + async def scan_virus(self, file_path: Path) -> ScanResult: + """ + Scanne un fichier avec antivirus. + + Utilise ClamAV si disponible, sinon mock pour développement. + + Args: + file_path: Chemin du fichier + + Returns: + ScanResult avec résultat du scan + + Example: + >>> result = await service.scan_virus(path) + >>> result.is_clean + True + """ + start_time = time.time() + + if self.clamav_enabled and self.clamav_client: + # Scan ClamAV réel + try: + scan_result = self.clamav_client.scan(str(file_path)) + scan_time_ms = int((time.time() - start_time) * 1000) + + if scan_result is None: + # Fichier clean + return ScanResult( + is_clean=True, + scanner="clamav", + scan_time_ms=scan_time_ms, + threat_found=None, + scanner_version=self.clamav_client.version(), + ) + else: + # Menace détectée + threat = list(scan_result.values())[0][1] if scan_result else "Unknown" + return ScanResult( + is_clean=False, + scanner="clamav", + scan_time_ms=scan_time_ms, + threat_found=threat, + scanner_version=self.clamav_client.version(), + ) + except Exception as e: + logger.error(f"Erreur ClamAV scan: {e}") + # Fallback sur mock en cas d'erreur + pass + + # Mock scanner (développement) + scan_time_ms = int((time.time() - start_time) * 1000) + + # Simuler un scan rapide + if scan_time_ms < 50: + scan_time_ms = 50 + (hash(str(file_path)) % 100) + + return ScanResult( + is_clean=True, + scanner="mock", + scan_time_ms=scan_time_ms, + threat_found=None, + scanner_version="1.0.0-dev", + ) + + async def extract_metadata(self, file_path: Path, file_format: str) -> FileMetadata: + """ + Extrait les métadonnées d'un fichier. + + Métadonnées extraites: + - PDF: titre, auteur, date création, nombre de pages + - DOCX: titre, auteur, date création, nombre de mots + - Tous: taille, hash SHA-256 + + Args: + file_path: Chemin du fichier + file_format: Format du fichier ('pdf' ou 'docx') + + Returns: + FileMetadata avec métadonnées extraites + + Example: + >>> metadata = await service.extract_metadata(path, "pdf") + >>> metadata.page_count + 45 + """ + # Métadonnées communes + file_size = file_path.stat().st_size + file_hash = get_file_hash(file_path) + + # Métadonnées spécifiques au format + if file_format == "pdf": + return await self._extract_pdf_metadata(file_path, file_size, file_hash) + elif file_format == "docx": + return await self._extract_docx_metadata(file_path, file_size, file_hash) + else: + # Format inconnu, métadonnées minimales + return FileMetadata( + title=None, + author=None, + creation_date=None, + page_count=None, + word_count=None, + file_size=file_size, + file_hash=file_hash, + ) + + async def _extract_pdf_metadata( + self, file_path: Path, file_size: int, file_hash: str + ) -> FileMetadata: + """Extrait métadonnées PDF.""" + try: + from pypdf import PdfReader + + reader = PdfReader(file_path) + metadata = reader.metadata + + # Extraire les champs + title = metadata.get("/Title") if metadata else None + author = metadata.get("/Author") if metadata else None + creation_date_str = metadata.get("/CreationDate") if metadata else None + + # Parser la date PDF (format: D:YYYYMMDDHHmmSS) + creation_date = None + if creation_date_str: + try: + # Format PDF: D:20240115103000 + date_str = creation_date_str.replace("D:", "")[:14] + creation_date = datetime.strptime(date_str, "%Y%m%d%H%M%S") + except Exception: + pass + + page_count = len(reader.pages) + + return FileMetadata( + title=title, + author=author, + creation_date=creation_date, + page_count=page_count, + word_count=None, + file_size=file_size, + file_hash=file_hash, + ) + + except Exception as e: + logger.warning(f"Échec extraction métadonnées PDF: {e}") + return FileMetadata( + title=None, + author=None, + creation_date=None, + page_count=None, + word_count=None, + file_size=file_size, + file_hash=file_hash, + ) + + async def _extract_docx_metadata( + self, file_path: Path, file_size: int, file_hash: str + ) -> FileMetadata: + """Extrait métadonnées DOCX.""" + try: + from docx import Document + + doc = Document(file_path) + core_props = doc.core_properties + + # Extraire les champs + title = core_props.title + author = core_props.author + creation_date = core_props.created + + # Compter les mots + word_count = sum(len(paragraph.text.split()) for paragraph in doc.paragraphs) + + return FileMetadata( + title=title, + author=author, + creation_date=creation_date, + page_count=None, + word_count=word_count, + file_size=file_size, + file_hash=file_hash, + ) + + except Exception as e: + logger.warning(f"Échec extraction métadonnées DOCX: {e}") + return FileMetadata( + title=None, + author=None, + creation_date=None, + page_count=None, + word_count=None, + file_size=file_size, + file_hash=file_hash, + ) + + async def cleanup_old_files(self, max_age_hours: Optional[int] = None) -> Dict[str, Any]: + """ + Nettoie les fichiers expirés. + + Args: + max_age_hours: Âge maximum en heures (défaut: self.cleanup_hours) + + Returns: + Statistiques de nettoyage + + Example: + >>> stats = await service.cleanup_old_files() + >>> stats['deleted_count'] + 5 + """ + max_age = max_age_hours or self.cleanup_hours + cutoff_time = datetime.utcnow() - timedelta(hours=max_age) + + deleted_count = 0 + deleted_size = 0 + errors = [] + + logger.info(f"🧹 Nettoyage des fichiers > {max_age}h...") + + try: + for file_path in self.storage_path.glob("*"): + if not file_path.is_file(): + continue + + # Vérifier l'âge du fichier + file_mtime = datetime.fromtimestamp(file_path.stat().st_mtime) + if file_mtime < cutoff_time: + try: + file_size = file_path.stat().st_size + file_path.unlink() + deleted_count += 1 + deleted_size += file_size + logger.debug(f" 🗑️ Supprimé: {file_path.name}") + except Exception as e: + errors.append(f"Échec suppression {file_path.name}: {e}") + + logger.info( + f"✅ Nettoyage terminé: {deleted_count} fichiers supprimés " + f"({deleted_size / 1024 / 1024:.2f} MB libérés)" + ) + + return { + "deleted_count": deleted_count, + "deleted_size_mb": round(deleted_size / 1024 / 1024, 2), + "errors": errors, + "cutoff_time": cutoff_time.isoformat(), + } + + except Exception as e: + logger.error(f"❌ Erreur lors du nettoyage: {e}") + return { + "deleted_count": deleted_count, + "deleted_size_mb": round(deleted_size / 1024 / 1024, 2), + "errors": errors + [str(e)], + "cutoff_time": cutoff_time.isoformat(), + } + + def health_check(self) -> Dict[str, Any]: + """ + Vérifie l'état de santé du service. + + Returns: + Dictionnaire avec statut de chaque composant + + Example: + >>> health = service.health_check() + >>> health['status'] + 'healthy' + """ + status = { + "service": "FileUploadService", + "status": "healthy", + "storage": {}, + "scanner": {}, + "timestamp": datetime.utcnow().isoformat(), + } + + # Vérifier le stockage + try: + storage_exists = self.storage_path.exists() + storage_writable = storage_exists and self.storage_path.is_dir() + + # Compter les fichiers + total_files = len(list(self.storage_path.glob("*"))) if storage_exists else 0 + total_size = ( + sum(f.stat().st_size for f in self.storage_path.glob("*") if f.is_file()) + if storage_exists + else 0 + ) + + status["storage"] = { + "path": str(self.storage_path), + "exists": storage_exists, + "writable": storage_writable, + "total_files": total_files, + "total_size_mb": round(total_size / 1024 / 1024, 2), + } + + if not storage_exists or not storage_writable: + status["status"] = "degraded" + + except Exception as e: + status["storage"] = {"error": str(e)} + status["status"] = "unhealthy" + + # Vérifier le scanner + try: + if self.clamav_enabled and self.clamav_client: + self.clamav_client.ping() + status["scanner"] = { + "type": "clamav", + "available": True, + "version": self.clamav_client.version(), + } + else: + status["scanner"] = { + "type": "mock", + "available": True, + "version": "1.0.0-dev", + } + except Exception as e: + status["scanner"] = {"type": "clamav", "available": False, "error": str(e)} + status["status"] = "degraded" + + return status + + +# Singleton instance +_upload_service_instance = None + +def get_upload_service() -> FileUploadService: + """Get or create the FileUploadService singleton instance.""" + global _upload_service_instance + if _upload_service_instance is None: + _upload_service_instance = FileUploadService( + storage_path="./data/uploads", + max_size_mb=50, + allowed_formats=("pdf", "docx"), + clamav_enabled=False, + ) + return _upload_service_instance diff --git a/app/services/gemini_service.py b/app/services/gemini_service.py new file mode 100644 index 0000000000000000000000000000000000000000..9687cce0a8867deff73e086eb99f97562c28800e --- /dev/null +++ b/app/services/gemini_service.py @@ -0,0 +1,223 @@ +""" +GeminiService - Client for Google Gemini API (LLM for RAG). + +Uses the new google-genai SDK (replaces deprecated google-generativeai). + +Provides async methods for: +- generate(): Single response generation +- generate_stream(): Streaming response generation +- health_check(): API connectivity check +""" + +import logging +from functools import lru_cache +from typing import AsyncIterator, Dict, Optional + +from google import genai +from google.genai import types + +from app.core.config import settings + +logger = logging.getLogger(__name__) + + +class GeminiServiceError(Exception): + """Base exception for Gemini service errors.""" + pass + + +class GeminiService: + """ + Client for Google Gemini API. + + Uses the new google-genai SDK for LLM operations. + """ + + # Default generation config + DEFAULT_TEMPERATURE = 0.7 + DEFAULT_MAX_TOKENS = 1000 + + # System instruction for legal assistant + SYSTEM_INSTRUCTION = """You are an expert legal assistant specializing in Cameroonian law (JuriX). + +CORE RULES: +1. Answer ONLY based on legal documents provided in the context +2. ALWAYS cite articles with exact references (e.g., "Article 5 of the Constitution") +3. If information is not in the context, say: "Je ne trouve pas cette information dans les documents disponibles." +4. Adapt language style to user persona (citizen=simple, lawyer=technical, student=educational) +5. Respond in French by default, or English if user writes in English + +RESPONSE FORMAT: +- Be concise but complete (max 300 words unless complex topic) +- Use clear paragraphs +- End with "Sources:" listing cited documents + +FORBIDDEN: +- Never invent laws, articles, or legal interpretations +- No personalized legal advice +- No speculation on court decisions""" + + def __init__( + self, + api_key: Optional[str] = None, + model_name: Optional[str] = None + ): + """ + Initialize Gemini service. + + Args: + api_key: Gemini API key (defaults to settings.GEMINI_API_KEY) + model_name: Model to use (defaults to settings.GEMINI_MODEL) + """ + self.api_key = api_key or settings.GEMINI_API_KEY + self.model_name = model_name or settings.GEMINI_MODEL + + if not self.api_key: + raise GeminiServiceError( + "GEMINI_API_KEY not configured. Set it in .env" + ) + + # Initialize client with API key + self.client = genai.Client(api_key=self.api_key) + + logger.info(f"✅ GeminiService initialized: model={self.model_name}") + + async def generate( + self, + prompt: str, + system: Optional[str] = None, + temperature: float = DEFAULT_TEMPERATURE, + max_tokens: int = DEFAULT_MAX_TOKENS, + **kwargs + ) -> Dict: + """ + Generate a response from Gemini. + + Args: + prompt: User prompt/question with context + system: Optional system prompt override + temperature: Generation temperature (0.0-1.0) + max_tokens: Maximum tokens to generate + + Returns: + Dict with 'response' key containing generated text + """ + try: + logger.debug(f"🤖 Generating response (temp={temperature}, max={max_tokens})") + + # Use provided system or default + system_instruction = system or self.SYSTEM_INSTRUCTION + + # Create generation config + config = types.GenerateContentConfig( + temperature=temperature, + max_output_tokens=max_tokens, + system_instruction=system_instruction + ) + + # Generate content (sync call wrapped for async compatibility) + response = self.client.models.generate_content( + model=self.model_name, + contents=prompt, + config=config + ) + + # Extract text from response + if response.text: + logger.info(f"✅ Generated {len(response.text)} chars") + return {"response": response.text} + else: + logger.warning("⚠️ Empty response from Gemini") + return {"response": "Je ne peux pas répondre à cette question."} + + except Exception as e: + logger.error(f"❌ Gemini generation error: {e}") + raise GeminiServiceError(f"Erreur de génération: {str(e)}") + + async def generate_stream( + self, + prompt: str, + system: Optional[str] = None, + temperature: float = DEFAULT_TEMPERATURE, + max_tokens: int = DEFAULT_MAX_TOKENS, + **kwargs + ) -> AsyncIterator[str]: + """ + Stream a response from Gemini. + + Yields text chunks as they are generated. + """ + try: + logger.debug(f"🤖 Streaming response (temp={temperature}, max={max_tokens})") + + system_instruction = system or self.SYSTEM_INSTRUCTION + + config = types.GenerateContentConfig( + temperature=temperature, + max_output_tokens=max_tokens, + system_instruction=system_instruction + ) + + # Stream content + response_stream = self.client.models.generate_content_stream( + model=self.model_name, + contents=prompt, + config=config + ) + + for chunk in response_stream: + if chunk.text: + yield chunk.text + + logger.info("✅ Streaming complete") + + except Exception as e: + logger.error(f"❌ Gemini streaming error: {e}") + raise GeminiServiceError(f"Erreur de streaming: {str(e)}") + + async def health_check(self) -> Dict: + """ + Check if Gemini API is accessible. + + Returns: + Dict with 'status' key ('healthy' or 'unhealthy') + """ + try: + # Simple test generation + response = self.client.models.generate_content( + model=self.model_name, + contents="Say 'OK' in one word.", + config=types.GenerateContentConfig(max_output_tokens=10) + ) + + if response.text: + return {"status": "healthy", "model": self.model_name} + else: + return {"status": "unhealthy", "reason": "Empty response"} + + except Exception as e: + logger.error(f"❌ Health check failed: {e}") + return {"status": "unhealthy", "reason": str(e)} + + +# Singleton instance +_gemini_service: Optional[GeminiService] = None + + +@lru_cache() +def get_gemini_service() -> GeminiService: + """ + Singleton factory for GeminiService. + """ + global _gemini_service + if _gemini_service is None: + _gemini_service = GeminiService() + return _gemini_service + + +def clear_gemini_service_cache(): + """Clear the singleton instance (for testing).""" + global _gemini_service + _gemini_service = None + get_gemini_service.cache_clear() + logger.info("🗑️ GeminiService cache cleared") diff --git a/app/services/language_detector.py b/app/services/language_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..e1bc9db679ead6c3ef172f27fb002c648ddac853 --- /dev/null +++ b/app/services/language_detector.py @@ -0,0 +1,493 @@ +""" +Service de détection automatique de langue pour documents juridiques. + +Ce service utilise 3 méthodes de détection en parallèle: +- langdetect: Détection rapide basée sur n-grams +- spaCy: Détection NLP basée sur reconnaissance d'entités +- fastText: Détection ML basée sur embeddings + +Vote majoritaire (2/3 minimum) pour robustesse maximale. +Objectif: 99% précision, <1s temps de réponse. + +Usage: + detector = LanguageDetector() + result = detector.detect("Article 1. La présente loi...") + # {'language': 'fr', 'confidence': 0.98, 'consensus': True, ...} +""" + +import logging +import re +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor, TimeoutError, as_completed +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import fasttext +import langdetect +import spacy +from langdetect import DetectorFactory +from spacy.language import Language + +# Configuration langdetect pour reproductibilité +DetectorFactory.seed = 0 + +# Configuration logging +logger = logging.getLogger(__name__) + + +class LanguageDetectionError(Exception): + """Exception levée lors d'erreurs de détection.""" + + pass + + +class LanguageDetector: + """ + Service de détection automatique de langue avec triple ensemble. + + Utilise 3 méthodes en parallèle pour une précision maximale: + - langdetect (n-grams, rapide) + - spaCy (NLP, entités) + - fastText (ML, embeddings) + + Vote majoritaire (2/3) pour décision finale. + Précision cible: 99% + Temps cible: <1 seconde + + Attributes: + nlp_fr: Modèle spaCy français + nlp_en: Modèle spaCy anglais + fasttext_model: Modèle fastText de détection de langue + """ + + def __init__(self, fasttext_model_path: Optional[str] = None): + """ + Initialise le détecteur avec chargement eager des modèles. + + Args: + fasttext_model_path: Chemin vers le modèle fastText (optionnel) + Par défaut: backend/models/fasttext/lid.176.bin + + Raises: + LanguageDetectionError: Si les modèles ne peuvent pas être chargés + """ + logger.info("🚀 Initialisation du LanguageDetector...") + + try: + # Charger modèles spaCy + logger.info(" → Chargement des modèles spaCy...") + self.nlp_fr: Language = spacy.load("fr_core_news_sm") + self.nlp_en: Language = spacy.load("en_core_web_sm") + logger.info(" ✅ Modèles spaCy chargés") + + # Charger modèle fastText + logger.info(" → Chargement du modèle fastText...") + if fasttext_model_path is None: + # Chemin par défaut + backend_dir = Path(__file__).parent.parent.parent + fasttext_model_path = str(backend_dir / "models" / "fasttext" / "lid.176.bin") + + # Suppress fastText warnings + fasttext.FastText.eprint = lambda x: None + self.fasttext_model = fasttext.load_model(fasttext_model_path) + logger.info(" ✅ Modèle fastText chargé") + + logger.info("✅ LanguageDetector initialisé avec succès (3 modèles prêts)") + + except Exception as e: + error_msg = f"Échec du chargement des modèles: {e}" + logger.error(f"❌ {error_msg}") + raise LanguageDetectionError(error_msg) from e + + def detect( + self, text: str, min_confidence: float = 0.80, min_length: int = 50 + ) -> Dict[str, Any]: + """ + Détecte la langue d'un texte avec vote majoritaire. + + Exécute les 3 méthodes en parallèle pour performance optimale. + Applique un vote majoritaire sur les résultats. + + Args: + text: Texte à analyser (recommandé: >100 caractères) + min_confidence: Seuil de confiance minimum (0-1) + min_length: Longueur minimale requise en caractères + + Returns: + Dictionnaire contenant: + - language (str): Code langue détectée ('fr' | 'en') + - confidence (float): Score de confiance (0-1) + - method_votes (Dict[str, str]): Votes de chaque méthode + - consensus (bool): True si 2+ méthodes d'accord + - processing_time_ms (int): Temps de traitement en ms + - text_length (int): Longueur du texte analysé + + Raises: + ValueError: Si texte trop court ou vide + LanguageDetectionError: Si toutes les méthodes échouent + + Example: + >>> detector = LanguageDetector() + >>> result = detector.detect("Article 1. La présente loi...") + >>> print(result['language']) + 'fr' + >>> print(result['confidence']) + 0.98 + """ + assert isinstance(min_confidence, float) and 0.0 <= min_confidence <= 1.0, "min_confidence must be between 0.0 and 1.0" + assert isinstance(min_length, int) and min_length > 0, "min_length must be a positive integer" + + start_time = time.time() + + # Validation input + if not text or not text.strip(): + raise ValueError("Le texte ne peut pas être vide") + + text = text.strip() + + if len(text) < min_length: + raise ValueError(f"Texte trop court: {len(text)} caractères (minimum: {min_length})") + + # Prétraitement + processed_text = self._preprocess_text(text) + + # Log pour debug + logger.debug(f"Détection de langue pour texte de {len(processed_text)} caractères") + + # Exécution parallèle des 3 méthodes + votes: List[Tuple[str, str, float]] = [] # (method_name, language, confidence) + + with ThreadPoolExecutor(max_workers=3) as executor: + # Soumettre les 3 tâches + future_to_method = { + executor.submit(self._detect_langdetect, processed_text): "langdetect", + executor.submit(self._detect_spacy, processed_text): "spacy", + executor.submit(self._detect_fasttext, processed_text): "fasttext", + } + + # Collecter les résultats avec timeout + for future in as_completed(future_to_method.keys(), timeout=2.0): + method_name = future_to_method[future] + try: + language, confidence = future.result(timeout=1.0) + votes.append((method_name, language, confidence)) + logger.debug(f" {method_name}: {language} ({confidence:.2%})") + except TimeoutError: + logger.warning(f" {method_name}: timeout (>1s)") + except Exception as e: + logger.warning(f" {method_name}: erreur ({e})") + + # Vérifier qu'au moins une méthode a réussi + if not votes: + raise LanguageDetectionError("Toutes les méthodes de détection ont échoué") + + # Appliquer le vote majoritaire + detected_language, final_confidence, consensus = self._vote_majority(votes) + + # Vérifier seuil de confiance + if final_confidence < min_confidence: + logger.warning(f"Confiance {final_confidence:.2%} < seuil {min_confidence:.2%}") + + # Calculer temps de traitement + processing_time_ms = int((time.time() - start_time) * 1000) + + # Construire résultat + result = { + "language": detected_language, + "confidence": round(final_confidence, 4), + "method_votes": {method: lang for method, lang, _ in votes}, + "consensus": consensus, + "processing_time_ms": processing_time_ms, + "text_length": len(text), + } + + logger.info( + f"Détection: {detected_language} " + f"({final_confidence:.2%}, {processing_time_ms}ms, " + f"consensus={consensus})" + ) + + return result + + def _preprocess_text(self, text: str) -> str: + """ + Prétraite le texte pour améliorer la détection. + + Opérations: + - Normalisation des espaces + - Suppression de la numérotation des articles + - Préservation des accents français (critique!) + - Préservation des symboles légaux + - Sampling à 5000 caractères pour performance + + Args: + text: Texte brut + + Returns: + Texte nettoyé + """ + # Normaliser les espaces multiples + text = re.sub(r"\s+", " ", text) + + # Supprimer la numérotation des articles (bruit) + # Ex: "Art. 1", "Article 2", "Section 3" + text = re.sub(r"\b(Art\.|Article|Section)\s+\d+\.?\s*", "", text) + + # IMPORTANT: Préserver les accents (é, è, à, ç, etc.) + # Ne PAS normaliser/supprimer les accents - critiques pour FR! + + # Sampling pour performance (5000 premiers caractères suffisent) + if len(text) > 5000: + text = text[:5000] + + return text.strip() + + def _detect_langdetect(self, text: str) -> Tuple[str, float]: + """ + Détection via langdetect (rapide, n-grams). + + Args: + text: Texte à analyser + + Returns: + Tuple (language_code, confidence) + + Raises: + Exception: Si détection échoue + """ + try: + # langdetect retourne liste de (lang, prob) + detections = langdetect.detect_langs(text) + + if not detections: + raise ValueError("Aucune langue détectée") + + # Prendre la langue avec probabilité maximale + top_detection = detections[0] + language = top_detection.lang + confidence = top_detection.prob + + # Normaliser: garder seulement 'fr' ou 'en' + # Si autre langue détectée, fallback sur fr + if language not in ["fr", "en"]: + logger.debug(f"langdetect: langue {language} → fallback fr") + language = "fr" + confidence *= 0.5 # Réduire confiance pour fallback + + return language, confidence + + except Exception as e: + raise Exception(f"langdetect error: {e}") from e + + def _detect_spacy(self, text: str) -> Tuple[str, float]: + """ + Détection via spaCy (NLP, reconnaissance d'entités). + + Stratégie: Tester avec les 2 modèles (FR et EN) et comparer + les scores basés sur le nombre d'entités reconnues. + + Args: + text: Texte à analyser + + Returns: + Tuple (language_code, confidence) + + Raises: + Exception: Si détection échoue + """ + try: + # Limiter la longueur pour performance spaCy + sample = text[:1000] + + # Analyser avec modèle FR + doc_fr = self.nlp_fr(sample) + score_fr = self._calculate_spacy_score(doc_fr) + + # Analyser avec modèle EN + doc_en = self.nlp_en(sample) + score_en = self._calculate_spacy_score(doc_en) + + # Déterminer langue gagnante + total_score = score_fr + score_en + + if total_score == 0: + # Aucune entité reconnue, fallback sur FR + return "fr", 0.5 + + if score_fr > score_en: + confidence = min(score_fr / total_score, 0.99) + return "fr", confidence + else: + confidence = min(score_en / total_score, 0.99) + return "en", confidence + + except Exception as e: + raise Exception(f"spaCy error: {e}") from e + + def _calculate_spacy_score(self, doc) -> float: + """ + Calcule un score de qualité pour un document spaCy. + + Basé sur: + - Nombre d'entités nommées reconnues + - Nombre de tokens reconnus + + Args: + doc: Document spaCy analysé + + Returns: + Score de qualité (float) + """ + if len(doc) == 0: + return 0.0 + + # Score basé sur entités (poids fort) + entity_score = len(doc.ents) * 2.0 + + # Score basé sur tokens (poids faible) + token_score = len([token for token in doc if not token.is_punct]) * 0.1 + + return entity_score + token_score + + def _detect_fasttext(self, text: str) -> Tuple[str, float]: + """ + Détection via fastText (ML, embeddings). + + Args: + text: Texte à analyser + + Returns: + Tuple (language_code, confidence) + + Raises: + Exception: Si détection échoue + """ + try: + # Remplacer les sauts de ligne par espaces + text_cleaned = text.replace("\n", " ") + + # Prédiction (k=1 pour top langue seulement) + predictions = self.fasttext_model.predict(text_cleaned, k=1) + + # Format retour: (('__label__fr',), array([0.98])) + label = predictions[0][0].replace("__label__", "") + confidence = float(predictions[1][0]) + + # Normaliser: garder seulement 'fr' ou 'en' + if label not in ["fr", "en"]: + logger.debug(f"fasttext: langue {label} → fallback fr") + label = "fr" + confidence *= 0.5 + + return label, confidence + + except Exception as e: + raise Exception(f"fastText error: {e}") from e + + def _vote_majority(self, votes: List[Tuple[str, str, float]]) -> Tuple[str, float, bool]: + """ + Applique un vote majoritaire sur les résultats des méthodes. + + Logique: + - 3/3 d'accord: consensus parfait + - 2/3 d'accord: consensus majoritaire + - 1/3 ou moins: pas de consensus, prendre le score le plus élevé + + Args: + votes: Liste de (method_name, language, confidence) + + Returns: + Tuple (detected_language, final_confidence, consensus) + """ + if not votes: + raise ValueError("Aucun vote disponible") + + # Compter les votes par langue + language_votes = Counter([lang for _, lang, _ in votes]) + + # Langue avec le plus de votes + most_common = language_votes.most_common(1)[0] + winning_language = most_common[0] + vote_count = most_common[1] + + # Déterminer consensus + total_votes = len(votes) + + # Cas spécial: Si seulement 2 votes et pas d'accord (1-1), utiliser langdetect comme tiebreaker + if total_votes == 2 and vote_count == 1: + # Chercher le vote de langdetect + langdetect_vote = next((lang for method, lang, _ in votes if method == "langdetect"), None) + if langdetect_vote: + winning_language = langdetect_vote + vote_count = 1 + # Considérer comme consensus si langdetect a haute confiance + langdetect_conf = next((conf for method, lang, conf in votes if method == "langdetect"), 0) + consensus = langdetect_conf >= 0.95 + else: + consensus = False + else: + consensus = vote_count >= 2 # Au moins 2/3 + + # Calculer confiance: moyenne des votes concordants avec bonus pour consensus + matching_confidences = [conf for _, lang, conf in votes if lang == winning_language] + + if matching_confidences: + # Moyenne de base + base_confidence = sum(matching_confidences) / len(matching_confidences) + + # Bonus si consensus (2+ méthodes d'accord) + if consensus and vote_count >= 2: + # Appliquer un boost de 5% quand il y a consensus entre 2+ méthodes + # Cela compense la perte d'une méthode (fasttext) + boost = 0.05 + final_confidence = min(base_confidence + boost, 0.99) + else: + final_confidence = base_confidence + else: + # Cas extrême: aucune confidence (ne devrait pas arriver) + final_confidence = 0.5 + + # Log pour monitoring + if not consensus: + logger.warning( + f"Pas de consensus: {vote_count}/{total_votes} votes pour {winning_language}" + ) + logger.warning(f"Détail votes: {votes}") + + return winning_language, final_confidence, consensus + + def health_check(self) -> Dict[str, Any]: + """ + Vérifie l'état de santé du service. + + Returns: + Dictionnaire avec statut de chaque composant + """ + status = {"service": "LanguageDetector", "status": "healthy", "models": {}} + + # Vérifier spaCy FR + try: + test_doc = self.nlp_fr("Test") + status["models"]["spacy_fr"] = "✅ OK" + except Exception as e: + status["models"]["spacy_fr"] = f"❌ Error: {e}" + status["status"] = "degraded" + + # Vérifier spaCy EN + try: + test_doc = self.nlp_en("Test") + status["models"]["spacy_en"] = "✅ OK" + except Exception as e: + status["models"]["spacy_en"] = f"❌ Error: {e}" + status["status"] = "degraded" + + # Vérifier fastText + try: + self.fasttext_model.predict("Test", k=1) + status["models"]["fasttext"] = "✅ OK" + except Exception as e: + status["models"]["fasttext"] = f"❌ Error: {e}" + status["status"] = "degraded" + + return status diff --git a/app/services/law_service.py b/app/services/law_service.py new file mode 100644 index 0000000000000000000000000000000000000000..cadd15703a6f309627a8fcd4523b8a2aa3c11ec7 --- /dev/null +++ b/app/services/law_service.py @@ -0,0 +1,932 @@ +""" +LawService - Service for managing legal documents (laws) with CRUD operations. + +Provides comprehensive law management with v2.1 features: +- Complete CRUD operations (create, read, update, delete, list, search) +- v2.1 language filtering and auto-detection +- v2.1 automatic category suggestions with confidence scores +- Integration with LanguageDetector and DocumentClassifier +- Advanced filtering and pagination +- Search functionality + +Author: JuriX Development Team +Date: 2026-01-10 +""" + +import logging +from datetime import datetime +from typing import Dict, List, Optional, Tuple + +from sqlalchemy import and_, extract, func, or_, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import joinedload, selectinload + +from app.core.dependencies import get_document_classifier, get_language_detector +from app.models.law import Article, Category, Law +from app.schemas.law import ( + CategoryCreate, + CategoryResponse, + CategoryStats, + LanguageStats, + LawCreate, + LawFilters, + LawListResponse, + LawResponse, + LawStats, + LawUpdate, + SearchResponse, + SearchResult, +) + +logger = logging.getLogger(__name__) + + +# ============================================================================ +# Custom Exceptions +# ============================================================================ + +class LawServiceError(Exception): + """Base exception for LawService operations.""" + pass + + +class LawNotFoundError(LawServiceError): + """Raised when a law is not found.""" + pass + + +class CategoryNotFoundError(LawServiceError): + """Raised when a category is not found.""" + pass + + +class DuplicateReferenceError(LawServiceError): + """Raised when trying to create/update a law with duplicate reference.""" + pass + + +# ============================================================================ +# LawService Implementation +# ============================================================================ + +class LawService: + """ + Service for managing legal documents (laws) with CRUD operations. + + This is a stateful service that requires a database session. + Instantiate per request via dependency injection. + + Features: + - Complete CRUD operations + - v2.1 language detection and filtering + - v2.1 category suggestions with confidence + - Advanced filtering and search + - Integration with ML services + + Usage: + async with get_db() as db: + service = LawService(db) + law = await service.create_law(law_data) + """ + + def __init__(self, db: AsyncSession): + """ + Initialize LawService with database session. + + Args: + db: AsyncSession for database operations + """ + self.db = db + self.language_detector = get_language_detector() + self.document_classifier = get_document_classifier() + logger.info("✅ LawService initialized") + + # ======================================================================== + # Law CRUD Operations + # ======================================================================== + + async def create_law(self, law_data: LawCreate) -> Law: + """ + Create a new law with automatic language and category detection. + + v2.1 Features: + - Auto-detects language if not provided + - Auto-suggests top 3 categories + - Stores confidence scores + + Args: + law_data: Law creation data + + Returns: + Created Law object with v2.1 fields populated + + Raises: + DuplicateReferenceError: If reference already exists + LawServiceError: If creation fails + """ + assert law_data is not None, "law_data must not be None" + assert hasattr(law_data, "reference") and law_data.reference, "law_data must have a reference" + + try: + logger.info(f"📝 Creating law: {law_data.reference}") + + # Validate unique reference + await self._validate_unique_reference(law_data.reference) + + # Auto-detect language (v2.1) + detected_lang, lang_confidence = await self._detect_language_and_confidence( + law_data.content + ) + + # Auto-suggest categories (v2.1) + suggested_cats, cat_confidence = await self._suggest_categories( + law_data.content + ) + + # Prepare law data + law_dict = law_data.model_dump(exclude_unset=True) + + # Use provided language or detected language + final_language = law_data.language if law_data.language else detected_lang + + # Update law_dict with v2.1 detection fields + law_dict.update({ + "language": final_language, + "detected_language": detected_lang, + "language_confidence": lang_confidence, + "suggested_categories": suggested_cats, + "category_confidence": cat_confidence, + }) + + # Create Law object + law = Law(**law_dict) + + self.db.add(law) + await self.db.commit() + await self.db.refresh(law) + + logger.info( + f"✅ Law created: {law.reference} (id={law.id}, " + f"lang={final_language}, confidence={lang_confidence:.2f})" + ) + + # Auto-index in Meilisearch (v2.1 SearchService integration) + try: + from app.services.search_service import SearchService + search_service = SearchService(self.db, use_cache=True) + await search_service.index_law(law) + logger.info(f"🔍 Auto-indexed law {law.id} in search engine") + except Exception as e: + logger.warning(f"⚠️ Failed to auto-index law {law.id}: {e}") + + return law + + except DuplicateReferenceError: + await self.db.rollback() + raise # Re-raise without wrapping + + except IntegrityError as e: + await self.db.rollback() + if "unique constraint" in str(e).lower(): + raise DuplicateReferenceError( + f"Law with reference '{law_data.reference}' already exists" + ) + raise LawServiceError(f"Database integrity error: {str(e)}") + + except Exception as e: + await self.db.rollback() + logger.error(f"❌ Error creating law: {str(e)}") + raise LawServiceError(f"Failed to create law: {str(e)}") + + async def get_law( + self, + law_id: int, + include_articles: bool = False, + include_category: bool = True + ) -> Law: + """ + Get a single law by ID with optional eager loading. + + Args: + law_id: Law ID + include_articles: Whether to include articles + include_category: Whether to include category + + Returns: + Law object + + Raises: + LawNotFoundError: If law not found + """ + try: + logger.debug(f"🔍 Fetching law id={law_id}") + + # Build query with eager loading + query = select(Law).where(Law.id == law_id) + + if include_category: + query = query.options(joinedload(Law.category)) + + if include_articles: + query = query.options(selectinload(Law.articles)) + + result = await self.db.execute(query) + law = result.scalar_one_or_none() + + if not law: + raise LawNotFoundError(f"Law with id={law_id} not found") + + logger.debug(f"✅ Law found: {law.reference}") + return law + + except LawNotFoundError: + raise + except Exception as e: + logger.error(f"❌ Error fetching law: {str(e)}") + raise LawServiceError(f"Failed to fetch law: {str(e)}") + + async def get_law_by_reference(self, reference: str) -> Optional[Law]: + """ + Get a law by its unique reference. + + Args: + reference: Law reference (e.g., "LOI-2024-001") + + Returns: + Law object or None if not found + """ + try: + query = select(Law).where(Law.reference == reference) + result = await self.db.execute(query) + return result.scalar_one_or_none() + except Exception as e: + logger.error(f"❌ Error fetching law by reference: {str(e)}") + raise LawServiceError(f"Failed to fetch law: {str(e)}") + + async def list_laws(self, filters: LawFilters) -> LawListResponse: + """ + List laws with filtering and pagination. + + v2.1 Features: + - Filter by language (fr/en) + - Filter by category, status, type + - Date range filtering + - Sorting and pagination + + Args: + filters: Filter and pagination parameters + + Returns: + LawListResponse with paginated results + """ + try: + logger.info(f"📋 Listing laws (page={filters.page}, filters={filters.model_dump(exclude_unset=True)})") + + # Build base query + query = select(Law).options(joinedload(Law.category)) + + # Apply filters + query = self._build_filters_query(query, filters) + + # Count total (before pagination) + count_query = select(func.count()).select_from(Law) + count_query = self._build_filters_query(count_query, filters) + total_result = await self.db.execute(count_query) + total = total_result.scalar_one() + + # Apply sorting + query = self._apply_sorting(query, filters) + + # Apply pagination + offset = (filters.page - 1) * filters.per_page + query = query.offset(offset).limit(filters.per_page) + + # Execute query + result = await self.db.execute(query) + laws = result.scalars().unique().all() + + # Calculate pages + pages = (total + filters.per_page - 1) // filters.per_page + + logger.info(f"✅ Found {total} laws ({len(laws)} on page {filters.page})") + + return LawListResponse( + items=[LawResponse.model_validate(law) for law in laws], + total=total, + page=filters.page, + per_page=filters.per_page, + pages=pages, + filters_applied=filters.model_dump(exclude_unset=True) + ) + + except Exception as e: + logger.error(f"❌ Error listing laws: {str(e)}") + raise LawServiceError(f"Failed to list laws: {str(e)}") + + async def update_law(self, law_id: int, law_data: LawUpdate) -> Law: + """ + Update an existing law with partial data. + + v2.1 Features: + - Re-detects language if content changed + - Re-suggests categories if content changed + + Args: + law_id: Law ID to update + law_data: Partial update data + + Returns: + Updated Law object + + Raises: + LawNotFoundError: If law not found + DuplicateReferenceError: If new reference conflicts + """ + assert isinstance(law_id, int) and law_id > 0, "law_id must be a positive integer" + assert law_data is not None, "law_data must not be None" + + try: + logger.info(f"✏️ Updating law id={law_id}") + + # Fetch existing law + law = await self.get_law(law_id, include_category=True) + + # Get update data + update_dict = law_data.model_dump(exclude_unset=True) + + # Check if reference changed + if "reference" in update_dict and update_dict["reference"] != law.reference: + await self._validate_unique_reference( + update_dict["reference"], + exclude_id=law_id + ) + + # Check if content changed (requires re-detection) + content_changed = "content" in update_dict + + if content_changed: + new_content = update_dict["content"] + + # Re-detect language + detected_lang, lang_confidence = await self._detect_language_and_confidence( + new_content + ) + + # Re-suggest categories + suggested_cats, cat_confidence = await self._suggest_categories( + new_content + ) + + # Update v2.1 fields + update_dict["detected_language"] = detected_lang + update_dict["language_confidence"] = lang_confidence + update_dict["suggested_categories"] = suggested_cats + update_dict["category_confidence"] = cat_confidence + + # Update language if not explicitly provided + if "language" not in update_dict: + update_dict["language"] = detected_lang + + logger.info( + f"🔄 Content changed, re-detected: lang={detected_lang}, " + f"confidence={lang_confidence:.2f}" + ) + + # Apply updates + for key, value in update_dict.items(): + setattr(law, key, value) + + # Update timestamp + law.updated_at = datetime.utcnow() + + await self.db.commit() + await self.db.refresh(law) + + logger.info(f"✅ Law updated: {law.reference}") + + # Update search index (v2.1 SearchService integration) + try: + from app.services.search_service import SearchService + search_service = SearchService(self.db, use_cache=True) + await search_service.update_law_index(law_id, law) + logger.info(f"🔍 Updated search index for law {law_id}") + except Exception as e: + logger.warning(f"⚠️ Failed to update search index for law {law_id}: {e}") + + return law + + except (LawNotFoundError, DuplicateReferenceError): + raise + except IntegrityError as e: + await self.db.rollback() + raise LawServiceError(f"Database integrity error: {str(e)}") + except Exception as e: + await self.db.rollback() + logger.error(f"❌ Error updating law: {str(e)}") + raise LawServiceError(f"Failed to update law: {str(e)}") + + async def delete_law(self, law_id: int) -> bool: + """ + Delete a law by ID. + + Note: Articles are cascade deleted automatically. + + Args: + law_id: Law ID to delete + + Returns: + True if deleted successfully + + Raises: + LawNotFoundError: If law not found + """ + try: + logger.info(f"🗑️ Deleting law id={law_id}") + + # Fetch law (raises if not found) + law = await self.get_law(law_id) + + # Delete from search index first (v2.1 SearchService integration) + try: + from app.services.search_service import SearchService + search_service = SearchService(self.db, use_cache=True) + await search_service.delete_law_index(law_id) + logger.info(f"🔍 Deleted law {law_id} from search index") + except Exception as e: + logger.warning(f"⚠️ Failed to delete law {law_id} from search index: {e}") + + # Delete law (cascade deletes articles) + await self.db.delete(law) + await self.db.commit() + + logger.info(f"✅ Law deleted: {law.reference}") + return True + + except LawNotFoundError: + raise + except Exception as e: + await self.db.rollback() + logger.error(f"❌ Error deleting law: {str(e)}") + raise LawServiceError(f"Failed to delete law: {str(e)}") + + # ======================================================================== + # Search Operations + # ======================================================================== + + async def search_laws( + self, + query: str, + filters: Optional[LawFilters] = None + ) -> SearchResponse: + """ + Search laws by title and content with optional filters. + + Args: + query: Search query string + filters: Optional filters to apply + + Returns: + SearchResponse with matching laws and relevance scores + """ + try: + import time + start_time = time.time() + + logger.info(f"🔍 Searching laws: query='{query}'") + + # Build search query + search_filter = or_( + Law.title.ilike(f"%{query}%"), + Law.content.ilike(f"%{query}%"), + Law.reference.ilike(f"%{query}%") + ) + + base_query = select(Law).where(search_filter).options(joinedload(Law.category)) + + # Apply additional filters if provided + if filters: + base_query = self._build_filters_query(base_query, filters) + + # Execute search + result = await self.db.execute(base_query) + laws = result.scalars().unique().all() + + # Calculate simple relevance scores + search_results = [] + for law in laws: + score = self._calculate_relevance_score(query, law) + search_results.append(SearchResult( + **LawResponse.model_validate(law).model_dump(), + relevance_score=score + )) + + # Sort by relevance + search_results.sort(key=lambda x: x.relevance_score, reverse=True) + + search_time_ms = (time.time() - start_time) * 1000 + + logger.info(f"✅ Search complete: {len(search_results)} results in {search_time_ms:.1f}ms") + + return SearchResponse( + query=query, + results=search_results, + total=len(search_results), + search_time_ms=search_time_ms + ) + + except Exception as e: + logger.error(f"❌ Error searching laws: {str(e)}") + raise LawServiceError(f"Failed to search laws: {str(e)}") + + # ======================================================================== + # Category Operations + # ======================================================================== + + async def create_category(self, category_data: CategoryCreate) -> Category: + """Create a new category.""" + try: + logger.info(f"📝 Creating category: {category_data.name}") + + category = Category(**category_data.model_dump()) + self.db.add(category) + await self.db.commit() + await self.db.refresh(category) + + logger.info(f"✅ Category created: {category.name} (id={category.id})") + return category + + except IntegrityError as e: + await self.db.rollback() + raise LawServiceError(f"Category creation failed: {str(e)}") + except Exception as e: + await self.db.rollback() + logger.error(f"❌ Error creating category: {str(e)}") + raise LawServiceError(f"Failed to create category: {str(e)}") + + async def get_category(self, category_id: int) -> Category: + """Get a category by ID.""" + try: + query = select(Category).where(Category.id == category_id) + result = await self.db.execute(query) + category = result.scalar_one_or_none() + + if not category: + raise CategoryNotFoundError(f"Category with id={category_id} not found") + + return category + except CategoryNotFoundError: + raise + except Exception as e: + logger.error(f"❌ Error fetching category: {str(e)}") + raise LawServiceError(f"Failed to fetch category: {str(e)}") + + async def list_categories(self) -> List[CategoryResponse]: + """List all categories with law counts.""" + try: + query = ( + select( + Category, + func.count(Law.id).label("law_count") + ) + .outerjoin(Law, Category.id == Law.category_id) + .group_by(Category.id) + .order_by(Category.name) + ) + + result = await self.db.execute(query) + rows = result.all() + + categories = [] + for category, law_count in rows: + cat_dict = CategoryResponse.model_validate(category).model_dump() + cat_dict["law_count"] = law_count + categories.append(CategoryResponse(**cat_dict)) + + logger.info(f"✅ Listed {len(categories)} categories") + return categories + + except Exception as e: + logger.error(f"❌ Error listing categories: {str(e)}") + raise LawServiceError(f"Failed to list categories: {str(e)}") + + # ======================================================================== + # Statistics Operations + # ======================================================================== + + async def get_language_stats(self) -> LanguageStats: + """ + Get statistics on law distribution by language (v2.1 feature). + + Returns: + LanguageStats with counts by language + """ + try: + logger.info("📊 Calculating language statistics") + + # Count by language + query = ( + select( + Law.language, + func.count(Law.id).label("count") + ) + .group_by(Law.language) + ) + + result = await self.db.execute(query) + rows = result.all() + + # Build stats + french = 0 + english = 0 + unknown = 0 + + for lang, count in rows: + if lang == "fr": + french = count + elif lang == "en": + english = count + else: + unknown += count + + total = french + english + unknown + + stats = LanguageStats( + french=french, + english=english, + unknown=unknown, + total=total + ) + + logger.info(f"✅ Language stats: fr={french}, en={english}, unknown={unknown}") + return stats + + except Exception as e: + logger.error(f"❌ Error calculating language stats: {str(e)}") + raise LawServiceError(f"Failed to calculate language stats: {str(e)}") + + async def get_law_stats(self) -> LawStats: + """Get comprehensive law statistics.""" + assert self.db is not None, "Database session must be initialized" + assert hasattr(self, "db"), "LawService must have a db attribute" + + try: + logger.info("📊 Calculating comprehensive law statistics") + + # Total laws and articles + total_laws_query = select(func.count(Law.id)) + total_laws = (await self.db.execute(total_laws_query)).scalar_one() + + total_articles_query = select(func.count(Article.id)) + total_articles = (await self.db.execute(total_articles_query)).scalar_one() + + # Language stats + lang_stats = await self.get_language_stats() + + # Status distribution + status_query = ( + select(Law.status, func.count(Law.id)) + .group_by(Law.status) + ) + status_result = await self.db.execute(status_query) + by_status = {status: count for status, count in status_result.all()} + + # Type distribution + type_query = ( + select(Law.type, func.count(Law.id)) + .group_by(Law.type) + ) + type_result = await self.db.execute(type_query) + by_type = {law_type: count for law_type, count in type_result.all()} + + # Top categories + cat_query = ( + select( + Category.id, + Category.name, + func.count(Law.id).label("law_count") + ) + .outerjoin(Law, Category.id == Law.category_id) + .group_by(Category.id, Category.name) + .order_by(func.count(Law.id).desc()) + .limit(5) + ) + cat_result = await self.db.execute(cat_query) + top_categories = [ + CategoryStats( + category_id=cat_id, + category_name=cat_name, + law_count=count, + percentage=(count / total_laws * 100) if total_laws > 0 else 0.0 + ) + for cat_id, cat_name, count in cat_result.all() + ] + + # Latest publication + latest_query = ( + select(Law.publication_date) + .where(Law.publication_date.isnot(None)) + .order_by(Law.publication_date.desc()) + .limit(1) + ) + latest_result = await self.db.execute(latest_query) + latest_publication = latest_result.scalar_one_or_none() + + # Average articles per law + avg_articles = total_articles / total_laws if total_laws > 0 else 0.0 + + stats = LawStats( + total_laws=total_laws, + total_articles=total_articles, + by_language=lang_stats, + by_status=by_status, + by_type=by_type, + top_categories=top_categories, + avg_articles_per_law=avg_articles, + latest_publication=latest_publication + ) + + logger.info(f"✅ Law stats calculated: {total_laws} laws, {total_articles} articles") + return stats + + except Exception as e: + logger.error(f"❌ Error calculating law stats: {str(e)}") + raise LawServiceError(f"Failed to calculate law stats: {str(e)}") + + # ======================================================================== + # Private Helper Methods + # ======================================================================== + + async def _detect_language_and_confidence(self, text: str) -> Tuple[str, float]: + """ + Detect language and confidence score using LanguageDetector. + + Args: + text: Text to analyze + + Returns: + Tuple of (language_code, confidence_score) + """ + try: + result = self.language_detector.detect(text) + return result["language"], result["confidence"] + except Exception as e: + logger.warning(f"⚠️ Language detection failed: {str(e)}, defaulting to 'fr'") + return "fr", 0.0 + + async def _suggest_categories(self, text: str) -> Tuple[List[int], Optional[float]]: + """ + Suggest top 3 categories using DocumentClassifier. + + Args: + text: Text to classify + + Returns: + Tuple of (category_ids, top_confidence) + """ + try: + # Get top 3 category suggestions (using top_k parameter) + predictions = self.document_classifier.classify(text, top_k=3) + + if not predictions: + return [], None + + # Extract category IDs and top confidence + category_ids = [pred[0] for pred in predictions] + top_confidence = predictions[0][1] if predictions else None + + return category_ids, top_confidence + + except Exception as e: + logger.warning(f"⚠️ Category suggestion failed: {str(e)}") + return [], None + + async def _validate_unique_reference( + self, + reference: str, + exclude_id: Optional[int] = None + ) -> None: + """ + Validate that reference is unique. + + Args: + reference: Reference to check + exclude_id: Optional law ID to exclude (for updates) + + Raises: + DuplicateReferenceError: If reference exists + """ + query = select(Law).where(Law.reference == reference) + + if exclude_id: + query = query.where(Law.id != exclude_id) + + result = await self.db.execute(query) + existing = result.scalar_one_or_none() + + if existing: + raise DuplicateReferenceError( + f"Law with reference '{reference}' already exists (id={existing.id})" + ) + + def _build_filters_query(self, query, filters: LawFilters): + """ + Apply filters to a SQLAlchemy query. + + Args: + query: Base SQLAlchemy query + filters: LawFilters object + + Returns: + Modified query with filters applied + """ + conditions = [] + + # Language filter (v2.1) + if filters.language: + conditions.append(Law.language == filters.language) + + # Category filter + if filters.category_id: + conditions.append(Law.category_id == filters.category_id) + + # Status filter + if filters.status: + conditions.append(Law.status == filters.status) + + # Type filter + if filters.type: + conditions.append(Law.type == filters.type) + + # Date range filters + if filters.year_from: + conditions.append(extract("year", Law.publication_date) >= filters.year_from) + + if filters.year_to: + conditions.append(extract("year", Law.publication_date) <= filters.year_to) + + # Apply all conditions + if conditions: + query = query.where(and_(*conditions)) + + return query + + def _apply_sorting(self, query, filters: LawFilters): + """ + Apply sorting to a query. + + Args: + query: Base query + filters: Filters with sorting params + + Returns: + Query with sorting applied + """ + # Map sort fields to model attributes + sort_field_map = { + "created_at": Law.created_at, + "updated_at": Law.updated_at, + "publication_date": Law.publication_date, + "title": Law.title, + "reference": Law.reference, + } + + sort_field = sort_field_map.get(filters.sort_by, Law.created_at) + + if filters.sort_order == "asc": + query = query.order_by(sort_field.asc()) + else: + query = query.order_by(sort_field.desc()) + + return query + + def _calculate_relevance_score(self, query: str, law: Law) -> float: + """ + Calculate simple relevance score for search results. + + Args: + query: Search query + law: Law object + + Returns: + Relevance score (0.0-1.0) + """ + query_lower = query.lower() + score = 0.0 + + # Title match (highest weight) + if query_lower in law.title.lower(): + score += 0.5 + if law.title.lower().startswith(query_lower): + score += 0.2 + + # Reference match + if query_lower in law.reference.lower(): + score += 0.3 + + # Content match (lower weight) + if query_lower in law.content.lower(): + score += 0.2 + + # Normalize to 0.0-1.0 + return min(score, 1.0) diff --git a/app/services/llama_parse_service.py b/app/services/llama_parse_service.py new file mode 100644 index 0000000000000000000000000000000000000000..01f09296d8c7939bc749172315115d02c8bac140 --- /dev/null +++ b/app/services/llama_parse_service.py @@ -0,0 +1,413 @@ +""" +Service LlamaParse pour extraction de texte de haute qualité depuis PDFs juridiques. + +Ce service utilise LlamaParse (LlamaIndex Cloud) pour: +- Extraction propre sans tampons/signatures +- Compréhension de la structure des articles +- Nettoyage automatique des artefacts OCR +""" + +import asyncio +import logging +import os +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + +# Instructions personnalisées pour parser les documents juridiques camerounais +LEGAL_DOCUMENT_PARSING_INSTRUCTIONS = """ +You are parsing official Cameroonian legal documents (laws, decrees, ordinances, constitutional texts). + +## WHAT TO EXTRACT (IMPORTANT): +- All articles with their numbers (Article 1, Article 2, Article premier, etc.) +- All article content/text in full +- Section headers (LIVRE, CHAPITRE, SECTION, TITRE) +- Preamble text if present +- Any numbered or lettered subsections within articles + +## WHAT TO COMPLETELY IGNORE AND EXCLUDE: + +### 1. OFFICIAL STAMPS (Tampons officiels): +- Rectangular stamps with "PRESIDENCE DE LA REPUBLIQUE" / "PRESIDENCY OF THE REPUBLIC" +- Stamps containing "SECRETARIAT GENERAL" +- "SERVICE DU FICHIER LEGISLATIF ET REGLEMENTAIRE" stamps +- "LEGISLATIVE AND STATUTORY AFFAIRS CARD INDEX SERVICE" stamps +- "COPIE CERTIFIEE CONFORME" / "CERTIFIED TRUE COPY" stamps (any color: black, red/pink, blue) +- Any rectangular border stamps with official text + +### 2. OFFICIAL SEALS (Sceaux ronds): +- Round seals with "REPUBLIQUE DU CAMEROUN" around the edge +- Seals with "LE PRESIDENT" or "THE PRESIDENT" text +- Seals with stars and national emblems +- Seals containing "PAIX TRAVAIL PATRIE" or "PEACE WORK FATHERLAND" +- Any circular official emblems + +### 3. SIGNATURES: +- Handwritten signatures (cursive scribbles) +- The signature of "Paul BIYA" or any president's signature +- Paraphs and initials +- Any handwritten marks or annotations + +### 4. DATE STAMPS: +- "Yaoundé, le [DATE]" location/date headers +- Date stamps like "25 FEV 2021", "12 JUIL 2016" +- Any administrative date markings + +### 5. HEADERS AND FOOTERS: +- Page numbers +- Running headers/footers +- "PAIX - TRAVAIL - PATRIE" motto + +### 6. SCAN ARTIFACTS: +- Blurry or partial text from stamps +- OCR garbage characters (e.g., "ET 2£ SLE", "Ei 5", "SS---") +- Text that appears faded, partial, or illegible +- Random characters from poor scanning + +## OUTPUT FORMAT: +- Preserve the legal structure exactly +- Use clear article separation +- Keep French language as-is (do not translate) +- Format section headers in UPPERCASE +- Each article should start with "Article X" or "ARTICLE X" on its own line +- Paragraph text should follow directly + +## EXAMPLE OUTPUT: +CHAPITRE I +DISPOSITIONS GÉNÉRALES + +Article 1 +La présente loi régit les conditions d'exercice de la profession... + +Article 2 +Au sens de la présente loi, on entend par: +a) Professionnel: toute personne physique ou morale... +b) Activité: tout acte exercé... +""" + + +class LlamaParseError(Exception): + """Exception levée lors d'erreurs LlamaParse.""" + pass + + +class LlamaParseService: + """ + Service d'extraction de texte de haute qualité via LlamaParse. + + LlamaParse utilise des LLMs pour comprendre et extraire le contenu + des documents de manière intelligente, ignorant les éléments + non pertinents comme les tampons et signatures. + """ + + def __init__(self, api_key: Optional[str] = None): + """ + Initialise le service LlamaParse. + + Args: + api_key: Clé API LlamaCloud. Si non fournie, utilise + settings.LLAMA_CLOUD_API_KEY puis LLAMA_CLOUD_API_KEY env var. + """ + if api_key: + self.api_key = api_key + else: + try: + from app.core.config import settings + self.api_key = settings.LLAMA_CLOUD_API_KEY or os.getenv("LLAMA_CLOUD_API_KEY") + except Exception: + self.api_key = os.getenv("LLAMA_CLOUD_API_KEY") + + self._parser = None + self._initialized = False + + if not self.api_key: + logger.warning("LLAMA_CLOUD_API_KEY not set. LlamaParse will not be available.") + else: + logger.info(f"✅ LlamaParseService configured with key ...{self.api_key[-6:]}") + + async def _ensure_initialized(self): + """Initialise le parser LlamaParse si nécessaire.""" + if self._initialized: + return + + if not self.api_key: + raise LlamaParseError("LLAMA_CLOUD_API_KEY is not configured") + + try: + from llama_parse import LlamaParse + + self._parser = LlamaParse( + api_key=self.api_key, + result_type="markdown", + parsing_instruction=LEGAL_DOCUMENT_PARSING_INSTRUCTIONS, + language="fr", + verbose=False, + ) + self._initialized = True + logger.info("LlamaParse service initialized successfully") + + except ImportError: + raise LlamaParseError( + "llama-parse package not installed. Run: pip install llama-parse" + ) + except Exception as e: + raise LlamaParseError(f"Failed to initialize LlamaParse: {e}") + + async def extract_text( + self, + pdf_path: Path, + timeout_seconds: int = 300, + ) -> str: + """ + Extrait le texte d'un PDF juridique via LlamaParse. + + Args: + pdf_path: Chemin vers le fichier PDF + timeout_seconds: Timeout maximum (défaut: 5 minutes) + + Returns: + Texte extrait et nettoyé + + Raises: + LlamaParseError: Si l'extraction échoue + """ + await self._ensure_initialized() + + if not pdf_path.exists(): + raise LlamaParseError(f"PDF file not found: {pdf_path}") + + try: + logger.info(f"Starting LlamaParse extraction for: {pdf_path.name}") + + # LlamaParse is sync, run in thread pool + loop = asyncio.get_event_loop() + documents = await asyncio.wait_for( + loop.run_in_executor( + None, + lambda: self._parser.load_data(str(pdf_path)) + ), + timeout=timeout_seconds + ) + + if not documents: + logger.warning(f"No content extracted from: {pdf_path.name}") + return "" + + # Combine all document chunks + full_text = "\n\n".join(doc.text for doc in documents if doc.text) + + # Post-processing cleanup + cleaned_text = self._post_process(full_text) + + logger.info( + f"LlamaParse extraction complete: {len(cleaned_text)} chars from {pdf_path.name}" + ) + + return cleaned_text + + except asyncio.TimeoutError: + raise LlamaParseError( + f"Extraction timeout after {timeout_seconds}s for {pdf_path.name}" + ) + except Exception as e: + logger.error(f"LlamaParse extraction failed: {e}") + raise LlamaParseError(f"Extraction failed: {e}") + + def _post_process(self, text: str) -> str: + """ + Nettoyage post-extraction. + + Même avec des instructions, certains artefacts peuvent persister. + Ce post-processing finalise le nettoyage des tampons camerounais. + """ + assert isinstance(text, str), "text must be a string" + assert len(text) < 10_000_000, "text is unreasonably large (>10MB)" + + import re + + if not text: + return "" + + cleaned = text + + # Remove any remaining page markers + cleaned = re.sub(r"<>?", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\[?\s*Page\s*\d+\s*\]?", "", cleaned, flags=re.IGNORECASE) + + # ======================================== + # OFFICIAL STAMPS (Tampons officiels) + # ======================================== + stamp_patterns = [ + # Presidency stamps + r"PRESIDENCE\s+DE\s+LA\s+REPUB[A-Z]*\.?", + r"PRESIDENCY\s+OF\s+THE\s+REPUBLIC", + # Secretariat stamps + r"SECRETARIAT\s+GEN[A-Z]*\s*\"?", + # Legislative service stamps + r"SERVICE\s+DU\s+FICHIER\s+LEGISLATIF[\s\w]*", + r"LEGISLATIVE\s+AND\s+STATUTORY\s+AFFAIRS[\s\w]*", + # Certified copy stamps + r"COPIE\s+CERTIFIEE\s+[A-Z\s]*", + r"CERTIFIED\s+TRUE\s+COPY", + # Republic + r"REPUBLIQUE\s+DU\s+CAMEROUN", + r"REPUBLIC\s+OF\s+CAMEROON", + ] + + # ======================================== + # OFFICIAL SEALS (Sceaux ronds) + # ======================================== + seal_patterns = [ + r"LE\s+PR[EÉ]SIDENT", + r"THE\s+PRESIDENT", + r"PAIX\s*[-–—]?\s*TRAVAIL\s*[-–—]?\s*PATRIE", + r"PEACE\s*[-–—]?\s*WORK\s*[-–—]?\s*FATHERLAND", + ] + + # ======================================== + # SIGNATURES + # ======================================== + signature_patterns = [ + r"Paul\s+BIYA", + r"PAUL\s+BIYA", + r"LE\s+PR[EÉ]SIDENT\s+DE\s+LA\s+R[EÉ]PUBLIQUE\s*,?", + ] + + # ======================================== + # DATE STAMPS + # ======================================== + date_patterns = [ + r"Yaound[eé],?\s+le\s+\d{1,2}\s+[A-Z]{3,4}\.?\s+\d{4}", + r"\d{1,2}\s+(JAN|FEV|MARS?|AVR|MAI|JUIN?|JUIL?|AO[UÛ]T?|SEPT?|OCT|NOV|DEC)\.?\s+\d{4}", + ] + + # Apply all patterns + all_patterns = stamp_patterns + seal_patterns + signature_patterns + date_patterns + for pattern in all_patterns: + cleaned = re.sub(pattern, "", cleaned, flags=re.IGNORECASE) + + # Remove lines of only dashes/underscores (stamp borders) + cleaned = re.sub(r"^[-–—_=]{3,}\s*$", "", cleaned, flags=re.MULTILINE) + + # Remove isolated short lines that look like stamp fragments + cleaned = re.sub(r"^[A-Z]{1,3}\s+\d*[£€$]?\s*[A-Z]{0,4}\s*[A-Za-z]{0,5}$", "", cleaned, flags=re.MULTILINE) + + # Remove lines with only stars (from seals) + cleaned = re.sub(r"^[\s\*★☆]+$", "", cleaned, flags=re.MULTILINE) + + # Normalize whitespace + cleaned = re.sub(r"\n{3,}", "\n\n", cleaned) + cleaned = re.sub(r"[ \t]+", " ", cleaned) + + return cleaned.strip() + + async def extract_articles( + self, + pdf_path: Path, + timeout_seconds: int = 300, + ) -> list[dict]: + """ + Extrait et parse les articles d'un PDF juridique. + + Args: + pdf_path: Chemin vers le fichier PDF + timeout_seconds: Timeout maximum + + Returns: + Liste de dictionnaires avec: + - number: Numéro de l'article + - title: Titre/en-tête de l'article + - content: Contenu de l'article + """ + full_text = await self.extract_text(pdf_path, timeout_seconds) + + if not full_text: + return [] + + return self._parse_articles(full_text) + + def _parse_articles(self, text: str) -> list[dict]: + """Parse le texte en articles structurés.""" + import re + + articles = [] + + # Pattern pour détecter les articles + article_pattern = re.compile( + r"^(Article\s+(\d+|premier|[IVX]+))\s*[-:.]?\s*(.*)$", + re.IGNORECASE | re.MULTILINE + ) + + # Split by article markers + parts = article_pattern.split(text) + + current_preamble = parts[0].strip() if parts else "" + + # Add preamble as first "article" if exists + if current_preamble: + articles.append({ + "number": "0", + "title": "Préambule", + "content": current_preamble, + }) + + # Process article matches + i = 1 + while i < len(parts) - 2: + full_header = parts[i].strip() + number = parts[i + 1].strip() + title_line = parts[i + 2].strip() if i + 2 < len(parts) else "" + content = parts[i + 3].strip() if i + 3 < len(parts) else "" + + # Normalize number + if number.lower() == "premier": + number = "1" + + articles.append({ + "number": number, + "title": title_line or f"Article {number}", + "content": content, + }) + + i += 4 + + return articles + + def is_available(self) -> bool: + """Vérifie si LlamaParse est disponible.""" + return bool(self.api_key) + + async def health_check(self) -> dict: + """Vérifie l'état du service.""" + status = { + "service": "llama_parse", + "available": self.is_available(), + "api_key_configured": bool(self.api_key), + } + + if self.api_key: + try: + await self._ensure_initialized() + status["initialized"] = True + status["status"] = "healthy" + except Exception as e: + status["initialized"] = False + status["status"] = "error" + status["error"] = str(e) + else: + status["status"] = "unavailable" + + return status + + +# Singleton instance +_llama_parse_service: Optional[LlamaParseService] = None + + +def get_llama_parse_service() -> LlamaParseService: + """Get or create the LlamaParseService singleton instance.""" + global _llama_parse_service + if _llama_parse_service is None: + _llama_parse_service = LlamaParseService() + return _llama_parse_service diff --git a/app/services/meilisearch_service.py b/app/services/meilisearch_service.py new file mode 100644 index 0000000000000000000000000000000000000000..0b718f41763d51b2e6928322750411c2b52eb3d0 --- /dev/null +++ b/app/services/meilisearch_service.py @@ -0,0 +1,220 @@ +""" +Meilisearch indexing service for automatic article indexing. +Provides methods to add/remove articles from Meilisearch index. +""" +import meilisearch +from typing import List, Optional +import logging + +from app.core.config import settings + +logger = logging.getLogger(__name__) + + +class MeilisearchService: + """ + Service for managing Meilisearch article indexing. + + Provides automatic indexing when articles are created and + automatic removal when laws are deleted. + """ + + ARTICLES_INDEX = "articles" + LAWS_INDEX = "laws" + + _client: Optional[meilisearch.Client] = None + + @classmethod + def get_client(cls) -> meilisearch.Client: + """Get or create Meilisearch client singleton.""" + if cls._client is None: + cls._client = meilisearch.Client( + settings.MEILISEARCH_URL, + settings.MEILISEARCH_KEY + ) + # Ensure indexes exist + cls._ensure_indexes() + return cls._client + + @classmethod + def _ensure_indexes(cls): + """Ensure required indexes exist with proper settings.""" + client = cls._client + + # Create articles index if not exists + try: + client.create_index(cls.ARTICLES_INDEX, {'primaryKey': 'id'}) + logger.info(f"✅ Created '{cls.ARTICLES_INDEX}' index") + except Exception: + pass # Index already exists + + # Create laws index if not exists + try: + client.create_index(cls.LAWS_INDEX, {'primaryKey': 'id'}) + logger.info(f"✅ Created '{cls.LAWS_INDEX}' index") + except Exception: + pass # Index already exists + + # Configure articles index + try: + articles_index = client.get_index(cls.ARTICLES_INDEX) + articles_index.update_settings({ + 'searchableAttributes': [ + 'content', 'number', 'title', 'section', + 'law_title', 'law_reference', 'category' + ], + 'filterableAttributes': ['law_id', 'category'], + 'sortableAttributes': ['number'], + }) + except Exception as e: + logger.warning(f"⚠️ Could not configure articles index: {e}") + + @classmethod + def index_articles(cls, articles: List[dict], law_id: int, + law_title: str, law_reference: str, + category_name: Optional[str] = None): + """ + Index articles in Meilisearch. + + Args: + articles: List of article dicts with id, number, title, section, content + law_id: Parent law ID + law_title: Parent law title + law_reference: Parent law reference + category_name: Optional category name + """ + client = cls.get_client() + + if not articles: + logger.warning("⚠️ No articles to index") + return + + # Prepare documents for indexing + documents = [] + for article in articles: + documents.append({ + 'id': article.get('id'), + 'number': article.get('number', ''), + 'title': article.get('title', ''), + 'section': article.get('section', ''), + 'content': article.get('content', ''), + 'law_id': law_id, + 'law_title': law_title, + 'law_reference': law_reference, + 'category': category_name or 'Non catégorisé' + }) + + # Index in Meilisearch + try: + articles_index = client.get_index(cls.ARTICLES_INDEX) + task = articles_index.add_documents(documents) + logger.info( + f"📤 Indexed {len(documents)} articles for law {law_id} " + f"(task: {task.task_uid})" + ) + return task + except Exception as e: + logger.error(f"❌ Failed to index articles for law {law_id}: {e}") + raise + + @classmethod + def remove_law_articles(cls, law_id: int): + """ + Remove all articles for a law from Meilisearch. + + Args: + law_id: ID of the law whose articles should be removed + """ + client = cls.get_client() + + try: + articles_index = client.get_index(cls.ARTICLES_INDEX) + + # Get all articles for this law + search_result = articles_index.search('', { + 'filter': f'law_id = {law_id}', + 'limit': 1000, + 'attributesToRetrieve': ['id'] + }) + + article_ids = [hit['id'] for hit in search_result.get('hits', [])] + + if article_ids: + task = articles_index.delete_documents(article_ids) + logger.info( + f"🗑️ Removed {len(article_ids)} articles for law {law_id} " + f"(task: {task.task_uid})" + ) + return task + else: + logger.info(f"ℹ️ No articles found for law {law_id} in Meilisearch") + return None + + except Exception as e: + logger.error(f"❌ Failed to remove articles for law {law_id}: {e}") + raise + + @classmethod + def index_law(cls, law_id: int, reference: str, title: str, + content: str, law_type: str, language: Optional[str], + status: str, category_id: Optional[int], + category_name: Optional[str], publication_year: Optional[int]): + """ + Index a law in Meilisearch. + + Args: + law_id: Law ID + reference: Law reference + title: Law title + content: Law content (full text) + law_type: Law type + language: Language code + status: Law status + category_id: Category ID + category_name: Category name + publication_year: Publication year + """ + client = cls.get_client() + + try: + laws_index = client.get_index(cls.LAWS_INDEX) + task = laws_index.add_documents([{ + 'id': law_id, + 'reference': reference, + 'title': title, + 'content': content[:50000] if content else '', # Meilisearch limit + 'type': law_type, + 'language': language, + 'status': status, + 'category_id': category_id, + 'category_name': category_name, + 'publication_year': publication_year + }]) + logger.info(f"📤 Indexed law {law_id} in Meilisearch (task: {task.task_uid})") + return task + except Exception as e: + logger.error(f"❌ Failed to index law {law_id}: {e}") + raise + + @classmethod + def remove_law(cls, law_id: int): + """ + Remove a law and its articles from Meilisearch. + + Args: + law_id: ID of the law to remove + """ + client = cls.get_client() + + # Remove articles first + cls.remove_law_articles(law_id) + + # Remove law + try: + laws_index = client.get_index(cls.LAWS_INDEX) + task = laws_index.delete_document(law_id) + logger.info(f"🗑️ Removed law {law_id} from Meilisearch (task: {task.task_uid})") + return task + except Exception as e: + logger.error(f"❌ Failed to remove law {law_id}: {e}") + raise diff --git a/app/services/ocr_service.py b/app/services/ocr_service.py new file mode 100644 index 0000000000000000000000000000000000000000..a857d7206ce5d78f34047a5c9acbc4699110eb1e --- /dev/null +++ b/app/services/ocr_service.py @@ -0,0 +1,461 @@ +""" +Service OCR pour extraction de texte depuis PDFs scannés. + +Ce service gère: +- Détection type PDF (natif vs scanné vs hybride) +- OCR avec Tesseract (FR + EN) +- Post-processing et nettoyage du texte +- Gestion performance et timeouts + +Objectif: Extraction texte fiable en 5-30s selon qualité scan. + +Usage: + service = OCRService() + result = await service.process_pdf(pdf_path, language="fra+eng") + # {'pdf_type': 'scanned', 'text': '...', 'confidence': 0.92} +""" + +import logging +import time +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError +from pathlib import Path +from typing import Any, Dict, List, Optional + +from app.schemas.ocr import ( + OCRResult, + PDFType, + PDFTypeDetectionResult, +) +from app.utils.ocr_utils import ( + assess_text_quality, + clean_ocr_text, + convert_pdf_to_images, + estimate_ocr_time, + has_embedded_text, + has_images, + merge_text_blocks, +) + +logger = logging.getLogger(__name__) + + +class OCRError(Exception): + """Exception levée lors d'erreurs OCR.""" + + pass + + +class OCRService: + """ + Service d'extraction de texte par OCR. + + Fonctionnalités: + - Détection automatique type PDF + - OCR Tesseract multi-langue (FR, EN) + - Nettoyage et post-processing + - Gestion timeouts et performance + + Attributes: + tesseract_path: Chemin vers Tesseract (auto-détecté) + tesseract_available: Si Tesseract est disponible + default_language: Langue par défaut + default_dpi: DPI par défaut + """ + + def __init__( + self, + tesseract_path: Optional[str] = None, + default_language: str = "fra+eng", + default_dpi: int = 300, + ): + """ + Initialise le service OCR. + + Args: + tesseract_path: Chemin vers Tesseract (optionnel, auto-détecté) + default_language: Langue par défaut (défaut: fra+eng) + default_dpi: DPI par défaut (défaut: 300) + + Raises: + OCRError: Si Tesseract requis mais non disponible + """ + logger.info("🚀 Initialisation du OCRService...") + + self.default_language = default_language + self.default_dpi = default_dpi + self.tesseract_path = tesseract_path + + # Vérifier disponibilité Tesseract + self.tesseract_available = self._check_tesseract() + + if self.tesseract_available: + logger.info(f" ✅ Tesseract disponible: {self._get_tesseract_version()}") + else: + logger.warning(" ⚠️ Tesseract non disponible, mode mock activé") + + logger.info( + f"✅ OCRService initialisé " + f"(langue: {self.default_language}, DPI: {self.default_dpi})" + ) + + def _check_tesseract(self) -> bool: + """Vérifie si Tesseract est disponible.""" + try: + import pytesseract + + if self.tesseract_path: + pytesseract.pytesseract.tesseract_cmd = self.tesseract_path + + # Test rapide + version = pytesseract.get_tesseract_version() + return version is not None + + except Exception as e: + logger.debug(f"Tesseract check failed: {e}") + return False + + def _get_tesseract_version(self) -> str: + """Récupère la version de Tesseract.""" + try: + import pytesseract + + version = pytesseract.get_tesseract_version() + return str(version) + except Exception: + return "unknown" + + async def detect_pdf_type(self, pdf_path: Path) -> PDFTypeDetectionResult: + """ + Détecte le type de PDF (natif, scanné, hybride). + + Args: + pdf_path: Chemin vers le PDF + + Returns: + PDFTypeDetectionResult avec classification + + Example: + >>> result = await service.detect_pdf_type(Path("doc.pdf")) + >>> result.pdf_type + PDFType.SCANNED + >>> result.needs_ocr + True + """ + logger.info(f"🔍 Détection type PDF: {pdf_path.name}") + + try: + from pypdf import PdfReader + + reader = PdfReader(pdf_path) + page_count = len(reader.pages) + + # Vérifier texte embarqué + has_text, text_coverage = has_embedded_text(pdf_path) + + # Vérifier images + has_imgs = has_images(pdf_path) + + # Classifier le PDF + # Classifier le PDF - MODE AGRESSIF + # User wants "Brutal" extraction. + # If text_coverage is NOT perfect (100%), we default to HYBRID/OCR to catch stamps, side-notes, etc. + # But scanning huge PDFs is slow. + # Compromise: High threshold. + if text_coverage >= 0.98: + # Almost certainly native text throughout + pdf_type = PDFType.NATIVE + needs_ocr = False + else: + # Even if 80% coverage (0.8), we trigger OCR to be safe ("Brutal") + # This covers "Valid text BUT header/footer is image" + # And also covers SCANNED (coverage < 0.2) + pdf_type = PDFType.HYBRID + needs_ocr = True + + logger.info( + f" ✅ Type détecté: {pdf_type.value} " + f"(texte: {text_coverage:.0%}, images: {has_imgs})" + ) + + return PDFTypeDetectionResult( + pdf_type=pdf_type, + has_text=has_text, + has_images=has_imgs, + page_count=page_count, + text_coverage=text_coverage, + needs_ocr=needs_ocr, + ) + + except Exception as e: + logger.error(f"❌ Erreur détection type PDF: {e}") + raise OCRError(f"Échec détection type PDF: {e}") from e + + async def process_pdf( + self, + pdf_path: Path, + language: Optional[str] = None, + dpi: Optional[int] = None, + timeout_seconds: int = 60, + ) -> OCRResult: + """ + Traite un PDF avec OCR si nécessaire. + + Workflow: + 1. Détecte type PDF + 2. Si scanné/hybride: OCR + 3. Si natif: extraction texte direct + 4. Nettoyage et post-processing + + Args: + pdf_path: Chemin vers le PDF + language: Langue OCR (défaut: fra+eng) + dpi: DPI conversion (défaut: 300) + timeout_seconds: Timeout par page (défaut: 60s) + + Returns: + OCRResult avec texte extrait + + Raises: + OCRError: Si traitement échoue + + Example: + >>> result = await service.process_pdf(Path("scan.pdf")) + >>> result.text[:50] + 'Article 1. La présente loi régit...' + """ + assert pdf_path is not None, "pdf_path must not be None" + assert timeout_seconds > 0, "timeout_seconds must be positive" + + start_time = time.time() + language = language or self.default_language + dpi = dpi or self.default_dpi + + logger.info(f"📄 Traitement PDF: {pdf_path.name} (langue: {language})") + + # 1. Détecter type PDF + detection = await self.detect_pdf_type(pdf_path) + logger.debug(f" Type: {detection.pdf_type.value}, OCR requis: {detection.needs_ocr}") + + # 2. Extraire texte selon type + if detection.pdf_type == PDFType.NATIVE: + # Extraction directe + text = await self._extract_native_text(pdf_path) + confidence = 1.0 # Texte natif = confiance maximale + elif detection.needs_ocr: + # OCR requis + if not self.tesseract_available: + raise OCRError( + "Tesseract non disponible. OCR requis mais impossible." + ) + text = await self.extract_text_ocr( + pdf_path, language, dpi, timeout_seconds + ) + confidence = assess_text_quality(text) + else: + # Fallback + text = await self._extract_native_text(pdf_path) + confidence = 0.5 + + # 3. Nettoyage + cleaned_text = await self.clean_ocr_text(text) + + # Calculer temps de traitement + processing_time_ms = int((time.time() - start_time) * 1000) + + logger.info( + f"✅ Traitement terminé: {len(cleaned_text)} caractères " + f"({processing_time_ms}ms, confiance: {confidence:.2%})" + ) + + return OCRResult( + pdf_type=detection.pdf_type, + text=cleaned_text, + page_count=detection.page_count, + processing_time_ms=processing_time_ms, + language=language, + confidence=confidence, + ocr_engine="tesseract" if self.tesseract_available else "native", + text_length=len(cleaned_text), + ) + + async def _extract_native_text(self, pdf_path: Path) -> str: + """Extrait texte d'un PDF natif.""" + try: + from pypdf import PdfReader + + reader = PdfReader(pdf_path) + text_blocks = [] + + for page_num, page in enumerate(reader.pages, start=1): + text = page.extract_text() + if text.strip(): + # Add page marker for article extraction + text_blocks.append(f"<>\n{text}") + + merged = merge_text_blocks(text_blocks) + logger.debug(f" Texte natif extrait: {len(merged)} caractères") + + return merged + + except Exception as e: + logger.error(f"Erreur extraction texte natif: {e}") + raise OCRError(f"Échec extraction texte: {e}") from e + + async def extract_text_ocr( + self, + pdf_path: Path, + language: str = "fra+eng", + dpi: int = 300, + timeout_seconds: int = 60, + ) -> str: + """ + Extrait texte via OCR Tesseract. + + Args: + pdf_path: Chemin vers le PDF + language: Langue Tesseract + dpi: DPI conversion + timeout_seconds: Timeout par page + + Returns: + Texte extrait + + Raises: + OCRError: Si OCR échoue + """ + logger.info(f"🔍 OCR en cours: {pdf_path.name} ({language}, {dpi} DPI)") + + try: + import pytesseract + + # 1. Convertir PDF en images + images = convert_pdf_to_images(pdf_path, dpi=dpi) + logger.debug(f" {len(images)} pages converties en images") + + # 2. OCR sur chaque page (parallèle) + text_blocks = [] + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [] + + for i, image in enumerate(images): + future = executor.submit( + self._ocr_single_page, image, language, i + 1 + ) + futures.append(future) + + # Collecter résultats avec timeout + for future in futures: + try: + page_text = future.result(timeout=timeout_seconds) + if page_text.strip(): + text_blocks.append(page_text) + except FuturesTimeoutError: + logger.warning(f" ⚠️ Timeout OCR page") + except Exception as e: + logger.warning(f" ⚠️ Erreur OCR page: {e}") + + # 3. Fusionner + merged = merge_text_blocks(text_blocks) + logger.info(f" ✅ OCR terminé: {len(merged)} caractères extraits") + + return merged + + except ImportError as e: + raise OCRError("pytesseract non disponible") from e + except Exception as e: + logger.error(f"❌ Erreur OCR: {e}") + raise OCRError(f"Échec OCR: {e}") from e + + def _ocr_single_page(self, image, language: str, page_num: int) -> str: + """OCR sur une seule page (méthode synchrone pour ThreadPoolExecutor).""" + try: + import pytesseract + + # Configuration Tesseract + # Configuration Tesseract + # psm 3: Fully automatic page segmentation, but no OSD. (Default) + # psm 6: Assume a single uniform block of text. + # psm 1: Auto + OSD (can be flaky with layout). + # We use psm 6 or 3 to try to keep blocks roughly in order. + config = "--psm 3" + + text = pytesseract.image_to_string(image, lang=language, config=config) + + logger.debug(f" Page {page_num}: {len(text)} caractères") + # Add page marker for article extraction + return f"<>\n{text}" + + except Exception as e: + logger.warning(f" Erreur OCR page {page_num}: {e}") + return "" + + async def clean_ocr_text(self, text: str) -> str: + """ + Nettoie et post-traite le texte OCR. + + NOTE: Not recursive — delegates to the imported utility function + ``clean_ocr_text`` from ``app.utils.ocr_utils``, not to itself. + + Args: + text: Texte brut OCR + + Returns: + Texte nettoyé + + Example: + >>> cleaned = await service.clean_ocr_text("Article 1. Text...") + >>> cleaned + 'Article 1. Text...' + """ + return clean_ocr_text(text) + + def health_check(self) -> Dict[str, Any]: + """ + Vérifie l'état de santé du service. + + Returns: + Dictionnaire avec statut de chaque composant + """ + status = { + "service": "OCRService", + "status": "healthy", + "tesseract": {}, + "poppler": {}, + "supported_languages": [], + } + + # Vérifier Tesseract + try: + if self.tesseract_available: + import pytesseract + + version = self._get_tesseract_version() + langs = pytesseract.get_languages() + + status["tesseract"] = { + "available": True, + "version": version, + "path": str(pytesseract.pytesseract.tesseract_cmd), + } + status["supported_languages"] = langs + else: + status["tesseract"] = {"available": False} + status["status"] = "degraded" + + except Exception as e: + status["tesseract"] = {"available": False, "error": str(e)} + status["status"] = "degraded" + + # Vérifier Poppler (pdf2image) + try: + from pdf2image import pdfinfo_from_path + + # Test simple + status["poppler"] = {"available": True} + + except Exception as e: + status["poppler"] = {"available": False, "error": str(e)} + status["status"] = "degraded" + + return status diff --git a/app/services/persona_service.py b/app/services/persona_service.py new file mode 100644 index 0000000000000000000000000000000000000000..d2d4505e08c06f7bf3c5e633f6080d918c1c60ba --- /dev/null +++ b/app/services/persona_service.py @@ -0,0 +1,1204 @@ +""" +Service 9: PersonaService - Gestion des personas utilisateurs + analytics. + +Features: +- Persona operations (info, list, validate) +- Real-time analytics per persona +- Daily aggregation for historical trends +- Message feedback system (👍/👎) +- Engagement metrics and popular questions +- Persona comparison and trend analysis + +Author: JuriX Team +Version: 2.1.0 +""" + +import logging +from typing import Dict, List, Optional, Any, Tuple +from datetime import datetime, date, timedelta + +from sqlalchemy import select, func, and_, or_, desc, distinct +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import selectinload + +from app.models.conversation import ( + Conversation, Message, PersonaStat, MessageFeedback, PersonaInteraction +) +from app.services.prompts import SYSTEM_PROMPTS + +# Configure logger +logger = logging.getLogger(__name__) + + +# ============================================================================ +# CUSTOM EXCEPTIONS +# ============================================================================ + +class PersonaServiceError(Exception): + """Base exception for PersonaService operations.""" + pass + + +class InvalidPersonaError(PersonaServiceError): + """Raised when persona is not one of the 4 valid types.""" + pass + + +class MessageNotFoundError(PersonaServiceError): + """Raised when message ID doesn't exist.""" + pass + + +class FeedbackAlreadyExistsError(PersonaServiceError): + """Raised when trying to add duplicate feedback.""" + pass + + +# ============================================================================ +# PERSONA SERVICE +# ============================================================================ + +class PersonaService: + """ + Service for managing user personas and analytics. + + Supports 4 personas: + - citoyen: Simplified, empathetic responses for citizens + - avocat: Technical, precise legal citations for lawyers + - entrepreneur: Practical, compliance-focused for business owners + - étudiant: Pedagogical, step-by-step for students + + Features: + - Persona information and validation + - Real-time statistics per persona + - Daily aggregation for historical data + - User feedback system (helpful/unhelpful, 1-5 rating) + - Engagement metrics (messages per conversation, session duration) + - Popular questions tracking + - Persona comparison across metrics + - Trend analysis over time + + Usage: + service = PersonaService(db_session) + personas = await service.list_personas() + stats = await service.get_persona_stats("citoyen") + await service.add_message_feedback(msg_id, helpful=True, rating=5) + """ + + VALID_PERSONAS = ["citoyen", "avocat", "entrepreneur", "étudiant"] + + # Persona metadata + PERSONA_INFO = { + "citoyen": { + "display_name": "Citoyen", + "description": "Assistant bienveillant qui aide les citoyens à comprendre leurs droits en termes simples", + "icon": "👤", + "tone": "Empathique et rassurant", + "example_questions": [ + "Quels sont mes droits en tant que locataire?", + "Comment porter plainte au commissariat?", + "Que faire en cas de licenciement abusif?" + ] + }, + "avocat": { + "display_name": "Avocat", + "description": "Assistant expert fournissant des analyses juridiques précises avec références complètes", + "icon": "⚖️", + "tone": "Technique et nuancé", + "example_questions": [ + "Quelle est la jurisprudence sur l'article 5 du Code pénal?", + "Analyse des vices de consentement en droit camerounais", + "Procédure d'appel en matière civile" + ] + }, + "entrepreneur": { + "display_name": "Entrepreneur", + "description": "Consultant juridique spécialisé en droit des affaires avec conseils actionnables", + "icon": "💼", + "tone": "Professionnel et pragmatique", + "example_questions": [ + "Comment créer une SARL au Cameroun?", + "Obligations fiscales d'une entreprise", + "Réglementation des contrats commerciaux" + ] + }, + "étudiant": { + "display_name": "Étudiant", + "description": "Professeur patient qui explique les concepts juridiques de manière pédagogique", + "icon": "🎓", + "tone": "Pédagogique et encourageant", + "example_questions": [ + "Qu'est-ce que la hiérarchie des normes?", + "Expliquer le principe de légalité", + "Comment rédiger un cas pratique?" + ] + } + } + + def __init__(self, db: AsyncSession): + """ + Initialize PersonaService. + + Args: + db: Async database session + """ + self.db = db + logger.info("👥 PersonaService initialized") + + # ======================================================================== + # PERSONA OPERATIONS + # ======================================================================== + + async def get_persona_info(self, persona: str) -> Dict[str, Any]: + """ + Get information about a specific persona. + + Args: + persona: Persona name (citoyen|avocat|entrepreneur|étudiant) + + Returns: + Dict with persona metadata + + Raises: + InvalidPersonaError: If persona is invalid + """ + await self.validate_persona(persona) + + info = self.PERSONA_INFO[persona].copy() + info["name"] = persona + + logger.debug(f"📋 Retrieved info for persona: {persona}") + return info + + async def list_personas(self) -> List[Dict[str, Any]]: + """ + List all available personas with metadata. + + Returns: + List of persona info dicts + """ + personas = [] + for persona_name in self.VALID_PERSONAS: + info = self.PERSONA_INFO[persona_name].copy() + info["name"] = persona_name + personas.append(info) + + logger.debug(f"📋 Listed {len(personas)} personas") + return personas + + async def validate_persona(self, persona: str) -> bool: + """ + Validate that persona is one of the 4 supported types. + + Args: + persona: Persona name to validate + + Returns: + True if valid + + Raises: + InvalidPersonaError: If persona is not valid + """ + if persona not in self.VALID_PERSONAS: + logger.warning(f"❌ Invalid persona: {persona}") + raise InvalidPersonaError( + f"Invalid persona '{persona}'. Must be one of: {', '.join(self.VALID_PERSONAS)}" + ) + return True + + # ======================================================================== + # ANALYTICS - REAL-TIME + # ======================================================================== + + async def get_persona_stats( + self, + persona: str, + start_date: Optional[date] = None, + end_date: Optional[date] = None + ) -> Dict[str, Any]: + """ + Get statistics for a specific persona over a date range. + + Queries the persona_stats table for aggregated data. + + Args: + persona: Persona name + start_date: Start date (inclusive), defaults to 30 days ago + end_date: End date (inclusive), defaults to today + + Returns: + Dict with usage, performance, engagement, and quality metrics + + Raises: + InvalidPersonaError: If persona is invalid + """ + await self.validate_persona(persona) + + # Default date range: last 30 days + if not end_date: + end_date = date.today() + if not start_date: + start_date = end_date - timedelta(days=30) + + # Query persona_stats table + stmt = select(PersonaStat).where( + and_( + PersonaStat.persona == persona, + PersonaStat.date >= start_date, + PersonaStat.date <= end_date + ) + ).order_by(PersonaStat.date) + + result = await self.db.execute(stmt) + stats = result.scalars().all() + + if not stats: + logger.debug(f"📊 No stats found for {persona} between {start_date} and {end_date}") + return self._empty_stats_response(persona, start_date, end_date) + + # Aggregate stats across date range + aggregated = self._aggregate_stats(stats) + + response = { + "persona": persona, + "date_range": { + "start": start_date.isoformat(), + "end": end_date.isoformat() + }, + "usage": { + "total_questions": aggregated["total_questions"], + "total_conversations": aggregated["total_conversations"], + "unique_sessions": aggregated["unique_sessions"] + }, + "performance": { + "avg_confidence": round(aggregated["avg_confidence"], 3), + "avg_retrieval_time_ms": aggregated["avg_retrieval_time_ms"], + "avg_generation_time_ms": aggregated["avg_generation_time_ms"], + "avg_total_time_ms": aggregated["avg_total_time_ms"] + }, + "engagement": { + "avg_messages_per_conversation": round(aggregated["avg_messages_per_conversation"], 2), + "avg_session_duration_seconds": aggregated["avg_session_duration_seconds"] + }, + "quality": { + "helpful_count": aggregated["helpful_count"], + "unhelpful_count": aggregated["unhelpful_count"], + "satisfaction_rate": round(aggregated["satisfaction_rate"], 3) + } + } + + logger.info(f"📊 Retrieved stats for {persona}: {aggregated['total_questions']} questions") + return response + + async def get_all_persona_stats( + self, + start_date: Optional[date] = None, + end_date: Optional[date] = None + ) -> List[Dict[str, Any]]: + """ + Get statistics for all personas. + + Args: + start_date: Start date (inclusive) + end_date: End date (inclusive) + + Returns: + List of stats dicts for each persona + """ + all_stats = [] + for persona in self.VALID_PERSONAS: + stats = await self.get_persona_stats(persona, start_date, end_date) + all_stats.append(stats) + + logger.info(f"📊 Retrieved stats for all {len(all_stats)} personas") + return all_stats + + async def get_persona_usage_breakdown(self) -> Dict[str, float]: + """ + Get percentage breakdown of persona usage (last 30 days). + + Returns: + Dict mapping persona name to usage percentage + """ + end_date = date.today() + start_date = end_date - timedelta(days=30) + + # Query conversation counts per persona + stmt = select( + Conversation.persona, + func.count(Conversation.id).label('count') + ).where( + Conversation.created_at >= datetime.combine(start_date, datetime.min.time()) + ).group_by(Conversation.persona) + + result = await self.db.execute(stmt) + counts = dict(result.all()) + + total = sum(counts.values()) + if total == 0: + logger.debug("📊 No usage data found") + return {p: 0.0 for p in self.VALID_PERSONAS} + + breakdown = {} + for persona in self.VALID_PERSONAS: + count = counts.get(persona, 0) + percentage = (count / total) * 100 + breakdown[persona] = round(percentage, 2) + + logger.info(f"📊 Usage breakdown: {breakdown}") + return breakdown + + # ======================================================================== + # ANALYTICS - AGGREGATION + # ======================================================================== + + async def aggregate_daily_stats(self, target_date: Optional[date] = None) -> None: + """ + Aggregate conversation/message data into persona_stats table. + + Called by Celery task daily at midnight to process previous day's data. + + Args: + target_date: Date to aggregate (defaults to yesterday) + """ + if not target_date: + target_date = date.today() - timedelta(days=1) + + logger.info(f"📊 Starting daily aggregation for {target_date}") + + start_datetime = datetime.combine(target_date, datetime.min.time()) + end_datetime = datetime.combine(target_date, datetime.max.time()) + + for persona in self.VALID_PERSONAS: + await self._aggregate_persona_day(persona, target_date, start_datetime, end_datetime) + + await self.db.commit() + logger.info(f"✅ Completed daily aggregation for {target_date}") + + async def _aggregate_persona_day( + self, + persona: str, + target_date: date, + start_datetime: datetime, + end_datetime: datetime + ) -> None: + """ + Aggregate stats for one persona for one day. + + Args: + persona: Persona name + target_date: Date to aggregate + start_datetime: Start of day + end_datetime: End of day + """ + assert isinstance(persona, str) and len(persona) > 0, "Persona must be a non-empty string" + assert start_datetime < end_datetime, "Start datetime must be before end datetime" + + # Get conversations for this persona on this day + conv_stmt = select(Conversation).where( + and_( + Conversation.persona == persona, + Conversation.created_at >= start_datetime, + Conversation.created_at <= end_datetime + ) + ).options(selectinload(Conversation.messages)) + + conv_result = await self.db.execute(conv_stmt) + conversations = conv_result.scalars().all() + + if not conversations: + logger.debug(f"No conversations for {persona} on {target_date}") + return + + # Calculate metrics + metrics = self._calculate_conversation_metrics(conversations) + + # Fetch feedback quality metrics + message_ids = [m.id for m in metrics["all_messages"]] + feedback_stmt = select(MessageFeedback).where( + and_( + MessageFeedback.message_id.in_(message_ids), + MessageFeedback.created_at >= start_datetime, + MessageFeedback.created_at <= end_datetime + ) + ) + feedback_result = await self.db.execute(feedback_stmt) + feedbacks = feedback_result.scalars().all() + + helpful_count = sum(1 for f in feedbacks if f.helpful) + unhelpful_count = sum(1 for f in feedbacks if not f.helpful) + total_feedback = len(feedbacks) + satisfaction_rate = helpful_count / total_feedback if total_feedback > 0 else 0.0 + + metrics.update({ + "helpful_count": helpful_count, + "unhelpful_count": unhelpful_count, + "satisfaction_rate": satisfaction_rate, + }) + + # Upsert persona_stats + await self._upsert_persona_stat(persona, target_date, metrics) + + def _calculate_conversation_metrics(self, conversations: list) -> dict: + """ + Calculate usage/performance/engagement metrics from conversations. + + Args: + conversations: List of Conversation ORM objects with messages loaded + + Returns: + Dict with total_questions, total_conversations, unique_sessions, + avg_confidence, avg_retrieval, avg_generation, avg_total, + messages_per_conv, avg_duration, and all_messages list. + """ + total_conversations = len(conversations) + unique_sessions = len(set(c.session_id for c in conversations)) + + all_messages = [] + for conv in conversations: + all_messages.extend([m for m in conv.messages if m.role == 'assistant']) + + total_questions = sum( + len([m for m in conv.messages if m.role == 'user']) + for conv in conversations + ) + + confidences = [m.confidence for m in all_messages if m.confidence is not None] + retrieval_times = [m.retrieval_time_ms for m in all_messages if m.retrieval_time_ms is not None] + generation_times = [m.generation_time_ms for m in all_messages if m.generation_time_ms is not None] + + avg_confidence = sum(confidences) / len(confidences) if confidences else 0.0 + avg_retrieval = sum(retrieval_times) // len(retrieval_times) if retrieval_times else 0 + avg_generation = sum(generation_times) // len(generation_times) if generation_times else 0 + + messages_per_conv = total_questions / total_conversations if total_conversations else 0.0 + + session_durations = [] + for conv in conversations: + if len(conv.messages) >= 2: + duration = (conv.messages[-1].created_at - conv.messages[0].created_at).total_seconds() + session_durations.append(duration) + avg_duration = int(sum(session_durations) / len(session_durations)) if session_durations else 0 + + return { + "total_questions": total_questions, + "total_conversations": total_conversations, + "unique_sessions": unique_sessions, + "avg_confidence": avg_confidence, + "avg_retrieval_time_ms": avg_retrieval, + "avg_generation_time_ms": avg_generation, + "avg_total_time_ms": avg_retrieval + avg_generation, + "avg_messages_per_conversation": messages_per_conv, + "avg_session_duration_seconds": avg_duration, + "all_messages": all_messages, + } + + async def _upsert_persona_stat( + self, persona: str, target_date: date, metrics: dict + ) -> None: + """ + Create or update a PersonaStat record for a persona on a given date. + + Args: + persona: Persona name + target_date: Date for the stats record + metrics: Dict with all computed metric fields + """ + existing_stmt = select(PersonaStat).where( + and_( + PersonaStat.persona == persona, + PersonaStat.date == target_date + ) + ) + existing_result = await self.db.execute(existing_stmt) + existing = existing_result.scalar_one_or_none() + + stat_fields = {k: v for k, v in metrics.items() if k != "all_messages"} + + if existing: + for key, value in stat_fields.items(): + setattr(existing, key, value) + existing.updated_at = datetime.utcnow() + logger.debug(f"Updated stats for {persona} on {target_date}") + else: + new_stat = PersonaStat( + persona=persona, + date=target_date, + **stat_fields + ) + self.db.add(new_stat) + logger.debug(f"Created stats for {persona} on {target_date}") + + async def update_interaction_metrics( + self, + conversation_id: int, + persona: str, + language: str + ) -> None: + """ + Update or create persona_interactions record for a conversation. + + Called by RAGService after each message exchange. + + Args: + conversation_id: ID of conversation + persona: Current persona + language: Current language + """ + assert isinstance(conversation_id, int) and conversation_id > 0, "Conversation ID must be a positive integer" + assert isinstance(persona, str) and len(persona) > 0, "Persona must be a non-empty string" + + # Check if interaction record exists + stmt = select(PersonaInteraction).where( + PersonaInteraction.conversation_id == conversation_id + ) + result = await self.db.execute(stmt) + interaction = result.scalar_one_or_none() + + # Get conversation with messages + conv_stmt = select(Conversation).where( + Conversation.id == conversation_id + ).options(selectinload(Conversation.messages)) + conv_result = await self.db.execute(conv_stmt) + conversation = conv_result.scalar_one_or_none() + + if not conversation: + logger.warning(f"❌ Conversation {conversation_id} not found") + return + + # Calculate metrics + user_messages = [m for m in conversation.messages if m.role == 'user'] + assistant_messages = [m for m in conversation.messages if m.role == 'assistant'] + + question_count = len(user_messages) + answer_count = len(assistant_messages) + + confidences = [m.confidence for m in assistant_messages if m.confidence is not None] + avg_confidence = sum(confidences) / len(confidences) if confidences else 0.0 + + response_times = [ + (m.retrieval_time_ms or 0) + (m.generation_time_ms or 0) + for m in assistant_messages + ] + total_response_time = sum(response_times) + + # Track persona switches + personas_used = [persona] if not interaction else interaction.personas_used or [persona] + if persona not in personas_used: + personas_used.append(persona) + persona_switches = len(personas_used) - 1 + + if interaction: + # Update existing + interaction.persona = persona + interaction.language = language + interaction.question_count = question_count + interaction.answer_count = answer_count + interaction.avg_confidence = avg_confidence + interaction.total_response_time_ms = total_response_time + interaction.personas_used = personas_used + interaction.persona_switches = persona_switches + interaction.updated_at = datetime.utcnow() + logger.debug(f"Updated interaction for conversation {conversation_id}") + else: + # Create new + new_interaction = PersonaInteraction( + conversation_id=conversation_id, + persona=persona, + language=language, + question_count=question_count, + answer_count=answer_count, + avg_confidence=avg_confidence, + total_response_time_ms=total_response_time, + personas_used=personas_used, + persona_switches=persona_switches + ) + self.db.add(new_interaction) + logger.debug(f"Created interaction for conversation {conversation_id}") + + await self.db.commit() + + # ======================================================================== + # MESSAGE FEEDBACK + # ======================================================================== + + async def add_message_feedback( + self, + message_id: int, + helpful: bool, + rating: Optional[int] = None, + comment: Optional[str] = None + ) -> Dict[str, Any]: + """ + Record user feedback on a message. + + Args: + message_id: ID of message + helpful: Whether message was helpful (👍/👎) + rating: Optional 1-5 star rating + comment: Optional text comment + + Returns: + Created feedback dict + + Raises: + MessageNotFoundError: If message doesn't exist + FeedbackAlreadyExistsError: If feedback already exists for this message + """ + # Verify message exists + msg_stmt = select(Message).where(Message.id == message_id) + msg_result = await self.db.execute(msg_stmt) + message = msg_result.scalar_one_or_none() + + if not message: + logger.warning(f"❌ Message {message_id} not found") + raise MessageNotFoundError(f"Message {message_id} not found") + + # Check for existing feedback + existing_stmt = select(MessageFeedback).where( + MessageFeedback.message_id == message_id + ) + existing_result = await self.db.execute(existing_stmt) + existing = existing_result.scalar_one_or_none() + + if existing: + logger.warning(f"❌ Feedback already exists for message {message_id}") + raise FeedbackAlreadyExistsError(f"Feedback already exists for message {message_id}") + + # Validate rating if provided + if rating is not None and (rating < 1 or rating > 5): + raise ValueError("Rating must be between 1 and 5") + + # Create feedback + feedback = MessageFeedback( + message_id=message_id, + helpful=helpful, + rating=rating, + comment=comment + ) + self.db.add(feedback) + await self.db.commit() + await self.db.refresh(feedback) + + logger.info(f"✅ Added feedback for message {message_id}: helpful={helpful}, rating={rating}") + + return { + "id": feedback.id, + "message_id": feedback.message_id, + "helpful": feedback.helpful, + "rating": feedback.rating, + "comment": feedback.comment, + "created_at": feedback.created_at.isoformat() + } + + async def get_message_feedback(self, message_id: int) -> Optional[Dict[str, Any]]: + """ + Get feedback for a specific message. + + Args: + message_id: ID of message + + Returns: + Feedback dict or None if no feedback exists + """ + stmt = select(MessageFeedback).where( + MessageFeedback.message_id == message_id + ) + result = await self.db.execute(stmt) + feedback = result.scalar_one_or_none() + + if not feedback: + return None + + return { + "id": feedback.id, + "message_id": feedback.message_id, + "helpful": feedback.helpful, + "rating": feedback.rating, + "comment": feedback.comment, + "created_at": feedback.created_at.isoformat() + } + + async def get_feedback_stats( + self, + persona: Optional[str] = None, + start_date: Optional[date] = None, + end_date: Optional[date] = None + ) -> Dict[str, Any]: + """ + Get aggregated feedback statistics. + + Args: + persona: Optional persona filter + start_date: Optional start date + end_date: Optional end date + + Returns: + Dict with feedback statistics + + Raises: + InvalidPersonaError: If persona is invalid + """ + if persona: + await self.validate_persona(persona) + + # Build query + query = select(MessageFeedback) + + filters = [] + if start_date: + filters.append(MessageFeedback.created_at >= datetime.combine(start_date, datetime.min.time())) + if end_date: + filters.append(MessageFeedback.created_at <= datetime.combine(end_date, datetime.max.time())) + + if persona: + # Join with messages and conversations to filter by persona + query = query.join(Message, MessageFeedback.message_id == Message.id)\ + .join(Conversation, Message.conversation_id == Conversation.id)\ + .where(Conversation.persona == persona) + + if filters: + query = query.where(and_(*filters)) + + result = await self.db.execute(query) + feedbacks = result.scalars().all() + + # Calculate stats + total_feedback = len(feedbacks) + helpful_count = sum(1 for f in feedbacks if f.helpful) + unhelpful_count = total_feedback - helpful_count + satisfaction_rate = helpful_count / total_feedback if total_feedback > 0 else 0.0 + + ratings = [f.rating for f in feedbacks if f.rating is not None] + avg_rating = sum(ratings) / len(ratings) if ratings else None + + # By persona breakdown + by_persona = {} + if not persona: + for p in self.VALID_PERSONAS: + p_feedbacks = await self._get_persona_feedbacks(p, start_date, end_date) + by_persona[p] = { + "total": len(p_feedbacks), + "helpful": sum(1 for f in p_feedbacks if f.helpful), + "unhelpful": len(p_feedbacks) - sum(1 for f in p_feedbacks if f.helpful) + } + + stats = { + "total_feedback": total_feedback, + "helpful_count": helpful_count, + "unhelpful_count": unhelpful_count, + "satisfaction_rate": round(satisfaction_rate, 3), + "avg_rating": round(avg_rating, 2) if avg_rating else None, + "by_persona": by_persona + } + + logger.info(f"📊 Feedback stats: {total_feedback} total, {satisfaction_rate:.1%} satisfaction") + return stats + + async def _get_persona_feedbacks( + self, + persona: str, + start_date: Optional[date], + end_date: Optional[date] + ) -> List[MessageFeedback]: + """Helper to get feedbacks for a specific persona.""" + query = select(MessageFeedback)\ + .join(Message, MessageFeedback.message_id == Message.id)\ + .join(Conversation, Message.conversation_id == Conversation.id)\ + .where(Conversation.persona == persona) + + filters = [] + if start_date: + filters.append(MessageFeedback.created_at >= datetime.combine(start_date, datetime.min.time())) + if end_date: + filters.append(MessageFeedback.created_at <= datetime.combine(end_date, datetime.max.time())) + + if filters: + query = query.where(and_(*filters)) + + result = await self.db.execute(query) + return result.scalars().all() + + # ======================================================================== + # ENGAGEMENT METRICS + # ======================================================================== + + async def get_engagement_metrics( + self, + persona: str, + days: int = 30 + ) -> Dict[str, Any]: + """ + Get engagement metrics for a persona over N days. + + Args: + persona: Persona name + days: Number of days to analyze (default 30) + + Returns: + Dict with engagement metrics + + Raises: + InvalidPersonaError: If persona is invalid + """ + await self.validate_persona(persona) + + start_date = date.today() - timedelta(days=days) + end_date = date.today() + + # Get conversations + start_datetime = datetime.combine(start_date, datetime.min.time()) + + conv_stmt = select(Conversation).where( + and_( + Conversation.persona == persona, + Conversation.created_at >= start_datetime + ) + ).options(selectinload(Conversation.messages)) + + conv_result = await self.db.execute(conv_stmt) + conversations = conv_result.scalars().all() + + if not conversations: + return { + "persona": persona, + "days": days, + "metrics": { + "total_conversations": 0, + "total_messages": 0, + "avg_messages_per_conversation": 0.0, + "avg_session_duration_seconds": 0, + "active_days": 0 + } + } + + # Calculate metrics + total_conversations = len(conversations) + total_messages = sum(len(c.messages) for c in conversations) + avg_messages = total_messages / total_conversations + + # Session durations + durations = [] + for conv in conversations: + if len(conv.messages) >= 2: + duration = (conv.messages[-1].created_at - conv.messages[0].created_at).total_seconds() + durations.append(duration) + avg_duration = int(sum(durations) / len(durations)) if durations else 0 + + # Active days (days with at least one conversation) + active_dates = set(c.created_at.date() for c in conversations) + active_days = len(active_dates) + + metrics = { + "persona": persona, + "days": days, + "metrics": { + "total_conversations": total_conversations, + "total_messages": total_messages, + "avg_messages_per_conversation": round(avg_messages, 2), + "avg_session_duration_seconds": avg_duration, + "active_days": active_days, + "conversations_per_active_day": round(total_conversations / active_days, 2) if active_days > 0 else 0.0 + } + } + + logger.info(f"📊 Engagement metrics for {persona}: {total_conversations} conversations over {days} days") + return metrics + + async def get_popular_questions( + self, + persona: str, + limit: int = 10 + ) -> List[Dict[str, Any]]: + """ + Get most frequently asked questions for a persona. + + Uses fuzzy matching to group similar questions. + + Args: + persona: Persona name + limit: Max number of questions to return + + Returns: + List of dicts with question and count + + Raises: + InvalidPersonaError: If persona is invalid + """ + await self.validate_persona(persona) + + # Get user messages for this persona (last 90 days) + start_date = datetime.now() - timedelta(days=90) + + stmt = select(Message.content, func.count(Message.id).label('count'))\ + .join(Conversation, Message.conversation_id == Conversation.id)\ + .where( + and_( + Conversation.persona == persona, + Message.role == 'user', + Message.created_at >= start_date + ) + )\ + .group_by(Message.content)\ + .order_by(desc('count'))\ + .limit(limit) + + result = await self.db.execute(stmt) + questions = result.all() + + popular = [ + { + "question": q.content, + "count": q.count + } + for q in questions + ] + + logger.info(f"📊 Found {len(popular)} popular questions for {persona}") + return popular + + # ======================================================================== + # COMPARISON & TRENDS + # ======================================================================== + + async def compare_personas( + self, + metric: str, + days: int = 30 + ) -> Dict[str, Any]: + """ + Compare personas across a specific metric. + + Args: + metric: Metric to compare (usage|confidence|satisfaction) + days: Number of days to analyze + + Returns: + Dict with comparison data for all personas + + Raises: + ValueError: If metric is invalid + """ + valid_metrics = ["usage", "confidence", "satisfaction"] + if metric not in valid_metrics: + raise ValueError(f"Invalid metric '{metric}'. Must be one of: {', '.join(valid_metrics)}") + + start_date = date.today() - timedelta(days=days) + end_date = date.today() + + comparison = { + "metric": metric, + "days": days, + "date_range": { + "start": start_date.isoformat(), + "end": end_date.isoformat() + }, + "personas": {} + } + + for persona in self.VALID_PERSONAS: + stats = await self.get_persona_stats(persona, start_date, end_date) + + if metric == "usage": + value = stats["usage"]["total_questions"] + elif metric == "confidence": + value = stats["performance"]["avg_confidence"] + elif metric == "satisfaction": + value = stats["quality"]["satisfaction_rate"] + else: + value = 0 + + comparison["personas"][persona] = { + "value": value, + "display_name": self.PERSONA_INFO[persona]["display_name"] + } + + logger.info(f"📊 Compared personas by {metric} over {days} days") + return comparison + + async def get_trends( + self, + persona: str, + metric: str, + days: int = 30 + ) -> List[Dict[str, Any]]: + """ + Get time-series trend data for a metric. + + Args: + persona: Persona name + metric: Metric to track (questions|confidence|satisfaction) + days: Number of days + + Returns: + List of dicts with date and value + + Raises: + InvalidPersonaError: If persona is invalid + ValueError: If metric is invalid + """ + await self.validate_persona(persona) + + valid_metrics = ["questions", "confidence", "satisfaction"] + if metric not in valid_metrics: + raise ValueError(f"Invalid metric '{metric}'. Must be one of: {', '.join(valid_metrics)}") + + start_date = date.today() - timedelta(days=days) + end_date = date.today() + + # Query persona_stats + stmt = select(PersonaStat).where( + and_( + PersonaStat.persona == persona, + PersonaStat.date >= start_date, + PersonaStat.date <= end_date + ) + ).order_by(PersonaStat.date) + + result = await self.db.execute(stmt) + stats = result.scalars().all() + + trends = [] + for stat in stats: + if metric == "questions": + value = stat.total_questions + elif metric == "confidence": + value = stat.avg_confidence + elif metric == "satisfaction": + value = stat.satisfaction_rate + else: + value = 0 + + trends.append({ + "date": stat.date.isoformat(), + "value": value + }) + + logger.info(f"📊 Retrieved {len(trends)} trend points for {persona} ({metric})") + return trends + + # ======================================================================== + # HEALTH CHECK + # ======================================================================== + + def health_check(self) -> Dict[str, Any]: + """ + Service health check. + + Returns: + Dict with service status + """ + return { + "service": "PersonaService", + "status": "healthy", + "personas_count": len(self.VALID_PERSONAS), + "personas": self.VALID_PERSONAS, + "timestamp": datetime.utcnow().isoformat() + } + + # ======================================================================== + # HELPER METHODS + # ======================================================================== + + def _empty_stats_response( + self, + persona: str, + start_date: date, + end_date: date + ) -> Dict[str, Any]: + """Generate empty stats response when no data exists.""" + return { + "persona": persona, + "date_range": { + "start": start_date.isoformat(), + "end": end_date.isoformat() + }, + "usage": { + "total_questions": 0, + "total_conversations": 0, + "unique_sessions": 0 + }, + "performance": { + "avg_confidence": 0.0, + "avg_retrieval_time_ms": 0, + "avg_generation_time_ms": 0, + "avg_total_time_ms": 0 + }, + "engagement": { + "avg_messages_per_conversation": 0.0, + "avg_session_duration_seconds": 0 + }, + "quality": { + "helpful_count": 0, + "unhelpful_count": 0, + "satisfaction_rate": 0.0 + } + } + + def _aggregate_stats(self, stats: List[PersonaStat]) -> Dict[str, Any]: + """Aggregate multiple PersonaStat records into summary metrics.""" + if not stats: + return { + "total_questions": 0, + "total_conversations": 0, + "unique_sessions": 0, + "avg_confidence": 0.0, + "avg_retrieval_time_ms": 0, + "avg_generation_time_ms": 0, + "avg_total_time_ms": 0, + "avg_messages_per_conversation": 0.0, + "avg_session_duration_seconds": 0, + "helpful_count": 0, + "unhelpful_count": 0, + "satisfaction_rate": 0.0 + } + + # Sum usage metrics + total_questions = sum(s.total_questions for s in stats) + total_conversations = sum(s.total_conversations for s in stats) + unique_sessions = sum(s.unique_sessions for s in stats) + + # Average performance metrics (weighted by question count) + total_weight = sum(s.total_questions for s in stats if s.total_questions > 0) + + if total_weight > 0: + avg_confidence = sum( + s.avg_confidence * s.total_questions for s in stats if s.total_questions > 0 + ) / total_weight + avg_retrieval = sum( + s.avg_retrieval_time_ms * s.total_questions for s in stats if s.total_questions > 0 + ) / total_weight + avg_generation = sum( + s.avg_generation_time_ms * s.total_questions for s in stats if s.total_questions > 0 + ) / total_weight + avg_total = sum( + s.avg_total_time_ms * s.total_questions for s in stats if s.total_questions > 0 + ) / total_weight + else: + avg_confidence = avg_retrieval = avg_generation = avg_total = 0.0 + + # Average engagement metrics + avg_messages = sum(s.avg_messages_per_conversation for s in stats) / len(stats) + avg_duration = sum(s.avg_session_duration_seconds for s in stats) // len(stats) + + # Sum quality metrics + helpful_count = sum(s.helpful_count for s in stats) + unhelpful_count = sum(s.unhelpful_count for s in stats) + total_feedback = helpful_count + unhelpful_count + satisfaction_rate = helpful_count / total_feedback if total_feedback > 0 else 0.0 + + return { + "total_questions": total_questions, + "total_conversations": total_conversations, + "unique_sessions": unique_sessions, + "avg_confidence": avg_confidence, + "avg_retrieval_time_ms": int(avg_retrieval), + "avg_generation_time_ms": int(avg_generation), + "avg_total_time_ms": int(avg_total), + "avg_messages_per_conversation": avg_messages, + "avg_session_duration_seconds": avg_duration, + "helpful_count": helpful_count, + "unhelpful_count": unhelpful_count, + "satisfaction_rate": satisfaction_rate + } diff --git a/app/services/prompts.py b/app/services/prompts.py new file mode 100644 index 0000000000000000000000000000000000000000..7a98f5c501aa657c197f56faaff4c7914711ca00 --- /dev/null +++ b/app/services/prompts.py @@ -0,0 +1,342 @@ +""" +Prompt templates for RAG system with persona adaptation. + +Each persona gets tailored system prompt for appropriate tone and complexity. +Provides utilities for building context strings and formatting conversation history. +""" + +from typing import List + + +# System prompts by language and persona +SYSTEM_PROMPTS = { + "fr": { + "citoyen": """Tu es un assistant juridique bienveillant qui aide les citoyens camerounais à comprendre leurs droits. + +Ton rôle: +- Expliquer les lois en termes simples et accessibles +- Éviter le jargon juridique complexe +- Donner des exemples concrets de la vie quotidienne +- Être empathique et rassurant + +Format de réponse (utilise le markdown): +- Utilise **titres en gras** pour structurer ta réponse +- Utilise des listes à puces pour les points clés +- TOUJOURS citer les sources: "Selon l'article X de la Loi Y..." +- Suggérer quand consulter un avocat si nécessaire + +Exemple de structure: +**Réponse directe** +[Explication claire] + +**En termes simples** +[Exemple concret] + +**Source légale** +[Citation de l'article]""", + + "avocat": """Tu es un assistant juridique expert pour avocats camerounais. + +Ton rôle: +- Fournir des analyses juridiques précises et nuancées +- Citer les articles de loi exacts avec références complètes +- Mentionner la jurisprudence pertinente si disponible +- Souligner les subtilités et cas limites + +Format de réponse (utilise le markdown): +- Utilise **titres en gras** pour structurer ta réponse +- Utilise des listes numérotées pour les étapes juridiques + +Exemple de structure: +**Principe juridique** +[Analyse] + +**Base légale** +[Articles et références] + +**Implications pratiques** +[Conseils pour le dossier]""", + + "entrepreneur": """Tu es un consultant juridique spécialisé en droit des affaires camerounais. + +Ton rôle: +- Expliquer les implications pratiques pour les entreprises +- Focus sur conformité, risques, et opportunités +- Langage professionnel mais accessible +- Conseils actionnables + +Format de réponse (utilise le markdown): +- Utilise **titres en gras** pour structurer ta réponse +- Utilise des listes à puces pour les obligations et risques + +Exemple de structure: +**Impact sur votre activité** +[Explication] + +**Obligations légales** +[Liste des obligations] + +**Recommandations** +[Conseils pratiques]""", + + "étudiant": """Tu es un professeur de droit patient qui aide les étudiants camerounais. + +Ton rôle: +- Expliquer les concepts juridiques de manière pédagogique +- Développer le raisonnement juridique étape par étape +- Fournir le contexte historique et les principes sous-jacents +- Encourager la réflexion critique + +Format de réponse (utilise le markdown): +- Utilise **titres en gras** pour structurer ta réponse +- Utilise des listes numérotées pour les étapes de raisonnement + +Exemple de structure: +**Définition** +[Concept juridique] + +**Principe fondamental** +[Explication pédagogique] + +**Application pratique** +[Cas d'école] + +**Références** +[Articles de loi]""" + }, + "en": { + "citoyen": """You are a helpful legal assistant helping Cameroonian citizens understand their rights. + +Your role: +- Explain laws in simple and accessible terms +- Avoid complex legal jargon +- Give concrete examples from everyday life +- Be empathetic and reassuring + +Response format (use markdown): +- Use **bold titles** to structure your response +- Use bullet points for key information +- ALWAYS cite sources: "According to Article X of Law Y..." +- Suggest when to consult a lawyer if necessary + +Example structure: +**Direct Answer** +[Clear explanation] + +**In Simple Terms** +[Concrete example] + +**Legal Source** +[Article citation]""", + + "avocat": """You are an expert legal assistant for Cameroonian lawyers. + +Your role: +- Provide precise and nuanced legal analysis +- Cite exact law articles with complete references +- Mention relevant case law if available +- Highlight subtleties and edge cases + +Response format (use markdown): +- Use **bold titles** to structure your response +- Use numbered lists for legal steps + +Example structure: +**Legal Principle** +[Analysis] + +**Legal Basis** +[Articles and references] + +**Practical Implications** +[Case recommendations]""", + + "entrepreneur": """You are a legal consultant specializing in Cameroonian business law. + +Your role: +- Explain practical implications for businesses +- Focus on compliance, risks, and opportunities +- Professional but accessible language +- Actionable advice + +Response format (use markdown): +- Use **bold titles** to structure your response +- Use bullet points for obligations and risks + +Example structure: +**Impact on Your Business** +[Explanation] + +**Legal Obligations** +[List of requirements] + +**Recommendations** +[Practical advice]""", + + "étudiant": """You are a patient law professor helping Cameroonian students. + +Your role: +- Explain legal concepts pedagogically +- Develop legal reasoning step by step +- Provide historical context and underlying principles +- Encourage critical thinking + +Response format (use markdown): +- Use **bold titles** to structure your response +- Use numbered lists for reasoning steps + +Example structure: +**Definition** +[Legal concept] + +**Fundamental Principle** +[Pedagogical explanation] + +**Practical Application** +[Case study] + +**References** +[Law articles]""" + } +} + + +def get_system_prompt(persona: str, language: str) -> str: + """Get system prompt for given persona and language.""" + lang = language if language in SYSTEM_PROMPTS else "fr" + persona_key = persona if persona in SYSTEM_PROMPTS[lang] else "citoyen" + + base_prompt = SYSTEM_PROMPTS[lang][persona_key] + + # Add explicit language instruction + if lang == "en": + language_instruction = "\n\nIMPORTANT: You MUST respond in ENGLISH regardless of the language of the legal documents in the context." + else: + language_instruction = "\n\nIMPORTANT: Tu DOIS répondre en FRANÇAIS quelle que soit la langue des documents juridiques dans le contexte." + + return base_prompt + language_instruction + + +# Context building template +CONTEXT_TEMPLATE = """Documents juridiques pertinents: + +{context_docs} + +Instructions: +- Base ta réponse UNIQUEMENT sur ces documents +- Cite TOUJOURS tes sources avec format: "Selon l'article X de [Référence Loi]" +- Si l'information n'est pas dans les documents, dis-le clairement +- Ne spécule pas, reste factuel +""" + + +def build_context_string(search_results: List) -> str: + """ + Format search results into context string. + + Args: + search_results: List of SearchResult from SearchService + + Returns: + Formatted context for prompt + """ + context_parts = [] + + for i, result in enumerate(search_results[:5], 1): # Top 5 + # Get highlights content + highlights_content = result.highlights.get('content', '') + if not highlights_content: + # Fallback to full content if no highlights - no truncation for better Gemini context + highlights_content = getattr(result, 'content', '') + + context_parts.append(f""" +Document {i}: {result.reference} - {result.title} +Pertinence: {result.relevance_score:.2f} +Catégorie: {result.category_name or 'Non spécifiée'} + +Contenu: +{highlights_content} + +Articles pertinents: +{format_matched_articles(result.matched_articles[:3] if hasattr(result, 'matched_articles') else [])} +""") + + return "\n---\n".join(context_parts) + + +def format_matched_articles(articles: List) -> str: + """ + Format matched articles for context. + + Args: + articles: List of matched articles + + Returns: + Formatted articles string + """ + if not articles: + return "Aucun article spécifique identifié" + + formatted = [] + for a in articles: + article_text = f"- Article {a.number}" + if hasattr(a, 'snippet') and a.snippet: + article_text += f": {a.snippet[:200]}..." + formatted.append(article_text) + + return "\n".join(formatted) + + +# Conversation history template +HISTORY_TEMPLATE = """Historique de conversation: + +{history} +""" + + +def format_conversation_history(messages: List) -> str: + """ + Format last N messages for context. + + Args: + messages: List of Message objects (chronologically ordered) + + Returns: + Formatted history string + """ + if not messages: + return "Pas d'historique (première question)" + + history_parts = [] + for msg in messages[-5:]: # Last 5 messages + role = "👤 Utilisateur" if msg.role == "user" else "🤖 Assistant" + content = msg.content[:300] + if len(msg.content) > 300: + content += "..." + history_parts.append(f"{role}: {content}") + + return "\n\n".join(history_parts) + + +# No results fallback messages +NO_RESULTS_MESSAGE = { + "fr": """Je n'ai pas trouvé d'information pertinente dans la base de données juridique camerounaise pour répondre à votre question. + +Suggestions: +- Reformulez votre question de manière plus précise +- Vérifiez l'orthographe des termes juridiques +- Essayez des termes alternatifs +- Consultez un avocat pour des conseils personnalisés + +Puis-je vous aider autrement?""", + + "en": """I couldn't find relevant information in the Cameroonian legal database to answer your question. + +Suggestions: +- Rephrase your question more precisely +- Check spelling of legal terms +- Try alternative terms +- Consult a lawyer for personalized advice + +How else can I help you?""" +} diff --git a/app/services/rag_service.py b/app/services/rag_service.py new file mode 100644 index 0000000000000000000000000000000000000000..d8d8748a6fb1ebd2d3454565dac043e8b13c8e44 --- /dev/null +++ b/app/services/rag_service.py @@ -0,0 +1,1274 @@ +""" +RAGService - Retrieval-Augmented Generation service for legal Q&A. + +Orchestrates the complete RAG pipeline: +1. Document retrieval (SearchService) +2. Conversation history loading +3. Prompt construction (persona-adapted) +4. Answer generation (GeminiService) - TODO: Implement +5. Citation extraction +6. Persistence (Database) +""" + +import json +import logging +import re +import time +import uuid +from datetime import datetime +from typing import AsyncIterator, List, Optional, Tuple + +from sqlalchemy import desc, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import joinedload + +from app.models.conversation import Conversation, Message +from app.schemas.rag import Citation, RAGRequest, RAGResponse +from app.schemas.search import SearchFilters, SearchRequest +from app.services.gemini_service import get_gemini_service, GeminiServiceError +from app.services.prompts import ( + CONTEXT_TEMPLATE, + NO_RESULTS_MESSAGE, + SYSTEM_PROMPTS, + build_context_string, + format_conversation_history, + get_system_prompt, +) +from app.services.search_service import SearchService + +logger = logging.getLogger(__name__) + + +class RAGServiceError(Exception): + """Base exception for RAG service.""" + pass + + +class RAGService: + """ + RAG (Retrieval-Augmented Generation) service for legal Q&A. + + Orchestrates: + 1. Document retrieval (SearchService) + 2. Conversation history loading + 3. Prompt construction (persona-adapted) + 4. Answer generation (GeminiService) - TODO + 5. Citation extraction + 6. Persistence (Database) + + Stateful service - requires database session per request. + """ + + MAX_HISTORY_MESSAGES = 5 + TOP_K_DOCUMENTS = 5 + CITATION_REGEX = r"[Aa]rticle\s+(\d+[a-z]?)\s+(?:de\s+)?(?:la\s+)?([A-Z][^,\.]+)" + + # French and English stopwords to filter out for better search + STOPWORDS = { + # French + "le", "la", "les", "un", "une", "des", "de", "du", "d", "l", + "ce", "cette", "ces", "mon", "ma", "mes", "ton", "ta", "tes", + "son", "sa", "ses", "notre", "nos", "votre", "vos", "leur", "leurs", + "je", "tu", "il", "elle", "on", "nous", "vous", "ils", "elles", + "me", "te", "se", "lui", "y", "en", "qui", "que", "quoi", "dont", "où", + "et", "ou", "mais", "donc", "or", "ni", "car", "si", "quand", "comme", + "à", "au", "aux", "avec", "sans", "sous", "sur", "dans", "par", "pour", + "est", "sont", "être", "avoir", "fait", "faire", "dit", "dire", + "c", "qu", "n", "s", "m", "t", + "moi", "toi", "soi", "eux", + "ne", "pas", "plus", "moins", "très", "bien", "mal", + "tout", "tous", "toute", "toutes", "autre", "autres", + "quel", "quelle", "quels", "quelles", + "comment", "pourquoi", "quand", "combien", + "explique", "expliquer", "donne", "donner", "dis", "parle", "parler", + # English + "the", "a", "an", "is", "are", "was", "were", "be", "been", "being", + "have", "has", "had", "do", "does", "did", "will", "would", "could", + "should", "may", "might", "must", "shall", "can", + "i", "you", "he", "she", "it", "we", "they", "me", "him", "her", "us", "them", + "my", "your", "his", "its", "our", "their", + "this", "that", "these", "those", + "what", "which", "who", "whom", "whose", "where", "when", "why", "how", + "and", "or", "but", "if", "then", "so", "because", + "of", "to", "in", "on", "at", "by", "for", "with", "about", "from", + "explain", "tell", "give", "show", "describe", + } + + @staticmethod + def extract_keywords(question: str) -> str: + """ + Extract important keywords from a natural language question. + Removes stopwords to improve Meilisearch results. + """ + import re + # Normalize: lowercase and remove punctuation except apostrophes + words = re.findall(r"[a-zA-ZàâäéèêëïîôùûüçÀÂÄÉÈÊËÏÎÔÙÛÜÇ]+", question.lower()) + + # Filter stopwords and short words + keywords = [w for w in words if w not in RAGService.STOPWORDS and len(w) > 2] + + # Return at least 2 keywords, or original if too few keywords found + if len(keywords) < 2: + return question + + return " ".join(keywords[:10]) # Limit to 10 keywords + + def __init__(self, db: AsyncSession): + """ + Initialize RAG service. + + Args: + db: Async database session + """ + self.db = db + self.llm = get_gemini_service() + self.search_service = SearchService(db) + + async def ask(self, request: RAGRequest) -> RAGResponse: + """ + Main entry point for RAG question answering. + + Pipeline: load history → retrieve context → generate answer → save. + + Args: + request: RAGRequest with question, persona, etc. + + Returns: + RAGResponse with answer and citations + """ + assert request and request.question, "RAGRequest with question is required" + start_time = time.time() + + logger.info( + f"🔍 RAG request: persona={request.persona}, " + f"question=\"{request.question[:50]}...\"" + ) + + try: + # 1. Load conversation history + conversation, history = await self._load_or_create_conversation( + request.session_id, request.persona, request.language + ) + + # 2. Retrieve and merge document context + retrieval_start = time.time() + search_results, early_response = await self._retrieve_and_merge_context( + request, conversation + ) + retrieval_time_ms = int((time.time() - retrieval_start) * 1000) + + # Handle early exit (article not found in specific document) + if early_response: + early_response.retrieval_time_ms = retrieval_time_ms + early_response.total_time_ms = retrieval_time_ms + return early_response + + # 3. Fallback if no results + if not search_results: + search_results = await self._fallback_search(request.question, history) + if not search_results: + logger.warning("⚠️ No results found") + return await self._handle_no_results(request, retrieval_time_ms) + + # 4. Generate answer, extract citations, save + return await self._generate_and_save_response( + request, conversation, history, search_results, + retrieval_time_ms, start_time + ) + + except Exception as e: + logger.error(f"❌ RAG error: {e}", exc_info=True) + raise RAGServiceError(f"Erreur lors du traitement: {str(e)}") + + async def _retrieve_and_merge_context(self, request: RAGRequest, conversation): + """ + Retrieve documents: priority doc first, then general search, merged. + + Returns: + Tuple of (search_results, early_response_or_None) + """ + priority_results = [] + if request.law_id: + priority_results = await self._get_priority_document_context( + request.law_id, request.question + ) + logger.info(f"📌 Priority doc {request.law_id}: {len(priority_results)} results") + + # General search + search_results = await self._retrieve_context(request.question, request.language) + + # Check if article was not found in priority document + if priority_results and hasattr(priority_results[0], 'article_not_found') and priority_results[0].article_not_found: + result = priority_results[0] + message = ( + f"Ce document ne contient pas d'article {result.requested_article}. " + f"Le document \"{result.title}\" contient {result.total_articles} article(s). " + ) + if result.total_articles > 0: + message += f"Veuillez demander un article entre 1 et {result.total_articles}." + else: + message += "Ce document ne contient aucun article extrait." + + logger.info(f"📌 Article not found response: {message}") + return [], RAGResponse( + answer=message, confidence=1.0, sources=[], + session_id=conversation.session_id, + retrieval_time_ms=0, generation_time_ms=0, total_time_ms=0, + persona=request.persona + ) + + # Merge: priority first, then general (deduplicated) + if priority_results: + seen_ids = {r.law_id for r in priority_results} + filtered = [r for r in search_results if r.law_id not in seen_ids] + search_results = priority_results + filtered[:3] + logger.info(f"📚 Merged: {len(priority_results)} priority + {len(filtered[:3])} general") + + return search_results, None + + async def _generate_and_save_response( + self, request, conversation, history, search_results, + retrieval_time_ms, start_time + ): + """Build prompt, generate LLM answer, extract citations, save.""" + prompt = self._build_prompt( + question=request.question, search_results=search_results, + history=history, persona=request.persona + ) + + generation_start = time.time() + system_prompt = get_system_prompt(request.persona, request.language) + + if self.llm is None: + raise RAGServiceError("LLM service not configured. Please set up Gemini API.") + + llm_response = await self.llm.generate( + prompt=prompt, system=system_prompt, temperature=0.7, max_tokens=1000 + ) + generation_time_ms = int((time.time() - generation_start) * 1000) + answer = llm_response["response"] + + # Extract citations + citations = self._extract_citations(answer, search_results) + if not citations and search_results: + citations = self._create_sources_from_results(search_results, request.question) + + confidence = self._calculate_confidence(answer, citations, search_results) + + await self._save_interaction( + conversation=conversation, question=request.question, answer=answer, + citations=citations, confidence=confidence, + retrieval_time_ms=retrieval_time_ms, generation_time_ms=generation_time_ms + ) + + total_time_ms = int((time.time() - start_time) * 1000) + logger.info( + f"✅ RAG complete: {total_time_ms}ms " + f"(retrieval={retrieval_time_ms}ms, generation={generation_time_ms}ms)" + ) + + return RAGResponse( + answer=answer, confidence=confidence, sources=citations, + session_id=conversation.session_id, + retrieval_time_ms=retrieval_time_ms, + generation_time_ms=generation_time_ms, + total_time_ms=total_time_ms, persona=request.persona + ) + + async def ask_stream( + self, + request: RAGRequest + ) -> AsyncIterator[str]: + """ + Streaming version of ask() for better UX. + + Yields answer chunks as they're generated. + Final chunk includes sources and confidence. + + Yields: + JSON strings for SSE: {"chunk": "...", "done": false} + """ + assert request is not None, "RAGRequest must not be None" + assert isinstance(request.question, str) and len(request.question) > 0, "Question must be non-empty" + + start_time = time.time() + + try: + # Retrieval (same as ask) + retrieval_start = time.time() + search_results = await self._retrieve_context( + request.question, + request.language + ) + retrieval_time_ms = int((time.time() - retrieval_start) * 1000) + + if not search_results: + yield json.dumps({ + "chunk": NO_RESULTS_MESSAGE[request.language], + "done": True, + "sources": [], + "confidence": 0.0 + }) + return + + # Load history & build prompt + conversation, history = await self._load_or_create_conversation( + request.session_id, + request.persona, + request.language + ) + + prompt = self._build_prompt( + request.question, + search_results, + history, + request.persona + ) + + # Stream generation + generation_start = time.time() + system_prompt = SYSTEM_PROMPTS[request.persona] + + # TODO: Implement Gemini streaming + if self.llm is None: + yield json.dumps({"chunk": "LLM service not configured. Please set up Gemini API.", "done": True, "error": True}) + return + + answer_parts = [] + async for chunk in self.llm.generate_stream( + prompt=prompt, + system=system_prompt, + temperature=0.7, + max_tokens=1000 + ): + answer_parts.append(chunk) + yield json.dumps({"chunk": chunk, "done": False}) + + generation_time_ms = int((time.time() - generation_start) * 1000) + + # Process complete answer + full_answer = "".join(answer_parts) + citations = self._extract_citations(full_answer, search_results) + confidence = self._calculate_confidence( + full_answer, citations, search_results + ) + + # Save to DB + await self._save_interaction( + conversation, request.question, full_answer, + citations, confidence, + retrieval_time_ms, generation_time_ms + ) + + # Final chunk with metadata + yield json.dumps({ + "chunk": "", + "done": True, + "sources": [c.model_dump() for c in citations], + "confidence": confidence, + "session_id": conversation.session_id + }) + + except Exception as e: + logger.error(f"❌ Streaming error: {e}") + yield json.dumps({ + "chunk": "", + "done": True, + "error": str(e) + }) + + async def _retrieve_context( + self, + question: str, + language: str + ) -> List: + """ + Retrieve relevant documents using SearchService. + + Uses text search for reliability (hybrid requires embeddings). + Searches all languages to maximize results. + Extracts keywords from natural language questions for better search. + """ + # Extract keywords from question for better Meilisearch results + search_query = self.extract_keywords(question) + logger.info(f"🔍 RAG Search: original='{question[:50]}...' keywords='{search_query}'") + + search_response = await self.search_service.search( + SearchRequest( + query=search_query, + mode="text", # Use text search (hybrid requires embeddings) + filters=SearchFilters(status="published"), # No language filter + limit=self.TOP_K_DOCUMENTS + ) + ) + + logger.info( + f"📚 Retrieved {len(search_response.results)} documents " + f"in {search_response.search_time_ms}ms" + ) + + # Debug: Log first result if any + if search_response.results: + first = search_response.results[0] + logger.info(f"📄 First result: {first.reference} - {first.title}") + else: + logger.warning(f"⚠️ No search results for query: {question[:100]}") + + return search_response.results + + async def _get_priority_document_context(self, law_id: int, question: str) -> List: + """ + Fetch context from the active document being viewed. + + When a user is viewing a specific document and asks a question, + prioritize content from that document to provide focused answers. + + If a specific article is requested but not found, returns a special + result indicating the article doesn't exist in this document. + + Args: + law_id: ID of the law being viewed + question: User's question (used to find relevant articles) + + Returns: + List of search-like results from the priority document + May include special 'article_not_found' marker + """ + assert isinstance(law_id, int) and law_id > 0, "law_id must be a positive integer" + assert isinstance(question, str) and len(question) > 0, "question must be a non-empty string" + + from app.models.law import Law, Article + + try: + # Get the law with its articles + query = select(Law).where(Law.id == law_id) + result = await self.db.execute(query) + law = result.scalar_one_or_none() + + if not law: + logger.warning(f"⚠️ Priority law_id {law_id} not found") + return [] + + # Get articles for this law + article_query = select(Article).where(Article.law_id == law_id).order_by(Article.order) + article_result = await self.db.execute(article_query) + articles = article_result.scalars().all() + total_articles = len(articles) + + # Check if question asks about a specific article + article_num = self._extract_article_number(question) + + if article_num: + logger.info(f"📌 Looking for article {article_num} in doc (has {total_articles} articles)") + + # If another document is mentioned, let general search handle it + if self._references_other_document(question, law.title): + return [] + + # Look for the article in the current document + if articles: + for article in articles: + if article.number and ( + str(article.number) == str(article_num) or + article_num.lower() in str(article.number).lower() + ): + logger.info(f"✅ Found article {article.number} in priority doc") + return [self._build_priority_result( + law, article.content or "", total_articles + )] + + # Article not found in this document! + logger.info(f"❌ Article {article_num} NOT found in doc (has {total_articles} articles)") + return [self._build_priority_result( + law, "", total_articles, + article_not_found=True, requested_article=article_num + )] + + # No specific article requested - use law content and first few articles + combined_content = law.content or "" + if articles: + for article in articles[:3]: + if article.content: + combined_content += f"\n\nArticle {article.number}:\n{article.content}" + + if combined_content: + return [self._build_priority_result(law, combined_content, total_articles)] + + except Exception as e: + logger.error(f"⚠️ Error fetching priority document: {e}") + + return [] + + def _references_other_document(self, question: str, current_title: str) -> bool: + """ + Check if the question mentions a law document different from the current one. + + Args: + question: User's question + current_title: Title of the currently viewed law + + Returns: + True if another document is explicitly mentioned + """ + import re + other_doc_patterns = [ + r'constitution', r'code\s+civil', r'code\s+p[ée]nal', + r'loi\s+ohada', r'charte', r'décret', + ] + question_lower = question.lower() + for pattern in other_doc_patterns: + if re.search(pattern, question_lower) and pattern not in current_title.lower(): + logger.info(f"📌 Another document mentioned, skipping priority") + return True + return False + + def _build_priority_result( + self, law, content: str, total_articles: int, + article_not_found: bool = False, requested_article: str = "" + ): + """ + Build a PriorityResult dataclass instance from law data. + + Args: + law: Law ORM object + content: Article or combined content text + total_articles: Total number of articles in this law + article_not_found: Whether the requested article was missing + requested_article: The article number that was requested + + Returns: + PriorityResult dataclass instance + """ + from dataclasses import dataclass + + @dataclass + class PriorityResult: + law_id: int + title: str + reference: str + content: str + source_language: str + category_name: str = "Document actif" + relevance_score: float = 1.0 + highlights: dict = None + matched_articles: list = None + article_not_found: bool = False + requested_article: str = "" + total_articles: int = 0 + + def __post_init__(self): + self.highlights = {"content": self.content if self.content else ""} + self.matched_articles = [] + + return PriorityResult( + law_id=law.id, + title=law.title, + reference=law.reference or "", + content=content, + source_language=law.language or "fr", + total_articles=total_articles, + article_not_found=article_not_found, + requested_article=requested_article, + ) + + async def _load_or_create_conversation( + self, + session_id: Optional[str], + persona: str, + language: str + ) -> Tuple[Conversation, List[Message]]: + """ + Load existing conversation or create new one. + + Returns: + (conversation, last_N_messages) + """ + if session_id: + # Try to load existing + stmt = ( + select(Conversation) + .where(Conversation.session_id == session_id) + .options(joinedload(Conversation.messages)) + ) + result = await self.db.execute(stmt) + conversation = result.scalar_one_or_none() + + if conversation: + # Get last N messages + msg_stmt = ( + select(Message) + .where(Message.conversation_id == conversation.id) + .order_by(desc(Message.created_at)) + .limit(self.MAX_HISTORY_MESSAGES) + ) + msg_result = await self.db.execute(msg_stmt) + messages = list(msg_result.scalars().all()) + + logger.info(f"📖 Loaded conversation: {len(messages)} messages") + return conversation, list(reversed(messages)) + + # Create new conversation + conversation = Conversation( + session_id=session_id or str(uuid.uuid4()), + persona=persona, + language=language + ) + self.db.add(conversation) + await self.db.flush() + + logger.info(f"🆕 New conversation: {conversation.session_id}") + return conversation, [] + + async def _fallback_search(self, question: str, history: List = None) -> List: + """ + Multilingual fallback search when Meilisearch returns no results. + + Logic: + 1. Detect prompt language (FR/EN) + 2. Extract document context from conversation history + 3. Search in same-language documents first + 4. If not found, search in other-language documents + 5. Translation handled by Gemini in response generation + """ + assert isinstance(question, str) and len(question) > 0, "question must be a non-empty string" + assert history is None or isinstance(history, list), "history must be a list or None" + + import re + + # Detect prompt language based on keywords + prompt_lang = self._detect_language(question) + logger.info(f"🌐 Detected prompt language: {prompt_lang}") + + # Extract article/section number from question + article_num = self._extract_article_number(question) + if not article_num: + return [] + + # Extract document context from conversation history + doc_context = self._extract_document_context(question, history) + logger.info(f"📌 Multilingual search: article {article_num}, context: {doc_context or 'none'}") + + try: + from sqlalchemy import text, create_engine + from app.core.config import settings + from dataclasses import dataclass + + engine = create_engine(settings.DATABASE_URL.replace('+asyncpg', '')) + + @dataclass + class FallbackResult: + law_id: int + title: str + reference: str + content: str + source_language: str + category_name: str = "Loi" + relevance_score: float = 1.0 + highlights: dict = None + matched_articles: list = None + + def __post_init__(self): + self.highlights = {"content": self.content} + self.matched_articles = [] + + with engine.connect() as conn: + # Define search order based on prompt language + if prompt_lang == "fr": + search_order = [("fr", "Article"), ("en", "Section")] + else: + search_order = [("en", "Section"), ("fr", "Article")] + + # Use document context or default to constitution + search_pattern = f"%{doc_context}%" if doc_context else "%constitution%" + + for lang, term_type in search_order: + # Search for laws in this language + sql = text(""" + SELECT l.id, l.title, l.reference, l.language, + a.number, a.content as article_content + FROM laws l + LEFT JOIN articles a ON a.law_id = l.id AND a.number = :article_num + WHERE (l.language = :lang OR l.language IS NULL) + AND (LOWER(l.title) LIKE :pattern + OR LOWER(l.reference) LIKE :pattern) + LIMIT 1 + """) + + result = conn.execute(sql, { + "lang": lang, + "article_num": article_num, + "pattern": search_pattern + }) + row = result.fetchone() + + if row and row[5]: # Found law with article content + law_id, title, reference, law_lang, number, article_content = row + logger.info(f"📄 Found in {lang.upper()}: {title} - Article/Section {number}") + + return [FallbackResult( + law_id=law_id, + title=title, + reference=reference, + content=article_content, + source_language=lang or prompt_lang + )] + elif row: # Found law but no article - try direct article search + law_id = row[0] + # Try ILIKE pattern for article number + sql2 = text(""" + SELECT number, content FROM articles + WHERE law_id = :law_id + AND (number = :num OR number ILIKE :pattern) + LIMIT 1 + """) + result2 = conn.execute(sql2, { + "law_id": law_id, + "num": article_num, + "pattern": f"%{article_num}%" + }) + row2 = result2.fetchone() + if row2: + logger.info(f"📄 Found article {row2[0]} in {lang.upper()}: {row[1]}") + return [FallbackResult( + law_id=law_id, + title=row[1], + reference=row[2], + content=row2[1], + source_language=lang or prompt_lang + )] + + logger.warning(f"⚠️ Article/Section {article_num} not found in any language") + + except Exception as e: + logger.warning(f"⚠️ Multilingual search failed: {e}") + + return [] + + def _detect_language(self, text: str) -> str: + """Detect if text is primarily French or English.""" + french_words = ["de", "la", "le", "les", "du", "des", "au", "aux", "un", "une", + "et", "ou", "pour", "dans", "sur", "avec", "moi", "explique"] + english_words = ["the", "of", "and", "or", "for", "in", "on", "with", "me", + "explain", "what", "how", "is", "are", "does", "section"] + + text_lower = text.lower() + fr_count = sum(1 for w in french_words if f" {w} " in f" {text_lower} ") + en_count = sum(1 for w in english_words if f" {w} " in f" {text_lower} ") + + return "fr" if fr_count >= en_count else "en" + + def _extract_article_number(self, question: str) -> str: + """Extract article/section number from question.""" + import re + + patterns = [ + r'article\s+(\d+|premier|première)', + r'section\s+(\d+|first)', + r"l'article\s+(\d+)", + r'art\.?\s+(\d+)', + ] + + for pattern in patterns: + match = re.search(pattern, question, re.IGNORECASE) + if match: + num = match.group(1) + # Normalize + if num.lower() in ["premier", "première", "first"]: + return "1" + return num + return None + + def _extract_document_context(self, question: str, history: List = None) -> str: + """ + Extract document context from current question or conversation history. + Used for follow-up questions like "explique l'article 8" after discussing constitution. + """ + import re + + # Document patterns to look for (FR and EN) + doc_patterns = [ + (r'constitution', 'constitution'), + (r'code\s+civil', 'code civil'), + (r'code\s+p[ée]nal', 'code penal'), + (r'code\s+du\s+travail', 'code du travail'), + (r'code\s+de\s+commerce', 'code de commerce'), + (r'loi\s+ohada', 'loi ohada'), + (r'civil\s+code', 'code civil'), + (r'penal\s+code', 'code penal'), + (r'labor\s+code', 'code du travail'), + ] + + # First check current question + question_lower = question.lower() + for pattern, doc_name in doc_patterns: + if re.search(pattern, question_lower, re.IGNORECASE): + logger.info(f"📚 Document context from question: {doc_name}") + return doc_name + + # If not in question, check conversation history + if history: + for msg in reversed(history): # Start from most recent + if hasattr(msg, 'content'): + content_lower = msg.content.lower() + for pattern, doc_name in doc_patterns: + if re.search(pattern, content_lower, re.IGNORECASE): + logger.info(f"📚 Document context from history: {doc_name}") + return doc_name + + # Also check sources in messages + if hasattr(msg, 'sources') and msg.sources: + try: + import json + sources = msg.sources if isinstance(msg.sources, list) else json.loads(msg.sources) + for source in sources: + if isinstance(source, dict): + title = source.get('law_title', '').lower() + for pattern, doc_name in doc_patterns: + if re.search(pattern, title, re.IGNORECASE): + logger.info(f"📚 Document context from sources: {doc_name}") + return doc_name + except: + pass + + return None + + def _build_prompt( + self, + question: str, + search_results: List, + history: List[Message], + persona: str + ) -> str: + """ + Build complete prompt with context and history. + + Structure: + 1. Context from search results + 2. Specific article content if question mentions an article + 3. Conversation history + 4. User question + """ + # Build context + context_str = build_context_string(search_results) + context_section = CONTEXT_TEMPLATE.format(context_docs=context_str) + + # Check if question mentions a specific article and get its content + article_section = self._get_article_content_for_prompt(question, search_results) + + # Build history + history_str = format_conversation_history(history) if history else "" + + # Combine + parts = [context_section] + + if article_section: + parts.append(f"\n{article_section}\n") + + if history_str: + parts.append(f"\n{history_str}\n") + + parts.append(f"\nQuestion actuelle de l'utilisateur:\n{question}") + + return "\n".join(parts) + + def _get_article_content_for_prompt(self, question: str, search_results: List) -> str: + """ + Detect if question asks about a specific article/section and fetch its content. + Supports both French and English questions. + Full article content is sent to ensure complete context for LLM. + """ + assert isinstance(question, str) and len(question) > 0, "question must be a non-empty string" + assert isinstance(search_results, list), "search_results must be a list" + + # Use shared method for article number extraction + article_num = self._extract_article_number(question) + if not article_num: + return "" + + prompt_lang = self._detect_language(question) + logger.info(f"📑 Detected article/section {article_num} in {prompt_lang.upper()} question") + + # Get FULL article content from PostgreSQL (no truncation) + try: + from sqlalchemy import create_engine, text + from app.core.config import settings + + # FIRST: Check if FallbackResult already has the article content + for result in search_results[:3]: + result_content = getattr(result, 'content', None) + # If result has content and appears to be article content (not truncated law content) + if result_content and len(result_content) < 5000: + source_lang = getattr(result, 'source_language', None) + translation_note = "" + if source_lang and source_lang != prompt_lang: + if prompt_lang == "fr": + translation_note = "\n\n⚠️ Ce contenu est extrait de la version anglaise. Merci de traduire en français dans ta réponse." + else: + translation_note = "\n\n⚠️ This content is from the French version. Please translate to English in your response." + + logger.info(f"📄 Using article content from FallbackResult ({len(result_content)} chars)") + return f""" +=== ARTICLE/SECTION SPÉCIFIQUE DEMANDÉ === +Document: {result.title} ({getattr(result, 'reference', '')}) +Article/Section {article_num}: + +CONTENU COMPLET: +{result_content}{translation_note} +========================================== +""" + + # SECOND: Try to find in database + engine = create_engine(settings.DATABASE_URL.replace('+asyncpg', '')) + with engine.connect() as conn: + for result in search_results[:3]: + sql = text(""" + SELECT number, title, content FROM articles + WHERE law_id = :law_id AND (number = :number OR number ILIKE :pattern) + LIMIT 1 + """) + res = conn.execute(sql, { + "law_id": result.law_id, + "number": article_num, + "pattern": f"%{article_num}%" + }) + row = res.fetchone() + if row: + number, title, content = row + source_lang = getattr(result, 'source_language', None) + translation_note = "" + if source_lang and source_lang != prompt_lang: + if prompt_lang == "fr": + translation_note = "\n\n⚠️ Ce contenu est extrait de la version anglaise. Merci de traduire en français dans ta réponse." + else: + translation_note = "\n\n⚠️ This content is from the French version. Please translate to English in your response." + + logger.info(f"📄 Found article/section {number} in DB ({len(content)} chars)") + return f""" +=== ARTICLE/SECTION SPÉCIFIQUE DEMANDÉ === +Document: {result.title} ({result.reference}) +Article/Section {number}: {title or ''} + +CONTENU COMPLET: +{content}{translation_note} +========================================== +""" + except Exception as e: + logger.warning(f"⚠️ Failed to get article for prompt: {e}") + + return "" + + def _extract_citations( + self, + answer: str, + search_results: List + ) -> List[Citation]: + """ + Extract and validate citations from answer. + + Looks for patterns like: + - "Article 161 de la Loi OHADA" + - "article 5 du Code civil" + + Validates against search_results to ensure cited docs exist. + """ + citations = [] + seen = set() # Dedup + + # Regex extraction + matches = re.finditer(self.CITATION_REGEX, answer, re.IGNORECASE) + + for match in matches: + article_num = match.group(1) + law_mention = match.group(2).strip() + + # Find matching document in search results + for result in search_results: + # Match by law title or reference + if (law_mention.lower() in result.title.lower() or + law_mention.lower() in result.reference.lower()): + + key = (result.law_id, article_num) + if key not in seen: + # Extract relevant excerpt + excerpt = self._find_article_excerpt( + result, article_num + ) + + citations.append(Citation( + law_id=result.law_id, + law_reference=result.reference, + law_title=result.title, + article_number=article_num, + excerpt=excerpt, + relevance_score=result.relevance_score + )) + seen.add(key) + break + + logger.info(f"📎 Extracted {len(citations)} citations") + return citations + + def _create_sources_from_results(self, search_results: List, question: str) -> List[Citation]: + """ + Create source citations from search results with intelligent filtering. + 1. Identifies relevant laws by matching question words to law titles. + 2. Prioritizes laws matching the question's language. + 3. Falls back to Top 1 result if no specific law is identified. + """ + sources = [] + article_num = self._extract_article_number(question) + question_lower = question.lower() + + # 1. candidate selection + candidates = [] + + # Check for title matches + matched_results = [] + for res in search_results[:3]: + # Extract significant words from title (skip short words) + title_words = {w for w in re.findall(r'\w+', res.title.lower()) if len(w) > 3} + q_words = {w for w in re.findall(r'\w+', question_lower) if len(w) > 3} + + # Check overlap "Constitution", "Charte", etc. + if title_words.intersection(q_words): + matched_results.append(res) + + if matched_results: + # 2. Filter by Language (Deduplication) + q_lang = self._detect_language(question) + same_lang_matches = [ + res for res in matched_results + if (getattr(res, 'language', 'fr') or 'fr').lower().startswith(q_lang) + ] + + # Use same-language matches if available, otherwise all matches + # Take Top 1 to ensure "only the law used" is cited + candidates = same_lang_matches[:1] if same_lang_matches else matched_results[:1] + logger.info(f"🔍 Filtered sources by title match: {[c.title for c in candidates]}") + else: + # 3. No match found - Use Top 1 result (most relevant) + candidates = search_results[:1] + logger.info("🔍 No title match, using top result") + + # Create citation + for result in candidates: + try: + excerpt = "" + if article_num: + excerpt = self._find_article_excerpt(result, article_num) + + if not excerpt and hasattr(result, 'highlights') and result.highlights: + excerpt = result.highlights.get('content', '')[:300] + + if not excerpt and hasattr(result, 'content'): + excerpt = result.content[:300] if result.content else "" + + sources.append(Citation( + law_id=result.law_id, + law_reference=getattr(result, 'reference', ''), + law_title=result.title, + article_number=article_num, + excerpt=excerpt[:300] if excerpt else "Contenu du document", + relevance_score=getattr(result, 'relevance_score', 0.8) + )) + except Exception as e: + logger.warning(f"⚠️ Error creating source from result: {e}") + + return sources + + def _find_article_excerpt(self, result, article_num: str) -> str: + """Find excerpt for specific article from search result.""" + print(f"DEBUG: _find_article_excerpt called for article {article_num}, law_id={result.law_id}") + + # PRIORITY 1: Query PostgreSQL articles table directly + try: + from sqlalchemy import create_engine, text + from app.core.config import settings + + engine = create_engine(settings.DATABASE_URL.replace('+asyncpg', '')) + with engine.connect() as conn: + # Try multiple number formats + number_variants = [article_num] + + # Handle "premier"/"première" for article 1 + if article_num == "1": + number_variants.extend(["premier", "première", "1er", "1ère"]) + elif article_num.lower() in ["premier", "première"]: + number_variants.extend(["1", "1er", "1ère"]) + + print(f"DEBUG: Trying number variants: {number_variants}") + + # Try each variant + for num in number_variants: + sql = text(""" + SELECT content FROM articles + WHERE law_id = :law_id AND (number = :number OR number ILIKE :pattern) + LIMIT 1 + """) + res = conn.execute(sql, { + "law_id": result.law_id, + "number": num, + "pattern": f"%{num}%" + }) + row = res.fetchone() + if row and row[0]: + content = row[0] + print(f"DEBUG: FOUND article {num} content: {content[:100]}...") + return content[:300] if len(content) > 300 else content + + print(f"DEBUG: Article {article_num} NOT FOUND in PostgreSQL") + except Exception as e: + print(f"DEBUG: PostgreSQL query failed: {e}") + + # FALLBACK 1: Check matched_articles from Meilisearch + if hasattr(result, 'matched_articles') and result.matched_articles: + for article in result.matched_articles: + if hasattr(article, 'number') and article.number == article_num: + if hasattr(article, 'snippet'): + return article.snippet[:300] + + # FALLBACK 2: Use highlights from Meilisearch + if hasattr(result, 'highlights'): + content = result.highlights.get("content", "") + if content: + print(f"DEBUG: Using Meilisearch highlights as fallback") + return content[:300] + + return "Voir document complet pour détails" + + def _calculate_confidence( + self, + answer: str, + citations: List[Citation], + search_results: List + ) -> float: + """ + Calculate confidence score based on multiple factors. + + Factors: + 1. Number of citations (more = better) + 2. Average relevance of cited docs + 3. Answer length (too short/long = lower confidence) + 4. Presence of hedge words (probably, maybe = lower) + + Returns: + Confidence score 0.0-1.0 + """ + score = 0.0 + + # Factor 1: Citations (0-0.4) + if citations: + citation_score = min(len(citations) / 3, 1.0) * 0.4 + score += citation_score + + # Factor 2: Relevance (0-0.3) + if citations: + avg_relevance = sum(c.relevance_score for c in citations) / len(citations) + score += avg_relevance * 0.3 + elif search_results: + # No citations but have results + avg_relevance = sum(r.relevance_score for r in search_results[:3]) / min(3, len(search_results)) + score += avg_relevance * 0.2 + + # Factor 3: Answer length (0-0.2) + answer_len = len(answer.split()) + if 50 <= answer_len <= 500: + score += 0.2 + elif 30 <= answer_len < 50 or 500 < answer_len <= 700: + score += 0.1 + + # Factor 4: Hedge detection (0-0.1) + hedge_words = ["peut-être", "probablement", "possiblement", "éventuellement"] + hedge_count = sum(1 for word in hedge_words if word in answer.lower()) + if hedge_count == 0: + score += 0.1 + elif hedge_count <= 2: + score += 0.05 + + return round(min(score, 1.0), 2) + + async def _save_interaction( + self, + conversation: Conversation, + question: str, + answer: str, + citations: List[Citation], + confidence: float, + retrieval_time_ms: int, + generation_time_ms: int + ): + """Save question and answer to database.""" + # User message + user_msg = Message( + conversation_id=conversation.id, + role="user", + content=question + ) + self.db.add(user_msg) + + # Assistant message + sources_json = [c.model_dump() for c in citations] + assistant_msg = Message( + conversation_id=conversation.id, + role="assistant", + content=answer, + sources=sources_json, + confidence=confidence, + retrieval_time_ms=retrieval_time_ms, + generation_time_ms=generation_time_ms + ) + self.db.add(assistant_msg) + + # Update conversation timestamp + conversation.updated_at = datetime.utcnow() + + await self.db.commit() + logger.info("💾 Saved interaction to database") + + # Update persona interaction metrics + try: + from app.services.persona_service import PersonaService + persona_service = PersonaService(self.db) + await persona_service.update_interaction_metrics( + conversation_id=conversation.id, + persona=conversation.persona, + language=conversation.language + ) + logger.debug("📊 Updated persona interaction metrics") + except Exception as e: + logger.warning(f"⚠️ Failed to update persona metrics: {e}") + + async def _handle_no_results( + self, + request: RAGRequest, + retrieval_time_ms: int + ) -> RAGResponse: + """Handle case when no documents found.""" + conversation, _ = await self._load_or_create_conversation( + request.session_id, + request.persona, + request.language + ) + + no_results_answer = NO_RESULTS_MESSAGE[request.language] + + await self._save_interaction( + conversation, + request.question, + no_results_answer, + [], + 0.0, + retrieval_time_ms, + 0 + ) + + return RAGResponse( + answer=no_results_answer, + confidence=0.0, + sources=[], + session_id=conversation.session_id, + retrieval_time_ms=retrieval_time_ms, + generation_time_ms=0, + total_time_ms=retrieval_time_ms, + persona=request.persona + ) diff --git a/app/services/search_service.py b/app/services/search_service.py new file mode 100644 index 0000000000000000000000000000000000000000..fdde47277ef75d56f51ba8f521f431f9346db969 --- /dev/null +++ b/app/services/search_service.py @@ -0,0 +1,1482 @@ +""" +Service de recherche hybride (textuelle + sémantique) pour JuriX. + +Ce service implémente 3 modes de recherche: +1. Textuelle (Meilisearch) - Recherche full-text rapide (<50ms) +2. Sémantique (pgvector) - Recherche contextuelle par similarité (<200ms) +3. Hybrid (RRF fusion 40/60) - Combinaison optimale (<200ms) + +Architecture: +- Meilisearch: Full-text search avec typo tolerance +- pgvector: Vector similarity avec HNSW index +- RRF Fusion: Weighted Reciprocal Rank Fusion (40% text + 60% semantic) +- Redis: Cache résultats avec TTL 1h + +Performance cible: +- Text search: <50ms +- Semantic search: <200ms +- Hybrid search: <200ms (spec requirement) +- Indexing: <100ms per law + +Usage: + service = SearchService(db_session) + + # Recherche hybrid + response = await service.search(SearchRequest( + query="responsabilité dirigeants société", + mode="hybrid", + filters=SearchFilters(language="fr"), + limit=15 + )) + + # Indexation automatique + await service.index_law(law) + +Author: JuriX Team +Version: 1.0.0 +""" + +import asyncio +import hashlib +import json +import logging +import time +from typing import Any, Dict, List, Optional, Tuple + +import meilisearch +import numpy as np +import redis +from sqlalchemy import func, select, text +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import joinedload, selectinload + +from app.core.config import settings +from app.models.law import Article, Category, Law +from app.schemas.search import ( + ArticleMatch, + ReindexResponse, + SearchFilters, + SearchRequest, + SearchResponse, + SearchResult, + SearchStats, +) +from app.services.embedding_service import EmbeddingService + +logger = logging.getLogger(__name__) + +# ==================== GLOBAL SINGLETONS ==================== +# These are initialized once at module load / first access to avoid +# reloading the heavy embedding model on every request. + +_embedding_service_instance: Optional[EmbeddingService] = None +_meilisearch_client_instance: Optional[meilisearch.Client] = None +_redis_client_instance: Optional[redis.Redis] = None +_singletons_initialized: bool = False + + +def _init_global_singletons() -> None: + """ + Initialize global singleton instances for performance. + Called once at first SearchService instantiation. + """ + global _embedding_service_instance, _meilisearch_client_instance, _redis_client_instance, _singletons_initialized + + if _singletons_initialized: + return + + logger.info("🚀 Initializing global search singletons (one-time)...") + + # 1. Meilisearch client (fast, lightweight) + try: + _meilisearch_client_instance = meilisearch.Client( + settings.MEILISEARCH_URL, + settings.MEILISEARCH_KEY + ) + _meilisearch_client_instance.health() + + # Ensure index exists + try: + _meilisearch_client_instance.get_index(SearchService.MEILISEARCH_INDEX) + except Exception: + _meilisearch_client_instance.create_index( + SearchService.MEILISEARCH_INDEX, + {"primaryKey": "id"} + ) + logger.info("✅ Meilisearch client initialized") + except Exception as e: + logger.error(f"❌ Meilisearch init failed: {e}") + _meilisearch_client_instance = None + + # 2. Redis client (fast) + try: + _redis_client_instance = redis.from_url( + settings.REDIS_URL, + decode_responses=False, + socket_connect_timeout=2, + socket_timeout=2 + ) + _redis_client_instance.ping() + logger.info("✅ Redis client initialized") + except Exception as e: + logger.warning(f"⚠️ Redis init failed (cache disabled): {e}") + _redis_client_instance = None + + # 3. EmbeddingService (SLOW - loads 120MB model) + try: + _embedding_service_instance = EmbeddingService( + redis_url=settings.REDIS_URL, + use_cache=True + ) + logger.info("✅ EmbeddingService initialized") + except Exception as e: + logger.error(f"❌ EmbeddingService init failed: {e}") + _embedding_service_instance = None + + _singletons_initialized = True + logger.info("✅ All global singletons ready") + + +def get_redis_client() -> Optional[redis.Redis]: + """ + Get the global Redis client instance. + + Returns: + Redis client or None if not initialized + """ + global _redis_client_instance, _singletons_initialized + + if not _singletons_initialized: + _init_global_singletons() + + return _redis_client_instance + + +# ==================== EXCEPTIONS ==================== + + +class SearchServiceError(Exception): + """Exception de base pour SearchService.""" + pass + + +class MeilisearchError(SearchServiceError): + """Erreur liée à Meilisearch (connexion, query, indexing).""" + pass + + +class VectorSearchError(SearchServiceError): + """Erreur liée à la recherche vectorielle pgvector.""" + pass + + +class IndexingError(SearchServiceError): + """Erreur lors des opérations d'indexation.""" + pass + + +# ==================== SEARCH SERVICE ==================== + + +class SearchService: + """ + Service de recherche hybride pour lois camerounaises. + + Fournit 3 modes de recherche avec fusion RRF et caching Redis: + - Text: Meilisearch full-text (mots-clés, typos, proximité) + - Semantic: pgvector cosine similarity (contexte, synonymes) + - Hybrid: RRF fusion 40% text + 60% semantic (optimal) + + Caractéristiques: + - Performance: <200ms hybrid search (spec requirement) + - Filters: language, category, type, status, dates + - Pagination: limit/offset + - Highlights: Extraits avec tags + - Caching: Redis TTL 1h avec invalidation + - Auto-indexing: Hooks dans LawService + + Attributes: + db: Session database async SQLAlchemy + embedding_service: Service embeddings (singleton) + meilisearch_client: Client Meilisearch + redis_client: Client Redis (optional) + use_cache: Flag activation cache + """ + + # Configuration + MEILISEARCH_INDEX = "laws" # Laws index + MEILISEARCH_ARTICLES_INDEX = "articles" # Articles index for article-level search + CACHE_TTL_SECONDS = 300 # 5 minutes (reduced from 1 hour to prevent stale data) + RRF_K = 60 # Constante RRF fusion + TEXT_WEIGHT = 0.4 # Poids recherche textuelle + SEMANTIC_WEIGHT = 0.6 # Poids recherche sémantique + MAX_RESULTS_PER_MODE = 20 # Limite avant fusion + CONTENT_TRUNCATE_LENGTH = 10000 # Chars pour Meilisearch + + def __init__(self, db: AsyncSession, use_cache: bool = True): + """ + Initialise le service de recherche. + + Args: + db: Session database async SQLAlchemy + use_cache: Active le cache Redis (défaut: True) + + Raises: + MeilisearchError: Si connexion Meilisearch échoue + """ + # Initialize global singletons once (avoids reloading model every request) + _init_global_singletons() + + self.db = db + self.use_cache = use_cache + + # Embedding service (singleton) + self.embedding_service = self._get_embedding_service() + + # Meilisearch client + self.meilisearch_client = self._init_meilisearch() + + # Redis client (optional) + self.redis_client = self._init_redis() if use_cache else None + + logger.debug( + f"SearchService ready (cache={use_cache})" + ) + + # ==================== ARTICLE REFERENCE PARSING ==================== + + def _parse_article_reference(self, query: str) -> Optional[Dict[str, str]]: + """ + Détecte et extrait les références d'articles dans la query. + + Patterns supportés: + - "article 5 de la constitution" + - "article premier du code civil" + - "art. 12 de la loi sur..." + - "article 1er du décret..." + + Args: + query: Requête de recherche originale + + Returns: + Dict avec 'article_num' et 'doc_hint', ou None si pas de référence + + Example: + >>> _parse_article_reference("article 5 de la constitution") + {'article_num': '5', 'doc_hint': 'constitution'} + """ + import re + + query_lower = query.lower().strip() + + # Patterns pour détecter les références d'articles (FR + EN) + patterns = [ + # French: "article 5 de la constitution" / "article premier du code" + r"article\s+(premier|1er|1ère|\d+)\s+(?:de\s+)?(?:la\s+|le\s+|l[''']|du\s+|des\s+)?(.+)", + # French: "art. 12 de la loi" + r"art\.?\s*(\d+)\s+(?:de\s+)?(?:la\s+|le\s+|l[''']|du\s+)?(.+)", + # English: "section 5 of the constitution" + r"section\s+(\d+|one|first)\s+(?:of\s+)?(?:the\s+)?(.+)", + ] + + for pattern in patterns: + match = re.search(pattern, query_lower, re.IGNORECASE) + if match: + article_num = match.group(1).strip() + doc_hint = match.group(2).strip() if match.group(2) else "" + + # Normaliser le numéro d'article + article_num_normalized = article_num.upper() + if article_num_normalized in ("PREMIER", "1ER", "1ÈRE"): + article_num_normalized = "PREMIER" + + logger.info( + f"📌 Detected article reference: article={article_num_normalized}, " + f"doc_hint='{doc_hint}'" + ) + + return { + "article_num": article_num_normalized, + "doc_hint": doc_hint + } + + return None + + # ==================== PUBLIC API ==================== + + async def search(self, request: SearchRequest) -> SearchResponse: + """ + Point d'entrée principal pour la recherche. + + Dispatches to text/semantic/hybrid search, builds response, + handles fallbacks on errors. + + Args: + request: Requête de recherche (query, mode, filters, pagination) + + Returns: + SearchResponse avec résultats et métadonnées + """ + assert request and request.query, "SearchRequest with query is required" + start_time = time.time() + + logger.info( + f"🔍 Search request: mode={request.mode}, " + f"query=\"{request.query[:50]}\", filters={request.filters}" + ) + + # Check cache + if self.use_cache and self.redis_client: + cached_response = self._get_from_cache(request) + if cached_response: + elapsed_ms = int((time.time() - start_time) * 1000) + logger.info(f"🎯 Cache HIT ({elapsed_ms}ms)") + return cached_response + + # Execute search with fallback handling + try: + results = await self._execute_search_by_mode(request) + elapsed_ms = int((time.time() - start_time) * 1000) + response = self._build_search_response(request, results, elapsed_ms) + + if self.use_cache and self.redis_client: + self._store_in_cache(request, response) + + logger.info( + f"✅ Search completed: {response.total} results in {elapsed_ms}ms " + f"(mode={request.mode})" + ) + return response + + except MeilisearchError as e: + return await self._handle_search_fallback( + request, start_time, "semantic", e, "Meilisearch" + ) + except VectorSearchError as e: + return await self._handle_search_fallback( + request, start_time, "text", e, "Vector search" + ) + + async def _execute_search_by_mode(self, request: SearchRequest): + """Dispatch search to the correct mode handler.""" + if request.mode == "text": + return await self.text_search( + request.query, request.filters, request.limit, request.offset + ) + elif request.mode == "semantic": + return await self.semantic_search( + request.query, request.filters, request.limit, request.offset + ) + elif request.mode == "hybrid": + return await self.hybrid_search( + request.query, request.filters, request.limit, request.offset + ) + else: + raise ValueError(f"Mode invalide: {request.mode}") + + def _build_search_response(self, request, results, elapsed_ms): + """Build SearchResponse with article reference detection.""" + total = len(results) + + article_ref = self._parse_article_reference(request.query) + target_article = None + direct_navigation = False + + if article_ref and total == 1: + target_article = article_ref["article_num"] + direct_navigation = True + logger.info(f"🎯 Direct navigation enabled: article {target_article}") + elif article_ref and total > 1: + target_article = article_ref["article_num"] + logger.info(f"📌 Article reference detected: {target_article} (multiple results)") + + return SearchResponse( + query=request.query, + mode=request.mode, + results=results, + total=total, + search_time_ms=elapsed_ms, + filters_applied=request.filters.model_dump() if request.filters else None, + target_article=target_article, + direct_navigation=direct_navigation + ) + + async def _handle_search_fallback(self, request, start_time, fallback_mode, error, source_name): + """Handle search error by falling back to alternative mode.""" + logger.warning(f"⚠️ {source_name} failed, fallback to {fallback_mode}: {error}") + if request.mode != "hybrid": + raise error + + if fallback_mode == "semantic": + results = await self.semantic_search( + request.query, request.filters, request.limit, request.offset + ) + else: + results = await self.text_search( + request.query, request.filters, request.limit, request.offset + ) + + elapsed_ms = int((time.time() - start_time) * 1000) + return SearchResponse( + query=request.query, + mode=fallback_mode, + results=results, + total=len(results), + search_time_ms=elapsed_ms, + filters_applied=request.filters.model_dump() if request.filters else None + ) + + async def text_search( + self, + query: str, + filters: Optional[SearchFilters] = None, + limit: int = 15, + offset: int = 0 + ) -> List[SearchResult]: + """ + Recherche textuelle via Meilisearch. + + Searches articles index first, falls back to laws index. + + Args: + query: Requête textuelle + filters: Filtres optionnels + limit: Nombre max résultats (défaut: 15) + offset: Offset pagination (défaut: 0) + + Returns: + Liste de SearchResult triée par relevance_score + + Raises: + MeilisearchError: Si recherche échoue + """ + assert query and isinstance(query, str), "query must be a non-empty string" + assert limit > 0, "limit must be positive" + start_time = time.time() + + try: + # Priority: articles index, then laws index as fallback + results = self._search_articles_index(query, limit, offset) + if not results: + results = self._search_laws_index(query, filters, limit, offset) + + elapsed_ms = int((time.time() - start_time) * 1000) + logger.info(f"📝 Text search total: {len(results)} results in {elapsed_ms}ms") + return results + + except Exception as e: + logger.error(f"❌ Text search failed: {e}") + raise MeilisearchError(f"Échec recherche textuelle: {e}") from e + + def _search_articles_index( + self, query: str, limit: int, offset: int + ) -> List[SearchResult]: + """Search articles Meilisearch index for article-level content.""" + try: + articles_index = self.meilisearch_client.index(self.MEILISEARCH_ARTICLES_INDEX) + articles_params = { + "limit": min(limit, self.MAX_RESULTS_PER_MODE), + "offset": offset, + "attributesToRetrieve": [ + "id", "number", "title", "section", "content", + "law_id", "law_title", "law_reference", "category" + ], + "attributesToHighlight": ["content", "title"], + "highlightPreTag": "", + "highlightPostTag": "", + } + + articles_response = articles_index.search(query, articles_params) + results = [] + seen_law_ids = set() + + for rank, hit in enumerate(articles_response.get("hits", []), start=1): + law_id = hit.get("law_id") + if law_id in seen_law_ids: + continue + seen_law_ids.add(law_id) + + relevance_score = 1.0 / (1.0 + rank * 0.1) + content = hit.get("content", "") + highlights = {} + if "_formatted" in hit: + formatted = hit["_formatted"] + if "content" in formatted: + highlights["content"] = self._extract_highlight_snippet( + formatted["content"], max_length=500 + ) + + results.append(SearchResult( + law_id=law_id, + reference=hit.get("law_reference", ""), + title=hit.get("law_title", ""), + type="loi", + language="fr", + status="published", + category_id=None, + category_name=hit.get("category"), + publication_date=None, + relevance_score=relevance_score, + matched_articles=[ArticleMatch( + article_id=hit.get("id", 0), + number=hit.get("number", ""), + title=hit.get("title"), + content_snippet=content[:300] if content else "", + relevance_score=relevance_score + )], + highlights=highlights, + content=content + )) + + logger.info(f"📝 Articles search: {len(results)} results") + return results + + except Exception as e: + logger.warning(f"⚠️ Articles index search failed: {e}") + return [] + + def _search_laws_index( + self, query: str, filters: Optional[SearchFilters], + limit: int, offset: int + ) -> List[SearchResult]: + """Search laws Meilisearch index as fallback.""" + try: + filter_string = self._apply_filters_meilisearch(filters) + index = self.meilisearch_client.index(self.MEILISEARCH_INDEX) + + search_params = { + "limit": min(limit, self.MAX_RESULTS_PER_MODE), + "offset": offset, + "attributesToRetrieve": [ + "id", "reference", "title", "type", "language", + "status", "category_id", "category_name", + "publication_year", "content" + ], + "attributesToHighlight": ["title", "content"], + "highlightPreTag": "", + "highlightPostTag": "", + } + if filter_string: + search_params["filter"] = filter_string + + meili_response = index.search(query, search_params) + results = [] + + for rank, hit in enumerate(meili_response["hits"], start=1): + relevance_score = 1.0 / (1.0 + rank * 0.1) + highlights = {} + if "_formatted" in hit: + formatted = hit["_formatted"] + if "title" in formatted: + highlights["title"] = formatted["title"] + if "content" in formatted: + highlights["content"] = self._extract_highlight_snippet( + formatted["content"], max_length=300 + ) + + results.append(SearchResult( + law_id=hit["id"], + reference=hit["reference"], + title=hit["title"], + type=hit["type"], + language=hit.get("language"), + status=hit["status"], + category_id=hit.get("category_id"), + category_name=hit.get("category_name"), + publication_date=None, + relevance_score=relevance_score, + matched_articles=[], + highlights=highlights, + content=hit.get("content", "") + )) + + logger.info(f"📝 Laws search: {len(results)} results") + return results + + except Exception as e: + logger.warning(f"⚠️ Laws index search failed: {e}") + return [] + + async def semantic_search( + self, + query: str, + filters: Optional[SearchFilters] = None, + limit: int = 15, + offset: int = 0 + ) -> List[SearchResult]: + """ + Recherche sémantique via pgvector. + + Effectue une recherche par similarité cosine sur embeddings: + 1. Génère embedding de la query (3072-dim) + 2. Recherche KNN dans articles.embedding + 3. Agrège résultats par law_id (score max) + 4. Applique filtres et pagination + + Args: + query: Requête textuelle + filters: Filtres optionnels + limit: Nombre max résultats (défaut: 15) + offset: Offset pagination (défaut: 0) + + Returns: + Liste de SearchResult triée par relevance_score (similarity) + + Raises: + VectorSearchError: Si recherche échoue + + Performance: <200ms avec HNSW index + """ + assert isinstance(query, str) and len(query) > 0, "Query must be a non-empty string" + assert isinstance(limit, int) and limit > 0, "Limit must be a positive integer" + + start_time = time.time() + + try: + # Check if embedding service is available + if self.embedding_service is None: + logger.warning("⚠️ Semantic search unavailable: EmbeddingService not loaded") + return [] + + # Génération embedding query + query_embedding = self.embedding_service.generate_embedding(query) + query_embedding_str = f"[{','.join(map(str, query_embedding.tolist()))}]" + + # Construction query pgvector avec similarité cosine réelle + # Using pgvector's cosine_distance: 1 - cosine_distance = cosine similarity + # Lower distance = higher similarity (0 = identical) + from sqlalchemy import cast + from sqlalchemy.types import String + + stmt = ( + select( + Law.id, + Law.reference, + Law.title, + Law.type, + Law.language, + Law.status, + Law.category_id, + Category.name.label("category_name"), + func.max( + # Cosine similarity: 1 - cosine_distance + # Higher score = more similar (0.0 to 1.0) + 1 - func.cosine_distance( + Article.embedding, + cast(query_embedding_str, String) + ) + ).label("similarity") + ) + .join(Article, Law.id == Article.law_id) + .outerjoin(Category, Law.category_id == Category.id) + .where(Article.embedding.isnot(None)) + ) + + # Application filtres + stmt = self._apply_filters_pgvector(stmt, filters) + + # Group by law, order by similarity + stmt = ( + stmt + .group_by( + Law.id, Law.reference, Law.title, Law.type, + Law.language, Law.status, Law.category_id, Category.name + ) + .order_by(text("similarity DESC")) + .limit(min(limit, self.MAX_RESULTS_PER_MODE)) + .offset(offset) + ) + + # Exécution + result = await self.db.execute(stmt) + rows = result.all() + + # Conversion vers SearchResult + results = [] + for row in rows: + result_obj = SearchResult( + law_id=row.id, + reference=row.reference, + title=row.title, + type=row.type, + language=row.language, + status=row.status, + category_id=row.category_id, + category_name=row.category_name, + publication_date=None, + relevance_score=float(row.similarity), # Déjà normalisé 0-1 + matched_articles=[], # TODO: Récupérer articles matchés + highlights={} # Semantic search ne produit pas highlights + ) + results.append(result_obj) + + elapsed_ms = int((time.time() - start_time) * 1000) + logger.info( + f"🧠 Semantic search: {len(results)} results in {elapsed_ms}ms" + ) + + return results + + except Exception as e: + logger.error(f"❌ Semantic search failed: {e}") + raise VectorSearchError(f"Échec recherche sémantique: {e}") from e + + async def hybrid_search( + self, + query: str, + filters: Optional[SearchFilters] = None, + limit: int = 15, + offset: int = 0 + ) -> List[SearchResult]: + """ + Recherche hybride via RRF fusion (40% text + 60% semantic). + + Stratégie: + 1. Exécute text_search et semantic_search en parallèle (asyncio.gather) + 2. Applique RRF fusion avec poids 40/60 + 3. Normalise scores finaux à [0, 1] + 4. Déduplique par law_id (garde meilleur score) + 5. Trie par RRF score décroissant + 6. Applique pagination + + Formule RRF: + rrf_score = 0.4 × (1/(60 + text_rank)) + 0.6 × (1/(60 + semantic_rank)) + + Args: + query: Requête textuelle + filters: Filtres optionnels + limit: Nombre max résultats (défaut: 15) + offset: Offset pagination (défaut: 0) + + Returns: + Liste de SearchResult triée par RRF score + + Raises: + SearchServiceError: Si les deux modes échouent + + Performance: <200ms (spec requirement) + """ + assert isinstance(query, str) and len(query) > 0, "Query must be a non-empty string" + assert isinstance(limit, int) and limit > 0, "Limit must be a positive integer" + + start_time = time.time() + + try: + # Exécution parallèle des deux modes + text_results, semantic_results = await asyncio.gather( + self.text_search( + query, + filters, + limit=self.MAX_RESULTS_PER_MODE, + offset=0 # Fusion puis pagination + ), + self.semantic_search( + query, + filters, + limit=self.MAX_RESULTS_PER_MODE, + offset=0 + ), + return_exceptions=True + ) + + # Gestion erreurs + if isinstance(text_results, Exception): + logger.warning(f"⚠️ Text search failed in hybrid: {text_results}") + text_results = [] + + if isinstance(semantic_results, Exception): + logger.warning(f"⚠️ Semantic search failed in hybrid: {semantic_results}") + semantic_results = [] + + if not text_results and not semantic_results: + logger.info("🔀 Hybrid search: No results found in either text or semantic search") + return [] + + # Application RRF fusion + fused_results = self._rrf_fusion( + text_results, + semantic_results, + k=self.RRF_K, + text_weight=self.TEXT_WEIGHT, + semantic_weight=self.SEMANTIC_WEIGHT + ) + + # Normalisation scores + fused_results = self._normalize_scores(fused_results) + + # Pagination + paginated_results = fused_results[offset:offset + limit] + + elapsed_ms = int((time.time() - start_time) * 1000) + logger.info( + f"🔀 Hybrid search: {len(paginated_results)} results in {elapsed_ms}ms " + f"(text={len(text_results)}, semantic={len(semantic_results)}, " + f"fused={len(fused_results)})" + ) + + return paginated_results + + except Exception as e: + logger.error(f"❌ Hybrid search failed: {e}") + raise SearchServiceError(f"Échec recherche hybride: {e}") from e + + # ==================== INDEXING METHODS ==================== + + async def index_law(self, law: Law) -> None: + """ + Indexe une loi dans Meilisearch. + + Appelé automatiquement lors de la création d'une loi (hook LawService). + Crée un document Meilisearch avec: + - Champs searchable: title, reference, content (tronqué 10k) + - Champs filterable: id, category_id, type, language, status, year + - Metadata: category_name, timestamps + + Args: + law: Instance Law à indexer + + Raises: + IndexingError: Si indexation échoue + + Performance: <100ms + """ + try: + # Construction document Meilisearch + document = { + "id": law.id, + "reference": law.reference, + "title": law.title, + "content": law.content[:self.CONTENT_TRUNCATE_LENGTH], + "type": law.type, + "language": law.language or "unknown", + "status": law.status, + "category_id": law.category_id, + "category_name": law.category.name if law.category else None, + "publication_year": ( + law.publication_date.year + if law.publication_date else None + ), + "created_at_timestamp": int(law.created_at.timestamp()) + } + + # Indexation + index = self.meilisearch_client.index(self.MEILISEARCH_INDEX) + index.add_documents([document]) + + logger.info(f"✅ Indexed law {law.id} ({law.reference})") + + except Exception as e: + logger.error(f"❌ Failed to index law {law.id}: {e}") + raise IndexingError(f"Échec indexation law {law.id}: {e}") from e + + async def update_law_index(self, law_id: int, law: Law) -> None: + """ + Met à jour l'index d'une loi existante. + + Meilisearch effectue automatiquement un upsert basé sur l'ID. + + Args: + law_id: ID de la loi (pour cohérence API) + law: Instance Law mise à jour + + Raises: + IndexingError: Si mise à jour échoue + """ + # Meilisearch upsert automatique via add_documents + await self.index_law(law) + logger.info(f"✅ Updated index for law {law_id}") + + async def delete_law_index(self, law_id: int) -> None: + """ + Supprime une loi de l'index Meilisearch. + + Args: + law_id: ID de la loi à supprimer + + Raises: + IndexingError: Si suppression échoue + """ + try: + index = self.meilisearch_client.index(self.MEILISEARCH_INDEX) + index.delete_document(law_id) + logger.info(f"✅ Deleted law {law_id} from index") + + except Exception as e: + logger.error(f"❌ Failed to delete law {law_id} from index: {e}") + raise IndexingError(f"Échec suppression index law {law_id}: {e}") from e + + async def reindex_all_laws(self) -> ReindexResponse: + """ + Réindexe toutes les lois publiées (opération admin). + + Stratégie: + 1. Récupère toutes les lois publiées depuis DB + 2. Construit documents batch + 3. Supprime ancien index + 4. Recrée index avec configuration + 5. Indexe documents en batch (1000 par batch) + + Returns: + ReindexResponse avec statistiques + + Raises: + IndexingError: Si réindexation échoue + + Performance: <5s pour 100 lois + """ + assert self.meilisearch_client is not None, "Meilisearch client must be initialized" + assert self.db is not None, "Database session must be initialized" + + start_time = time.time() + + try: + # Récupération lois publiées + stmt = ( + select(Law) + .where(Law.status == "published") + .options(joinedload(Law.category)) + ) + result = await self.db.execute(stmt) + laws = result.scalars().all() + + total_laws = len(laws) + logger.info(f"🔄 Reindexing {total_laws} laws...") + + # Construction documents + documents = [] + for law in laws: + doc = { + "id": law.id, + "reference": law.reference, + "title": law.title, + "content": law.content[:self.CONTENT_TRUNCATE_LENGTH], + "type": law.type, + "language": law.language or "unknown", + "status": law.status, + "category_id": law.category_id, + "category_name": law.category.name if law.category else None, + "publication_year": ( + law.publication_date.year + if law.publication_date else None + ), + "created_at_timestamp": int(law.created_at.timestamp()) + } + documents.append(doc) + + # Suppression ancien index (si existe) + try: + self.meilisearch_client.delete_index(self.MEILISEARCH_INDEX) + logger.info("Deleted old index") + except Exception: + pass # Index n'existait pas + + # Création index avec configuration + self.meilisearch_client.create_index( + self.MEILISEARCH_INDEX, + {"primaryKey": "id"} + ) + + # Get the index object + index = self.meilisearch_client.index(self.MEILISEARCH_INDEX) + + # Configuration ranking rules + index.update_settings({ + "searchableAttributes": ["title", "reference", "content"], + "filterableAttributes": [ + "id", "category_id", "type", "language", + "status", "publication_year" + ], + "sortableAttributes": ["publication_year", "created_at_timestamp"], + "rankingRules": [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness" + ], + "typoTolerance": { + "enabled": True, + "minWordSizeForTypos": { + "oneTypo": 4, + "twoTypos": 8 + } + } + }) + + # Indexation batch (1000 par batch) + batch_size = 1000 + indexed_count = 0 + + for i in range(0, len(documents), batch_size): + batch = documents[i:i + batch_size] + index.add_documents(batch) + indexed_count += len(batch) + logger.info(f"📦 Indexed batch {i//batch_size + 1}: {indexed_count}/{total_laws}") + + elapsed_seconds = time.time() - start_time + + logger.info( + f"✅ Reindexing complete: {indexed_count} laws in {elapsed_seconds:.1f}s" + ) + + return ReindexResponse( + status="success", + total_laws=total_laws, + indexed_count=indexed_count, + failed_count=0, + duration_seconds=int(elapsed_seconds) + ) + + except Exception as e: + logger.error(f"❌ Reindexing failed: {e}") + raise IndexingError(f"Échec réindexation: {e}") from e + + # ==================== PRIVATE METHODS ==================== + + def _get_embedding_service(self) -> Optional[EmbeddingService]: + """ + Récupère le singleton EmbeddingService. + Returns None if embedding service is unavailable (text search still works). + + Returns: + Instance EmbeddingService or None + """ + global _embedding_service_instance + + if _embedding_service_instance is None: + # Try to create instance, but don't fail if unavailable + try: + from app.services.embedding_service import EmbeddingService + _embedding_service_instance = EmbeddingService( + redis_url=settings.REDIS_URL, + use_cache=True + ) + except Exception as e: + logger.warning(f"⚠️ EmbeddingService unavailable (text search only): {e}") + return None + + return _embedding_service_instance + + def _init_meilisearch(self) -> meilisearch.Client: + """ + Récupère le singleton Meilisearch client. + + Returns: + Client Meilisearch + + Raises: + MeilisearchError: Si connexion échoue + """ + global _meilisearch_client_instance + + if _meilisearch_client_instance is not None: + return _meilisearch_client_instance + + # Fallback: create if not initialized + try: + client = meilisearch.Client( + settings.MEILISEARCH_URL, + settings.MEILISEARCH_KEY + ) + client.health() + + try: + client.get_index(self.MEILISEARCH_INDEX) + except Exception: + client.create_index( + self.MEILISEARCH_INDEX, + {"primaryKey": "id"} + ) + + _meilisearch_client_instance = client + return client + + except Exception as e: + logger.error(f"❌ Meilisearch connection failed: {e}") + raise MeilisearchError(f"Connexion Meilisearch échouée: {e}") from e + + def _init_redis(self) -> Optional[redis.Redis]: + """ + Récupère le singleton Redis client pour le cache. + + Returns: + Client Redis ou None si connexion échoue + """ + global _redis_client_instance + + if _redis_client_instance is not None: + return _redis_client_instance + + # Fallback: create if not initialized + try: + client = redis.from_url( + settings.REDIS_URL, + decode_responses=False, + socket_connect_timeout=2, + socket_timeout=2 + ) + client.ping() + _redis_client_instance = client + return client + + except Exception as e: + logger.warning(f"⚠️ Redis unavailable, cache disabled: {e}") + return None + + def _apply_filters_meilisearch( + self, + filters: Optional[SearchFilters] + ) -> Optional[str]: + """ + Construit la chaîne de filtres pour Meilisearch. + + Format Meilisearch: "language = fr AND (category_id = 1 OR category_id = 2)" + + Args: + filters: Filtres SearchFilters + + Returns: + Chaîne de filtres ou None si aucun filtre + """ + if not filters: + return None + + conditions = [] + + if filters.language: + conditions.append(f"language = {filters.language}") + + if filters.category_ids: + cat_conditions = " OR ".join( + f"category_id = {cid}" for cid in filters.category_ids + ) + conditions.append(f"({cat_conditions})") + + if filters.types: + type_conditions = " OR ".join( + f"type = {typ}" for typ in filters.types + ) + conditions.append(f"({type_conditions})") + + if filters.status: + conditions.append(f"status = {filters.status}") + + if filters.year_from: + conditions.append(f"publication_year >= {filters.year_from}") + + if filters.year_to: + conditions.append(f"publication_year <= {filters.year_to}") + + return " AND ".join(conditions) if conditions else None + + def _apply_filters_pgvector( + self, + stmt: Any, + filters: Optional[SearchFilters] + ) -> Any: + """ + Applique les filtres SQL à la query pgvector. + + Args: + stmt: Statement SQLAlchemy + filters: Filtres SearchFilters + + Returns: + Statement modifié avec WHERE clauses + """ + if not filters: + return stmt + + if filters.language: + stmt = stmt.where(Law.language == filters.language) + + if filters.category_ids: + stmt = stmt.where(Law.category_id.in_(filters.category_ids)) + + if filters.types: + stmt = stmt.where(Law.type.in_(filters.types)) + + if filters.status: + stmt = stmt.where(Law.status == filters.status) + + if filters.year_from or filters.year_to: + # TODO: Filtrer par année publication_date + pass + + return stmt + + def _rrf_fusion( + self, + text_results: List[SearchResult], + semantic_results: List[SearchResult], + k: int = 60, + text_weight: float = 0.4, + semantic_weight: float = 0.6 + ) -> List[SearchResult]: + """ + Applique l'algorithme RRF (Reciprocal Rank Fusion) avec poids. + + Formule: rrf_score = text_weight × (1/(k + text_rank)) + + semantic_weight × (1/(k + semantic_rank)) + + Args: + text_results: Résultats recherche textuelle + semantic_results: Résultats recherche sémantique + k: Constante RRF (défaut: 60) + text_weight: Poids résultats textuels (défaut: 0.4) + semantic_weight: Poids résultats sémantiques (défaut: 0.6) + + Returns: + Liste fusionnée triée par RRF score décroissant + """ + scores = {} + + # Indexation résultats textuels (rank-based) + for rank, result in enumerate(text_results, start=1): + scores[result.law_id] = { + "text_rank": rank, + "semantic_rank": None, + "result": result + } + + # Indexation résultats sémantiques (rank-based) + for rank, result in enumerate(semantic_results, start=1): + if result.law_id in scores: + scores[result.law_id]["semantic_rank"] = rank + # Merge highlights et matched_articles + scores[result.law_id]["result"].highlights.update(result.highlights) + else: + scores[result.law_id] = { + "text_rank": None, + "semantic_rank": rank, + "result": result + } + + # Calcul RRF scores + for law_id, data in scores.items(): + rrf_score = 0.0 + + if data["text_rank"]: + rrf_score += text_weight * (1.0 / (k + data["text_rank"])) + + if data["semantic_rank"]: + rrf_score += semantic_weight * (1.0 / (k + data["semantic_rank"])) + + data["rrf_score"] = rrf_score + data["result"].relevance_score = rrf_score # Temporaire, sera normalisé + + # Tri par RRF score décroissant + sorted_results = sorted( + scores.values(), + key=lambda x: x["rrf_score"], + reverse=True + ) + + return [item["result"] for item in sorted_results] + + def _normalize_scores( + self, + results: List[SearchResult] + ) -> List[SearchResult]: + """ + Normalise les scores à [0, 1]. + + Args: + results: Liste SearchResult avec scores bruts + + Returns: + Liste avec scores normalisés + """ + if not results: + return results + + # Min-max normalization + scores = [r.relevance_score for r in results] + min_score = min(scores) + max_score = max(scores) + + if max_score - min_score < 1e-9: # Tous les scores identiques + for result in results: + result.relevance_score = 1.0 + else: + for result in results: + result.relevance_score = ( + (result.relevance_score - min_score) / + (max_score - min_score) + ) + + return results + + def _extract_highlight_snippet( + self, + content: str, + max_length: int = 300 + ) -> str: + """ + Extrait un snippet de contenu centré sur les tags. + + Args: + content: Contenu avec tags + max_length: Longueur max snippet (défaut: 300) + + Returns: + Snippet tronqué avec "..." si nécessaire + """ + if len(content) <= max_length: + return content + + # Recherche premier + mark_pos = content.find("") + + if mark_pos == -1: + # Pas de highlight, retourne début + return content[:max_length] + "..." + + # Centrer snippet autour du highlight + start = max(0, mark_pos - max_length // 2) + end = start + max_length + + snippet = content[start:end] + + if start > 0: + snippet = "..." + snippet + if end < len(content): + snippet = snippet + "..." + + return snippet + + def _get_cache_key(self, request: SearchRequest) -> str: + """ + Génère la clé de cache pour une requête. + + Args: + request: Requête de recherche + + Returns: + Clé cache format "search:{mode}:{md5_hash}" + """ + request_dict = { + "query": request.query, + "mode": request.mode, + "filters": request.filters.model_dump() if request.filters else None, + "limit": request.limit, + "offset": request.offset + } + + request_str = json.dumps(request_dict, sort_keys=True) + hash_digest = hashlib.md5(request_str.encode()).hexdigest() + + return f"search:{request.mode}:{hash_digest}" + + def _get_from_cache( + self, + request: SearchRequest + ) -> Optional[SearchResponse]: + """ + Récupère une réponse du cache Redis. + + Args: + request: Requête de recherche + + Returns: + SearchResponse si trouvé, None sinon + """ + if not self.redis_client: + return None + + try: + cache_key = self._get_cache_key(request) + cached_data = self.redis_client.get(cache_key) + + if cached_data: + # Désérialisation JSON -> Pydantic + response_dict = json.loads(cached_data) + return SearchResponse(**response_dict) + + except Exception as e: + logger.warning(f"⚠️ Cache read error: {e}") + + return None + + def _store_in_cache( + self, + request: SearchRequest, + response: SearchResponse + ) -> None: + """ + Stocke une réponse dans le cache Redis. + + Args: + request: Requête de recherche + response: Réponse à cacher + """ + if not self.redis_client: + return + + try: + cache_key = self._get_cache_key(request) + + # Sérialisation Pydantic -> JSON + response_dict = response.model_dump() + cached_data = json.dumps(response_dict) + + # Stockage avec TTL + self.redis_client.setex( + cache_key, + self.CACHE_TTL_SECONDS, + cached_data + ) + + except Exception as e: + logger.warning(f"⚠️ Cache write error: {e}") + + def invalidate_cache(self) -> int: + """ + Invalide tout le cache de recherche. + + À appeler quand les lois sont ajoutées/modifiées/supprimées. + + Returns: + Nombre de clés supprimées + """ + if not self.redis_client: + return 0 + + try: + # Find all search-related keys + keys = self.redis_client.keys("search:*") + if keys: + deleted = self.redis_client.delete(*keys) + logger.info(f"🗑️ Cache invalidated: {deleted} keys deleted") + return deleted + return 0 + except Exception as e: + logger.warning(f"⚠️ Cache invalidation error: {e}") + return 0 + + +# Global function to invalidate cache from anywhere +def invalidate_search_cache() -> int: + """ + Invalidate all search cache entries. + + Call this when laws are added/updated/deleted to ensure + search results are up-to-date. + + Returns: + Number of cache keys deleted + """ + try: + redis_client = get_redis_client() + if redis_client: + keys = redis_client.keys("search:*") + if keys: + deleted = redis_client.delete(*keys) + logger.info(f"🗑️ Global cache invalidation: {deleted} search keys deleted") + return deleted + return 0 + except Exception as e: + logger.warning(f"⚠️ Global cache invalidation error: {e}") + return 0 + diff --git a/app/services/validation_service.py b/app/services/validation_service.py new file mode 100644 index 0000000000000000000000000000000000000000..702d3cc643c5b2203b2c14f4d9df4ff2156858b2 --- /dev/null +++ b/app/services/validation_service.py @@ -0,0 +1,458 @@ +""" +Service de validation et sécurisation des données d'entrée. + +Ce service gère: +- Validation schéma (Pydantic) +- Prévention injection SQL +- Prévention XSS +- Rate limiting + +Objectif: Sécuriser toutes les entrées utilisateur. + +Usage: + service = ValidationService() + is_safe, reason = service.validate_sql_input("user input") + cleaned_html = service.sanitize_html("

content

") +""" + +import logging +import time +from collections import defaultdict +from datetime import datetime, timedelta +from threading import Lock +from typing import Any, Dict, Optional, Tuple + +from app.utils.validation_utils import ( + check_length_limits, + contains_xss, + is_safe_sql_input, + normalize_whitespace, + sanitize_filename, + sanitize_html, + sanitize_sql_input, + validate_email, + validate_pattern, + validate_url, +) + +logger = logging.getLogger(__name__) + + +class ValidationError(Exception): + """Exception levée lors d'erreurs de validation.""" + + pass + + +class RateLimitExceeded(Exception): + """Exception levée quand rate limit dépassé.""" + + pass + + +class ValidationService: + """ + Service de validation et sécurisation. + + Fonctionnalités: + - Validation SQL injection + - Sanitization XSS + - Rate limiting + - Validation formats (email, URL) + + Attributes: + rate_limit_data: Données de rate limiting + rate_limit_lock: Lock pour thread safety + """ + + def __init__(self): + """Initialise le service de validation.""" + logger.info("🚀 Initialisation du ValidationService...") + + # Rate limiting storage (in-memory) + # Format: {key: [(timestamp, count), ...]} + self.rate_limit_data: Dict[str, list] = defaultdict(list) + self.rate_limit_lock = Lock() + + logger.info("✅ ValidationService initialisé") + + # ==================== SQL INJECTION PREVENTION ==================== + + def validate_sql_input( + self, input_str: str, strict: bool = True + ) -> Tuple[bool, Optional[str]]: + """ + NOTE: Not recursive — delegates to imported ``validate_sql_input`` from ``app.utils.validation_utils``. + + Valide une entrée contre injection SQL. + + Args: + input_str: Entrée à valider + strict: Mode strict (défaut: True) + + Returns: + Tuple (is_safe, reason) + + Example: + >>> service.validate_sql_input("John Doe") + (True, None) + >>> service.validate_sql_input("'; DROP TABLE users;") + (False, "Contains SQL keyword: DROP") + """ + return is_safe_sql_input(input_str, strict=strict) + + def sanitize_sql_input(self, input_str: str) -> str: + """ + NOTE: Not recursive — delegates to imported ``sanitize_sql_input`` from ``app.utils.validation_utils``. + + Sanitize une entrée SQL. + + Note: Toujours utiliser des requêtes paramétrées en priorité. + + Args: + input_str: Entrée à sanitizer + + Returns: + Entrée sanitizée + + Example: + >>> service.sanitize_sql_input("John'; DROP TABLE") + "John DROP TABLE" + """ + return sanitize_sql_input(input_str) + + # ==================== XSS PREVENTION ==================== + + def check_xss(self, input_str: str) -> Tuple[bool, Optional[str]]: + """ + NOTE: Not recursive — delegates to imported ``check_xss`` from ``app.utils.validation_utils``. + + Vérifie si l'entrée contient du XSS. + + Args: + input_str: Entrée à vérifier + + Returns: + Tuple (contains_xss, pattern_found) + + Example: + >>> service.check_xss("") + (True, "script tag") + """ + return contains_xss(input_str) + + def sanitize_html( + self, html: str, allowed_tags: Optional[list] = None + ) -> str: + """ + NOTE: Not recursive — delegates to imported ``sanitize_html`` from ``app.utils.validation_utils``. + + Sanitize HTML pour prévenir XSS. + + Args: + html: HTML à sanitizer + allowed_tags: Tags autorisés (défaut: p, br, strong, em, u, a, ul, ol, li) + + Returns: + HTML sanitizé + + Example: + >>> service.sanitize_html("

Safe

") + "

Safe

" + """ + return sanitize_html(html, allowed_tags=allowed_tags) + + # ==================== RATE LIMITING ==================== + + def check_rate_limit( + self, key: str, limit: int = 100, window_seconds: int = 60 + ) -> Tuple[bool, Optional[str]]: + """ + Vérifie le rate limit pour une clé. + + Utilise sliding window algorithm. + + Args: + key: Clé unique (ex: IP, user_id) + limit: Nombre max de requêtes + window_seconds: Fenêtre en secondes + + Returns: + Tuple (is_allowed, reason) + + Raises: + RateLimitExceeded: Si limite dépassée + + Example: + >>> service.check_rate_limit("192.168.1.1", limit=10, window_seconds=60) + (True, None) + """ + with self.rate_limit_lock: + current_time = time.time() + window_start = current_time - window_seconds + + # Nettoyer les anciennes entrées + self.rate_limit_data[key] = [ + timestamp + for timestamp in self.rate_limit_data[key] + if timestamp > window_start + ] + + # Compter les requêtes dans la fenêtre + request_count = len(self.rate_limit_data[key]) + + if request_count >= limit: + return False, f"Rate limit exceeded: {request_count}/{limit} in {window_seconds}s" + + # Ajouter la requête actuelle + self.rate_limit_data[key].append(current_time) + + return True, None + + def reset_rate_limit(self, key: str) -> None: + """ + Reset le rate limit pour une clé. + + Args: + key: Clé à reset + + Example: + >>> service.reset_rate_limit("192.168.1.1") + """ + with self.rate_limit_lock: + if key in self.rate_limit_data: + del self.rate_limit_data[key] + logger.debug(f"Rate limit reset pour: {key}") + + def get_rate_limit_stats(self, key: str, window_seconds: int = 60) -> Dict[str, Any]: + """ + Récupère les stats de rate limit. + + Args: + key: Clé à vérifier + window_seconds: Fenêtre en secondes + + Returns: + Dictionnaire avec stats + + Example: + >>> service.get_rate_limit_stats("192.168.1.1") + {'requests': 5, 'window_seconds': 60, 'oldest_request': 1234567890} + """ + with self.rate_limit_lock: + current_time = time.time() + window_start = current_time - window_seconds + + # Filtrer les requêtes dans la fenêtre + requests_in_window = [ + timestamp + for timestamp in self.rate_limit_data.get(key, []) + if timestamp > window_start + ] + + return { + "requests": len(requests_in_window), + "window_seconds": window_seconds, + "oldest_request": min(requests_in_window) if requests_in_window else None, + "newest_request": max(requests_in_window) if requests_in_window else None, + } + + # ==================== FORMAT VALIDATION ==================== + + def validate_email(self, email: str) -> Tuple[bool, Optional[str]]: + """ + NOTE: Not recursive — delegates to imported ``validate_email`` from ``app.utils.validation_utils``. + + Valide un email. + + Args: + email: Email à valider + + Returns: + Tuple (is_valid, reason) + + Example: + >>> service.validate_email("user@example.com") + (True, None) + """ + if validate_email(email): + return True, None + else: + return False, "Invalid email format" + + def validate_url( + self, url: str, allowed_schemes: Optional[list] = None + ) -> Tuple[bool, Optional[str]]: + """ + NOTE: Not recursive — delegates to imported ``validate_url`` from ``app.utils.validation_utils``. + + Valide une URL. + + Args: + url: URL à valider + allowed_schemes: Schémas autorisés (défaut: http, https) + + Returns: + Tuple (is_valid, reason) + + Example: + >>> service.validate_url("https://example.com") + (True, None) + """ + return validate_url(url, allowed_schemes=allowed_schemes) + + # ==================== GENERAL VALIDATION ==================== + + def validate_length( + self, + input_str: str, + min_length: Optional[int] = None, + max_length: Optional[int] = None, + ) -> Tuple[bool, Optional[str]]: + """ + Valide la longueur d'une chaîne. + + Args: + input_str: Chaîne à valider + min_length: Longueur minimale + max_length: Longueur maximale + + Returns: + Tuple (is_valid, reason) + + Example: + >>> service.validate_length("Hello", min_length=3, max_length=10) + (True, None) + """ + return check_length_limits(input_str, min_length, max_length) + + def validate_pattern( + self, input_str: str, pattern: str, pattern_name: str = "pattern" + ) -> Tuple[bool, Optional[str]]: + """ + Valide contre un pattern regex. + + Args: + input_str: Chaîne à valider + pattern: Pattern regex + pattern_name: Nom du pattern + + Returns: + Tuple (is_valid, reason) + + Example: + >>> service.validate_pattern("ABC123", r"^[A-Z]{3}\\d{3}$", "code") + (True, None) + """ + return validate_pattern(input_str, pattern, pattern_name) + + def sanitize_filename(self, filename: str) -> str: + """ + Sanitize un nom de fichier. + + Args: + filename: Nom de fichier + + Returns: + Nom sanitizé + + Example: + >>> service.sanitize_filename("../../etc/passwd") + "etc_passwd" + """ + return sanitize_filename(filename) + + def normalize_text(self, text: str) -> str: + """ + Normalise le texte (whitespace). + + Args: + text: Texte à normaliser + + Returns: + Texte normalisé + + Example: + >>> service.normalize_text("Hello World") + "Hello World" + """ + return normalize_whitespace(text) + + # ==================== COMBINED VALIDATION ==================== + + def validate_input( + self, + input_str: str, + check_sql: bool = True, + check_xss: bool = True, + min_length: Optional[int] = None, + max_length: Optional[int] = None, + ) -> Tuple[bool, list]: + """ + Validation combinée d'une entrée. + + Args: + input_str: Entrée à valider + check_sql: Vérifier SQL injection + check_xss: Vérifier XSS + min_length: Longueur minimale + max_length: Longueur maximale + + Returns: + Tuple (is_valid, errors) + + Example: + >>> service.validate_input("Hello World", min_length=5, max_length=20) + (True, []) + """ + errors = [] + + # Check SQL injection + if check_sql: + is_safe, reason = self.validate_sql_input(input_str) + if not is_safe: + errors.append(f"SQL injection detected: {reason}") + + # Check XSS + if check_xss: + has_xss, pattern = self.check_xss(input_str) + if has_xss: + errors.append(f"XSS detected: {pattern}") + + # Check length + if min_length is not None or max_length is not None: + is_valid_length, reason = self.validate_length( + input_str, min_length, max_length + ) + if not is_valid_length: + errors.append(f"Length validation failed: {reason}") + + return len(errors) == 0, errors + + # ==================== HEALTH CHECK ==================== + + def health_check(self) -> Dict[str, Any]: + """ + Vérifie l'état de santé du service. + + Returns: + Dictionnaire avec statut + """ + with self.rate_limit_lock: + total_keys = len(self.rate_limit_data) + total_requests = sum(len(v) for v in self.rate_limit_data.values()) + + return { + "service": "ValidationService", + "status": "healthy", + "rate_limiting": { + "active_keys": total_keys, + "total_requests_tracked": total_requests, + }, + "features": { + "sql_injection_prevention": True, + "xss_prevention": True, + "rate_limiting": True, + "format_validation": True, + }, + } diff --git a/app/tasks/__init__.py b/app/tasks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6239b24a2af510cee6cb4c2225869531e6202656 --- /dev/null +++ b/app/tasks/__init__.py @@ -0,0 +1,15 @@ +""" +Celery tasks package for JuriX. + +Contains asynchronous tasks for: +- Document processing pipeline +- Background jobs +- Scheduled tasks + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +from app.core.celery_app import celery_app + +__all__ = ["celery_app"] diff --git a/app/tasks/analytics.py b/app/tasks/analytics.py new file mode 100644 index 0000000000000000000000000000000000000000..f641c73ad85fa3a13faaa765e042f5337e0ac712 --- /dev/null +++ b/app/tasks/analytics.py @@ -0,0 +1,250 @@ +""" +Celery tasks for persona analytics aggregation. + +Daily task to aggregate conversation and message data into persona_stats table. +Runs at 00:05 UTC daily to process previous day's data. +""" + +import asyncio +import logging +from datetime import date, datetime, timedelta +from typing import Optional + +from celery import Celery +from celery.schedules import crontab + +from app.core.database import AsyncSessionLocal +from app.services.persona_service import PersonaService + +# Configure logger +logger = logging.getLogger(__name__) + +# Initialize Celery app +# Note: This should be configured with your broker URL (Redis/RabbitMQ) +celery_app = Celery( + 'jurix_tasks', + broker='redis://localhost:6379/0', # Update with your broker URL + backend='redis://localhost:6379/0' +) + +# Celery configuration +celery_app.conf.update( + task_serializer='json', + accept_content=['json'], + result_serializer='json', + timezone='UTC', + enable_utc=True, +) + + +# ============================================================================ +# PERSONA ANALYTICS AGGREGATION TASK +# ============================================================================ + +@celery_app.task(name="aggregate_persona_stats", bind=True, max_retries=3) +def aggregate_persona_stats_task(self, target_date: Optional[str] = None): + """ + Aggregate persona statistics for a specific date. + + Processes conversation and message data to calculate: + - Usage metrics (questions, conversations, sessions) + - Performance metrics (confidence, response times) + - Engagement metrics (messages per conversation, session duration) + - Quality metrics (helpful/unhelpful feedback, satisfaction rate) + + Args: + target_date: Date to aggregate (YYYY-MM-DD). Defaults to yesterday. + + Returns: + Dict with aggregation summary + + Scheduled: Daily at 00:05 UTC via Celery Beat + + Example: + # Manual execution + aggregate_persona_stats_task.delay() + aggregate_persona_stats_task.delay("2024-01-15") + """ + try: + logger.info("📊 Starting persona stats aggregation task") + + # Parse target date + if target_date: + try: + date_obj = date.fromisoformat(target_date) + logger.info(f"📅 Aggregating data for: {date_obj}") + except ValueError as e: + logger.error(f"❌ Invalid date format: {target_date}") + raise ValueError(f"Invalid date format. Expected YYYY-MM-DD, got: {target_date}") + else: + # Default to yesterday + date_obj = date.today() - timedelta(days=1) + logger.info(f"📅 No date specified, aggregating yesterday: {date_obj}") + + # Run async aggregation + result = asyncio.run(_run_aggregation(date_obj)) + + logger.info(f"✅ Aggregation complete: {result}") + return result + + except Exception as e: + logger.error(f"❌ Aggregation task failed: {e}", exc_info=True) + # Retry with exponential backoff + raise self.retry(exc=e, countdown=60 * (2 ** self.request.retries)) + + +async def _run_aggregation(target_date: date) -> dict: + """ + Execute aggregation within async context. + + Args: + target_date: Date to aggregate + + Returns: + Dict with aggregation results + """ + start_time = datetime.utcnow() + + async with AsyncSessionLocal() as db: + try: + persona_service = PersonaService(db) + + # Aggregate stats for target date + await persona_service.aggregate_daily_stats(target_date) + + await db.commit() + + end_time = datetime.utcnow() + duration_ms = int((end_time - start_time).total_seconds() * 1000) + + logger.info(f"✅ Aggregation completed in {duration_ms}ms") + + return { + "status": "success", + "date": target_date.isoformat(), + "duration_ms": duration_ms, + "timestamp": end_time.isoformat() + } + + except Exception as e: + await db.rollback() + logger.error(f"❌ Aggregation failed: {e}") + raise + + +# ============================================================================ +# CELERY BEAT SCHEDULE +# ============================================================================ + +celery_app.conf.beat_schedule = { + 'aggregate-persona-stats-daily': { + 'task': 'aggregate_persona_stats', + 'schedule': crontab(hour=0, minute=5), # Run at 00:05 UTC daily + 'options': { + 'expires': 3600, # Task expires after 1 hour if not picked up + } + }, +} + + +# ============================================================================ +# MANUAL EXECUTION UTILITIES +# ============================================================================ + +def aggregate_date_range(start_date: date, end_date: date): + """ + Manually aggregate a range of dates (for backfilling). + + Args: + start_date: Start date (inclusive) + end_date: End date (inclusive) + + Example: + from app.tasks.analytics import aggregate_date_range + from datetime import date + + aggregate_date_range( + date(2024, 1, 1), + date(2024, 1, 31) + ) + """ + current_date = start_date + tasks = [] + + logger.info(f"📊 Backfilling persona stats from {start_date} to {end_date}") + + while current_date <= end_date: + logger.info(f"📅 Queuing aggregation for {current_date}") + task = aggregate_persona_stats_task.delay(current_date.isoformat()) + tasks.append(task) + current_date += timedelta(days=1) + + logger.info(f"✅ Queued {len(tasks)} aggregation tasks") + return tasks + + +def aggregate_yesterday(): + """ + Manually trigger aggregation for yesterday. + + Useful for testing or manual runs. + + Example: + from app.tasks.analytics import aggregate_yesterday + aggregate_yesterday() + """ + yesterday = date.today() - timedelta(days=1) + logger.info(f"📊 Manually aggregating data for {yesterday}") + return aggregate_persona_stats_task.delay(yesterday.isoformat()) + + +# ============================================================================ +# HEALTH CHECK +# ============================================================================ + +@celery_app.task(name="analytics_health_check") +def health_check_task(): + """ + Health check task for analytics worker. + + Returns: + Dict with worker status + """ + return { + "status": "healthy", + "service": "analytics_worker", + "timestamp": datetime.utcnow().isoformat() + } + + +# ============================================================================ +# TASK MONITORING +# ============================================================================ + +@celery_app.task(bind=True) +def debug_task(self): + """Debug task for testing Celery configuration.""" + logger.info(f"Request: {self.request!r}") + return { + "status": "ok", + "worker_id": self.request.id, + "hostname": self.request.hostname + } + + +if __name__ == "__main__": + """ + For testing/debugging aggregation locally without Celery. + + Run: python -m app.tasks.analytics + """ + import sys + + if len(sys.argv) > 1: + target = sys.argv[1] + else: + target = None + + print(f"🧪 Testing aggregation task locally") + result = aggregate_persona_stats_task(target) + print(f"✅ Result: {result}") diff --git a/app/tasks/clean_text_helper.py b/app/tasks/clean_text_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..845e48a7900c85b762d3b00013a5a069aabca55f --- /dev/null +++ b/app/tasks/clean_text_helper.py @@ -0,0 +1,32 @@ + +def _clean_extracted_text(text: str) -> str: + """ + Clean extracted text by merging lines that were split by PDF layout, + while preserving paragraph breaks. + """ + if not text: + return "" + + # 1. Replace potential double newlines (paragraphs) with a placeholder + # PDF paragraphs are often separated by empty lines (\n\n) + text = text.replace("\r\n", "\n") + + # Handle hyphenated words at end of line (e.g. "re-\nvolu-\ntion") + # Replace "-\n" with "" to join the word + text = text.replace("-\n", "") + + # 2. Heuristic: If we have \n\n, it's definitely a paragraph break. + # Preserve it with a special token. + text = text.replace("\n\n", "___PARAGRAPH___") + + # 3. Replace remaining single newlines with a space + text = text.replace("\n", " ") + + # 4. Collapse multiple spaces + import re + text = re.sub(r'\s+', ' ', text) + + # 5. Restore paragraphs + text = text.replace("___PARAGRAPH___", "\n\n") + + return text.strip() diff --git a/app/tasks/process_law.py b/app/tasks/process_law.py new file mode 100644 index 0000000000000000000000000000000000000000..6a21fedf27294e24f88839bbaa0a20080369b9ab --- /dev/null +++ b/app/tasks/process_law.py @@ -0,0 +1,1037 @@ +""" +Celery task for processing law documents. + +Pipeline workflow: +1. Extract text (OCR if needed) +2. Detect language +3. Classify category +4. Split articles +5. Generate embeddings +6. Index in Meilisearch +7. Update database + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +import logging +import time +from pathlib import Path +from typing import Any, Dict + +from celery import Task +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.celery_app import celery_app +from app.core.database import AsyncSessionLocal +from app.models.law import Article, Law +from app.utils.text_chunker import extract_articles, ArticleExtractionError + +logger = logging.getLogger(__name__) + + +class ProcessLawTask(Task): + """Custom task class with error handling.""" + + def on_failure(self, exc, task_id, args, kwargs, einfo): + """Handle task failure.""" + logger.error(f"❌ Task {task_id} failed: {exc}") + logger.error(f"Error info: {einfo}") + + +@celery_app.task(bind=True, base=ProcessLawTask, name="process_law") +def process_law(self, law_id: int, file_id: str = None) -> Dict[str, Any]: + """ + Process a law document. If file_id is provided, extracts text first. + + Orchestrates the pipeline: load → ingest → analyze → index → update. + Each step is delegated to focused helper functions. + """ + assert isinstance(law_id, int) and law_id > 0, "law_id must be a positive integer" + start_time = time.time() + errors = [] + + logger.info(f"🚀 Starting pipeline for law ID: {law_id} (file_id={file_id})") + + try: + # 1. Load law from database + self.update_state( + state="PROGRESS", meta={"step": "loading", "progress": 5, "message": "Loading document"} + ) + law = _load_law(law_id) + if not law: + raise ValueError(f"Law {law_id} not found") + logger.info(f"📄 Loaded law: {law.title}") + + # 2. Ingest file content if file_id provided + if file_id: + self.update_state( + state="PROGRESS", + meta={"step": "extracting_file", "progress": 8, "message": "Processing uploaded file"}, + ) + law, file_errors = _ingest_file_content(law_id, law, file_id) + errors.extend(file_errors) + + # 3. Validate extracted text + self.update_state( + state="PROGRESS", + meta={"step": "extracting", "progress": 10, "message": "Analyzing text content"}, + ) + text = law.content + if not text or len(text) < 50: + raise ValueError("Insufficient content for processing") + + # 4. Extract title if pending + extracted_title = None + if law.title and law.title.startswith("PENDING-"): + extracted_title = _extract_title_from_text(text) + if extracted_title: + logger.info(f"📝 Extracted title: {extracted_title}") + + # 5. Run analysis pipeline (language, category, articles, embeddings, search) + result = _run_analysis_pipeline(self, law_id, law, text, extracted_title) + result["errors"] = errors + result["duration"] = round(time.time() - start_time, 2) + + logger.info(f"✅ Pipeline completed for law {law_id} in {result['duration']}s") + return result + + except Exception as e: + logger.error(f"❌ Pipeline failed for law {law_id}: {e}", exc_info=True) + errors.append(str(e)) + return { + "law_id": law_id, + "status": "failed", + "errors": errors, + "duration": round(time.time() - start_time, 2), + } + + +# ==================== PIPELINE HELPERS ==================== + + +def _ingest_file_content(law_id: int, law, file_id: str): + """ + Extract text content from uploaded file (PDF or DOCX). + + Tries LlamaParse first, then OCR, then pypdf as fallback. + + Args: + law_id: Law database ID + law: Law ORM object (will be mutated with new content) + file_id: UUID of uploaded file + + Returns: + Tuple of (updated_law, errors_list) + """ + assert file_id, "file_id must not be empty" + assert law is not None, "law object must be provided" + + errors = [] + from app.services.file_upload_service import get_upload_service + upload_service = get_upload_service() + + # Locate file on disk + file_path = None + for ext in [".pdf", ".docx"]: + p = upload_service.storage_path / f"{file_id}{ext}" + if p.exists(): + file_path = p + break + + if not file_path: + logger.error(f"❌ File {file_id} not found in storage") + errors.append(f"File {file_id} not found") + return law, errors + + logger.info(f"📂 Found file: {file_path}") + extracted_text = "" + + if file_path.suffix == ".pdf": + extracted_text, pdf_errors = _extract_pdf_text(file_path) + errors.extend(pdf_errors) + elif file_path.suffix == ".docx": + extracted_text, docx_errors = _extract_docx_text(file_path) + errors.extend(docx_errors) + + if extracted_text and len(extracted_text) > 50: + _update_law_content(law_id, extracted_text) + law.content = extracted_text + logger.info(f"✅ Extracted {len(extracted_text)} chars from file") + else: + logger.warning("⚠️ No text extracted or text too short") + + return law, errors + + +def _extract_pdf_text(file_path) -> tuple: + """Extract text from PDF using LlamaParse → OCR → pypdf fallback chain.""" + import asyncio + from app.core.config import settings + + try: + from app.services.llama_parse_service import LlamaParseService, LlamaParseError + logger.info(f"📄 Processing PDF with LlamaParse: {file_path.name}") + llama_service = LlamaParseService() + + if llama_service.is_available(): + text = asyncio.run(llama_service.extract_text(file_path)) + logger.info(f"✅ LlamaParse Complete: {len(text)} chars (clean)") + return text, [] + raise LlamaParseError("LlamaParse not configured") + + except Exception as e: + logger.warning(f"LlamaParse failed ({e}), trying OCR fallback...") + return _extract_pdf_ocr_fallback(file_path, settings) + + +def _extract_pdf_ocr_fallback(file_path, settings) -> tuple: + """OCR then pypdf fallback for PDF text extraction.""" + import asyncio + + try: + from app.services.ocr_service import OCRService + ocr_service = OCRService(tesseract_path=settings.TESSERACT_PATH) + ocr_result = asyncio.run(ocr_service.process_pdf(file_path)) + logger.info(f"✅ OCR Fallback: {len(ocr_result.text)} chars") + return ocr_result.text, [] + except Exception as ocr_e: + logger.error(f"OCR failed ({ocr_e}), reverting to pypdf fallback...") + from pypdf import PdfReader + reader = PdfReader(file_path) + pages_text = [] + for page_num, page in enumerate(reader.pages, start=1): + pages_text.append(f"<>\n{page.extract_text()}") + text = _clean_extracted_text("\n".join(pages_text)) + logger.info(f"✅ pypdf Fallback: {len(text)} chars from {len(reader.pages)} pages") + return text, [] + + +def _extract_docx_text(file_path) -> tuple: + """Extract text from DOCX file.""" + try: + import docx + doc = docx.Document(file_path) + text = "\n".join([p.text for p in doc.paragraphs]) + return _clean_extracted_text(text), [] + except Exception as e: + logger.error(f"DOCX extraction failed: {e}") + return "", [f"DOCX extraction failed: {str(e)}"] + + +def _run_analysis_pipeline(task, law_id: int, law, text: str, extracted_title=None): + """ + Run the full analysis pipeline: language → category → articles → embeddings → index. + + Args: + task: Celery task (for progress updates) + law_id: Law database ID + law: Law ORM object + text: Extracted text content + extracted_title: Optional title extracted from text + + Returns: + Result dict with analysis outcomes + """ + assert text and len(text) >= 50, "Text must be at least 50 chars" + assert isinstance(law_id, int) and law_id > 0, "law_id must be positive" + + # Language detection + task.update_state( + state="PROGRESS", meta={"step": "detecting", "progress": 30, "message": "Detecting language"} + ) + language_result = _detect_language(text) + logger.info( + f"🌍 Language: {language_result['language']} " + f"(confidence: {language_result['confidence']:.2%})" + ) + + # Category classification + task.update_state( + state="PROGRESS", meta={"step": "classifying", "progress": 45, "message": "Classifying category"} + ) + category_result = _classify_category(text) + logger.info( + f"📂 Category: {category_result['category']} " + f"(confidence: {category_result['confidence']:.2%})" + ) + + # Split and save articles + task.update_state( + state="PROGRESS", meta={"step": "splitting", "progress": 60, "message": "Splitting articles"} + ) + articles_count = _split_and_save_articles(law_id, text) + logger.info(f"📑 Articles extracted: {articles_count}") + + # Generate embeddings + task.update_state( + state="PROGRESS", meta={"step": "embeddings", "progress": 75, "message": "Generating embeddings"} + ) + embeddings_count = _generate_article_embeddings(law_id) + logger.info(f"🔢 Embeddings generated: {embeddings_count}") + + # Index in Meilisearch + task.update_state( + state="PROGRESS", meta={"step": "indexing", "progress": 85, "message": "Indexing in search"} + ) + search_indexed = _index_in_search(law_id, law) + logger.info(f"🔍 Search indexed: {search_indexed}") + + # Update metadata + task.update_state( + state="PROGRESS", meta={"step": "updating", "progress": 95, "message": "Updating database"} + ) + _update_law_metadata( + law_id, + language=language_result["language"], + language_confidence=language_result["confidence"], + category=category_result["category"], + category_confidence=category_result["confidence"], + title=extracted_title, + ) + + return { + "law_id": law_id, + "status": "completed", + "language": language_result["language"], + "language_confidence": language_result["confidence"], + "category": category_result["category"], + "category_confidence": category_result["confidence"], + "articles_count": articles_count, + } + + +# ==================== HELPER FUNCTIONS ==================== + + +def _extract_text_from_file(file_id: str) -> str: + """ + DEPRECATED: Use _extract_text_with_ocr instead. + + This stub redirects to _extract_text_with_ocr which properly prioritises + LlamaParse → OCR → pypdf fallback. + + Args: + file_id: UUID of uploaded file + + Returns: + Extracted text content + """ + logger.warning( + "_extract_text_from_file is deprecated. " + "Delegating to _extract_text_with_ocr (LlamaParse-first)." + ) + return _extract_text_with_ocr(file_id) + + +def _load_law(law_id: int) -> Law: + """Load law from database (synchronous).""" + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + from app.core.config import settings + + # Create synchronous engine for this operation + sync_engine = create_engine(settings.DATABASE_URL.replace('+asyncpg', '')) + Session = sessionmaker(bind=sync_engine) + + with Session() as session: + law = session.query(Law).filter(Law.id == law_id).first() + if law: + # Eagerly load relationships + _ = law.articles # Trigger lazy load + return law + + +def _detect_language(text: str) -> Dict[str, Any]: + """ + Detect language using LanguageDetector. + """ + try: + from app.services.language_detector import LanguageDetector + + detector = LanguageDetector() + result = detector.detect(text) + return {"language": result["language"], "confidence": result["confidence"]} + except Exception as e: + logger.warning(f"⚠️ Language detection failed, using fallback: {e}") + # Fallback to simple heuristic + if any(word in text.lower() for word in ["le", "la", "les", "de", "et", "article"]): + return {"language": "fr", "confidence": 0.75} + else: + return {"language": "en", "confidence": 0.75} + + +def _classify_category(text: str) -> Dict[str, Any]: + """ + Classify document category using DocumentClassifier. + """ + try: + from app.services.document_classifier import DocumentClassifier + + classifier = DocumentClassifier() + # classifier.classify returns a list of tuples: [(category_id, confidence, method), ...] + results = classifier.classify(text) + + if results: + cat_id, confidence, _ = results[0] + # Map ID to name if needed, though ProcessLaw expects name in result dict? + # Wait, process_law main function logs result['category']. + # And _update_law_metadata uses result['category']. + # In DocumentClassifier, verify what 'category' means. + # get_category_name(cat_id) returns the string name. + category_name = classifier.get_category_name(cat_id) + return {"category": category_name, "confidence": confidence} + + return {"category": "Autre", "confidence": 0.0} + + except Exception as e: + logger.warning(f"⚠️ Category classification failed, using fallback: {e}") + # Fallback to simple heuristic + if "fiscal" in text.lower() or "impôt" in text.lower(): + return {"category": "Droit Fiscal", "confidence": 0.70} + elif "pénal" in text.lower() or "crime" in text.lower(): + return {"category": "Droit Pénal", "confidence": 0.70} + else: + return {"category": "Droit Civil", "confidence": 0.70} + + +def _split_and_save_articles(law_id: int, text: str) -> int: + """ + Extract articles from law text and save them to database. + + Args: + law_id: Law database ID + text: Full law text content + + Returns: + Number of articles extracted and saved + """ + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + from app.core.config import settings + + logger.info(f"📑 Extracting articles from law {law_id}") + + try: + # Extract articles using text_chunker (strict=False allows fewer than 3 articles) + # min_article_length=1 ensures ALL articles are extracted, even very short ones + extracted = extract_articles(text, strict=False, min_article_length=1) + + if not extracted: + logger.warning(f"⚠️ No articles extracted from law {law_id}") + return 0 + + logger.info(f"📋 Found {len(extracted)} articles") + + # Create sync database session + sync_engine = create_engine(settings.DATABASE_URL.replace('+asyncpg', '')) + Session = sessionmaker(bind=sync_engine) + + with Session() as session: + # Delete existing articles for this law (in case of re-processing) + session.query(Article).filter(Article.law_id == law_id).delete() + + # Create Article objects + for article_data in extracted: + article = Article( + law_id=law_id, + number=str(article_data.get('number', '')), + title=article_data.get('title'), + section=article_data.get('section'), # TITRE/CHAPITRE section header + content=article_data.get('content', ''), + order=article_data.get('position', 0), + page_number=article_data.get('page_number'), # Save page number for navigation + # NOTE: embedding column is pgvector type in DB, leaving NULL for now + ) + session.add(article) + logger.debug(f"📄 Created Article {article.number}: section={article.section or 'None'} (page {article.page_number})") + + session.commit() + logger.info(f"✅ Saved {len(extracted)} articles for law {law_id}") + + return len(extracted) + + except ArticleExtractionError as e: + logger.warning(f"⚠️ Article extraction failed: {e}") + return 0 + except Exception as e: + logger.error(f"❌ Error saving articles: {e}", exc_info=True) + return 0 + + +def _generate_article_embeddings(law_id: int) -> int: + """ + Generate embeddings for all articles/chunks of a law using Gemini API. + + This function processes ALL chunks created by the text extraction: + - Legal basis (LEGAL_BASIS) + - Preamble (PREAMBULE) + - Articles (1, 2, 3...) + - Paragraphs (PARA_1, PARA_2...) for non-article documents + - Full text (FULL_TEXT) for unstructured docs + + Args: + law_id: Law database ID + + Returns: + Number of chunks with embeddings generated successfully + """ + assert isinstance(law_id, int) and law_id > 0, "law_id must be a positive integer" + assert law_id < 10_000_000, "law_id seems unreasonably large" + + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + from app.core.config import settings + from app.services.embedding_service import EmbeddingService, EmbeddingServiceError + + logger.info(f"🔢 Generating embeddings for law {law_id} chunks...") + + try: + # Initialize Gemini embedding service + embedding_service = EmbeddingService() + + # Connect to database + engine = create_engine(settings.DATABASE_URL.replace('+asyncpg', ''), pool_pre_ping=True) + Session = sessionmaker(bind=engine) + session = Session() + + try: + # Get all articles/chunks for this law + articles = session.query(Article).filter_by(law_id=law_id).all() + + if not articles: + logger.warning(f"⚠️ No articles found for law {law_id}") + return 0 + + logger.info(f"📄 Found {len(articles)} chunks to process") + + # Prepare texts for batch embedding generation + texts = [article.content for article in articles] + article_ids = [article.id for article in articles] + + # Generate embeddings in batch for efficiency + logger.info(f"🚀 Generating {len(texts)} embeddings via Gemini API...") + embeddings = embedding_service.generate_batch_embeddings( + texts=texts, + normalize=True # L2 normalization for cosine similarity + ) + + # Update articles with embeddings + success_count = 0 + for article, embedding in zip(articles, embeddings): + try: + # Convert numpy array to list for pgvector storage + article.embedding = embedding.tolist() + success_count += 1 + logger.debug( + f"✅ Generated embedding for {article.number}: " + f"{len(embedding)} dims" + ) + except Exception as e: + logger.error( + f"❌ Failed to save embedding for article {article.number}: {e}" + ) + + # Commit all updates + session.commit() + + logger.info( + f"✅ Successfully generated {success_count}/{len(articles)} embeddings " + f"for law {law_id}" + ) + + return success_count + + finally: + session.close() + + except EmbeddingServiceError as e: + logger.error(f"❌ Embedding service error: {e}") + return 0 + except Exception as e: + logger.error(f"❌ Error generating embeddings: {e}", exc_info=True) + return 0 + + +def _index_in_search(law_id: int, law: Law) -> bool: + """ + Index law AND its articles in Meilisearch for full-text search (synchronous). + + Returns: + True if indexed successfully, False otherwise + """ + try: + import meilisearch + from sqlalchemy import create_engine, text + from app.core.config import settings + + # Initialize Meilisearch client (synchronous) + client = meilisearch.Client(settings.MEILISEARCH_URL, settings.MEILISEARCH_KEY) + + # 1. Index the law in 'laws' index + laws_index = client.index("laws") + law_document = { + "id": law.id, + "law_id": law.id, # Explicit law_id for RAG service + "reference": law.reference, + "title": law.title, + "content": law.content[:50000] if law.content else "", # Truncate for search + "type": law.type, + "language": law.language or "unknown", + "status": law.status, + "category_id": law.category_id, + "category_name": law.category.name if law.category else None, + "publication_year": ( + law.publication_date.year if law.publication_date else None + ), + "created_at_timestamp": int(law.created_at.timestamp()) + } + laws_index.add_documents([law_document]) + logger.info(f"✅ Indexed law {law_id} in Meilisearch 'laws' index") + + # 2. Index articles in 'articles' index for article-level search + try: + articles_index = client.index("articles") + + # Ensure index exists with proper settings + try: + client.create_index('articles', {'primaryKey': 'id'}) + except Exception: + pass # Already exists + + # Fetch articles from database + sync_url = settings.DATABASE_URL.replace('+asyncpg', '') + engine = create_engine(sync_url) + + with engine.connect() as conn: + result = conn.execute(text(""" + SELECT id, number, title, section, content + FROM articles WHERE law_id = :law_id + """), {"law_id": law_id}) + + articles_docs = [] + for row in result: + articles_docs.append({ + 'id': row[0], + 'number': row[1] or '', + 'title': row[2] or '', + 'section': row[3] or '', + 'content': row[4] or '', + 'law_id': law_id, + 'law_title': law.title, + 'law_reference': law.reference, + 'category': law.category.name if law.category else 'Non catégorisé' + }) + + if articles_docs: + articles_index.add_documents(articles_docs) + logger.info(f"✅ Indexed {len(articles_docs)} articles for law {law_id} in Meilisearch 'articles' index") + + except Exception as articles_err: + logger.warning(f"⚠️ Articles indexing failed: {articles_err}") + + return True + + except Exception as e: + logger.warning(f"⚠️ Search indexing failed: {e}") + return False + + +def delete_from_meilisearch(law_id: int) -> bool: + """ + Delete a law AND its articles from Meilisearch indexes. + + Call this when a law is deleted from the database. + + Args: + law_id: ID of the law to delete + + Returns: + True if deleted successfully, False otherwise + """ + try: + import meilisearch + from app.core.config import settings + + client = meilisearch.Client(settings.MEILISEARCH_URL, settings.MEILISEARCH_KEY) + + # 1. Delete articles first + try: + articles_index = client.index("articles") + + # Find all articles for this law + search_result = articles_index.search('', { + 'filter': f'law_id = {law_id}', + 'limit': 1000, + 'attributesToRetrieve': ['id'] + }) + + article_ids = [hit['id'] for hit in search_result.get('hits', [])] + + if article_ids: + articles_index.delete_documents(article_ids) + logger.info(f"🗑️ Deleted {len(article_ids)} articles for law {law_id} from Meilisearch") + except Exception as e: + logger.warning(f"⚠️ Articles deletion failed: {e}") + + # 2. Delete law + laws_index = client.index("laws") + laws_index.delete_document(law_id) + logger.info(f"🗑️ Deleted law {law_id} from Meilisearch 'laws' index") + + return True + + except Exception as e: + logger.warning(f"⚠️ Meilisearch delete failed for law {law_id}: {e}") + return False + + +def _update_law_metadata( + law_id: int, + language: str, + language_confidence: float, + category: str, + category_confidence: float, + title: str = None, +) -> None: + """Update law metadata in database (synchronous).""" + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + from app.core.config import settings + + sync_engine = create_engine(settings.DATABASE_URL.replace('+asyncpg', '')) + Session = sessionmaker(bind=sync_engine) + + with Session() as session: + law = session.query(Law).filter(Law.id == law_id).first() + if law: + law.language = language # Set the main language field + law.detected_language = language + law.language_confidence = language_confidence + if title: # Update title if provided + law.title = title + logger.info(f"📝 Updated title to: {title}") + session.commit() + logger.info(f"✅ Updated metadata for law {law_id}") + + +def _extract_title_from_text(text: str) -> str: + """ + Extract document title from text content. + + Looks for title in first few lines, handling common patterns: + - "LAW NO. 96/06 OF 18 JANUARY 1996..." + - "LOI N° 96/06 DU 18 JANVIER 1996..." + - "THE CONSTITUTION OF..." + - "LA CONSTITUTION DU..." + + Returns: + Extracted title or None if not found + """ + if not text: + return None + + # Get first 500 characters (usually contains title) + header = text[:500].strip() + + # Split into lines + lines = [line.strip() for line in header.split('\n') if line.strip()] + + if not lines: + return None + + # Common patterns for legal documents + title_patterns = [ + r'^(LAW\s+N[Ooذ]\.?\s*\d+[/-]\d+.*?)(?:\n|$)', # English: LAW NO. 96/06... + r'^(LOI\s+N[Ooذ]\.?\s*\d+[/-]\d+.*?)(?:\n|$)', # French: LOI N° 96/06... + r'^(THE\s+CONSTITUTION.*?)(?:\n|$)', # English: THE CONSTITUTION... + r'^(LA\s+CONSTITUTION.*?)(?:\n|$)', # French: LA CONSTITUTION... + r'^(CONSTITUTION.*?)(?:\n|$)', # Generic: CONSTITUTION... + r'^(DECREE\s+N[Ooذ]\.?\s*\d+.*?)(?:\n|$)', # English: DECREE NO... + r'^(DÉCRET\s+N[Ooذ]\.?\s*\d+.*?)(?:\n|$)', # French: DÉCRET N°... + ] + + import re + + # Try to match patterns in first 3 lines + for line in lines[:3]: + for pattern in title_patterns: + match = re.search(pattern, line, re.IGNORECASE) + if match: + title = match.group(1).strip() + # Clean up common OCR artifacts + title = re.sub(r'\s+', ' ', title) # Normalize whitespace + title = title[:200] # Limit length + return title + + # Fallback: Use first non-empty line if it looks like a title + first_line = lines[0] + if len(first_line) > 10 and len(first_line) < 200: + # Check if it's all caps or starts with capital (likely a title) + if first_line.isupper() or first_line[0].isupper(): + return first_line[:200] + + return None + + +def _update_law_content(law_id: int, content: str) -> None: + """Update law content in database (synchronous).""" + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + from app.core.config import settings + + sync_engine = create_engine(settings.DATABASE_URL.replace('+asyncpg', '')) + Session = sessionmaker(bind=sync_engine) + + with Session() as session: + law = session.query(Law).filter(Law.id == law_id).first() + if law: + law.content = content + session.commit() + logger.info(f"✅ Updated content for law {law_id}") + + +# ==================== SYNCHRONOUS VERSION (Windows Fallback) ==================== + + + +def _extract_text_with_ocr(file_id: str) -> str: + """ + Extract text from file using OCRService (Sync wrapper). + Handles PDF (native/scanned) and DOCX. + """ + try: + from app.services.file_upload_service import get_upload_service + from app.core.config import settings + + upload_service = get_upload_service() + + # Locate file + file_path = None + for ext in [".pdf", ".docx"]: + p = upload_service.storage_path / f"{file_id}{ext}" + if p.exists(): + file_path = p + break + + if not file_path: + logger.error(f"❌ File {file_id} not found") + return "" + + if file_path.suffix.lower() == ".pdf": + try: + import asyncio + from app.services.llama_parse_service import LlamaParseService, LlamaParseError + + logger.info(f"📄 Processing PDF with LlamaParse: {file_path.name}") + llama_service = LlamaParseService() + + if llama_service.is_available(): + extracted_text = asyncio.run(llama_service.extract_text(file_path)) + logger.info(f"✅ LlamaParse Complete: {len(extracted_text)} chars (clean)") + return extracted_text + else: + raise LlamaParseError("LlamaParse not configured") + + except Exception as e: + logger.warning(f"LlamaParse failed ({e}), trying OCR fallback...") + try: + from app.services.ocr_service import OCRService + ocr_service = OCRService(tesseract_path=settings.TESSERACT_PATH) + ocr_result = asyncio.run(ocr_service.process_pdf(file_path)) + logger.info(f"✅ OCR Fallback: {len(ocr_result.text)} chars") + return ocr_result.text + except Exception as ocr_e: + logger.error(f"OCR failed ({ocr_e}), using pypdf fallback") + from pypdf import PdfReader + reader = PdfReader(file_path) + pages_text = [] + for page_num, page in enumerate(reader.pages, start=1): + page_text = f"<>\n{page.extract_text()}" + pages_text.append(page_text) + text = "\n".join(pages_text) + return _clean_extracted_text(text) + + elif file_path.suffix.lower() == ".docx": + import docx + doc = docx.Document(file_path) + text = "\n".join([p.text for p in doc.paragraphs]) + return _clean_extracted_text(text) + + return "" + + except Exception as e: + logger.error(f"Extraction error: {e}") + return "" + +def process_law_sync(law_id: int, file_id: str) -> Dict[str, Any]: + """ + Synchronous version of process_law for Windows compatibility. + """ + assert isinstance(law_id, int) and law_id > 0, "law_id must be a positive integer" + assert isinstance(file_id, str) and len(file_id) > 0, "file_id must be a non-empty string" + + import asyncio + + logger.info(f"🔄 Starting synchronous processing for law {law_id}, file {file_id}") + start_time = time.time() + errors = [] + + try: + # Load law from database + law = _load_law(law_id) + if not law: + error_msg = f"Law {law_id} not found in database" + logger.error(f"❌ {error_msg}") + return {"law_id": law_id, "status": "failed", "errors": [error_msg]} + + logger.info(f"📄 Processing law: {law.title} (ID={law_id})") + + # 1. Extract text from uploaded file + logger.info(f"📄 Extracting text from file {file_id}") + # USE NEW OCR FUNCTION HERE + text = _extract_text_with_ocr(file_id) + + if text and len(text) >= 50: + _update_law_content(law_id, text) + logger.info(f"✅ Extracted {len(text)} characters") + else: + text = law.content + if not text or len(text) < 50: + raise ValueError("Insufficient content for processing") + + # 2. Detect language + logger.info(f"🌍 Detecting language") + language_result = _detect_language(text) + logger.info( + f"🌍 Language detected: {language_result['language']} " + f"(confidence: {language_result['confidence']:.2%})" + ) + + # 3. Classify category + logger.info(f"📂 Classifying category") + category_result = _classify_category(text) + logger.info( + f"📂 Category classified: {category_result['category']} " + f"(confidence: {category_result['confidence']:.2%})" + ) + + # 4. Extract and save articles + print(f"DEBUG >>> About to extract articles for law {law_id}") + logger.info(f"📑 Extracting articles") + articles_count = _split_and_save_articles(law_id, text) + print(f"DEBUG >>> Articles extracted: {articles_count}") + logger.info(f"📑 Articles extracted: {articles_count}") + + # 5. Generate embeddings + print(f"DEBUG >>> About to generate embeddings for law {law_id}") + logger.info(f"🔢 Generating embeddings") + try: + embeddings_count = _generate_article_embeddings(law_id) + print(f"DEBUG >>> Embeddings generated: {embeddings_count}") + except Exception as emb_error: + print(f"DEBUG >>> Embedding error: {emb_error}") + embeddings_count = 0 + logger.info(f"🔢 Embeddings generated: {embeddings_count}") + + # 6. Index in search + logger.info(f"🔍 Indexing in search") + search_indexed = _index_in_search(law_id, law) + logger.info(f"🔍 Search indexed: {search_indexed}") + + # 7. Update database with results + logger.info(f"💾 Updating database metadata") + _update_law_metadata( + law_id, + language=language_result["language"], + language_confidence=language_result["confidence"], + category=category_result["category"], + category_confidence=category_result["confidence"], + ) + + # Calculate duration + duration = time.time() - start_time + + # Final result + result = { + "law_id": law_id, + "status": "completed", + "language": language_result["language"], + "language_confidence": language_result["confidence"], + "category": category_result["category"], + "category_confidence": category_result["confidence"], + "articles_count": articles_count, + "embeddings_generated": embeddings_count, + "search_indexed": search_indexed, + "duration": round(duration, 2), + "errors": errors, + } + + logger.info(f"✅ Synchronous processing completed for law {law_id} in {duration:.2f}s") + return result + + except Exception as e: + logger.error(f"❌ Synchronous processing failed for law {law_id}: {e}", exc_info=True) + errors.append(str(e)) + + return { + "law_id": law_id, + "status": "failed", + "errors": errors, + "duration": round(time.time() - start_time, 2), + } + + +def _clean_extracted_text(text: str) -> str: + """ + Clean extracted text with MINIMAL modifications. + + This "gentle" approach preserves the original document structure: + - Line breaks are kept (important for legal document formatting) + - Only obvious PDF artifacts are removed + - Hyphenated words are merged only when appropriate + + Args: + text: Raw extracted text from PDF/DOCX + + Returns: + Cleaned text with preserved formatting + """ + if not text: + return "" + + import re + + # 1. Normalize line endings (Windows to Unix) + text = text.replace("\r\n", "\n") + + # 2. Handle hyphenated words at end of line CAREFULLY + # Only merge if next line starts with lowercase (indicates word continuation) + text = re.sub(r'-\n([a-zàâçéèêëïîôùûüœæ])', r'\1', text) + + # 3. Handle French elision at end of line + text = re.sub(r"([ldnsjcmtLDNSJCMT])'\n([A-Za-zÀ-ÿ])", r"\1'\2", text) + text = re.sub(r"([ldnsjcmtLDNSJCMT])’\n([A-Za-zÀ-ÿ])", r"\1'\2", text) + text = re.sub(r"(qu|Qu)'\n([A-Za-zÀ-ÿ])", r"\1'\2", text) + + # 4. Remove isolated page numbers (1-3 digits alone on a line) + text = re.sub(r'^\s*\d{1,3}\s*$', '', text, flags=re.MULTILINE) + + # 5. Remove page markers but preserve surrounding structure + text = re.sub(r'<>\n?', '', text) + + # 6. Collapse excessive blank lines (3+ consecutive to 2) + text = re.sub(r'\n{3,}', '\n\n', text) + + # 7. Remove trailing spaces on each line + text = re.sub(r'[ \t]+$', '', text, flags=re.MULTILINE) + + # 8. Limit leading spaces to 4 max + text = re.sub(r'^[ \t]{5,}', ' ', text, flags=re.MULTILINE) + + return text.strip() diff --git a/app/utils/__init__.py b/app/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/app/utils/file_utils.py b/app/utils/file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..153194896881ad9f834b59d6a719c626e169b221 --- /dev/null +++ b/app/utils/file_utils.py @@ -0,0 +1,355 @@ +""" +Utility functions for file operations. + +Provides helpers for: +- File hashing (SHA-256) +- File type detection (magic bytes) +- File validation (PDF, DOCX structure) +- Filename generation (UUID-based) +- Storage management + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +import hashlib +import logging +import uuid +from pathlib import Path +from typing import Optional, Tuple + +logger = logging.getLogger(__name__) + + +# File type magic bytes signatures +MAGIC_BYTES = { + "pdf": b"%PDF", + "docx": b"PK\x03\x04", # ZIP signature (DOCX is a ZIP archive) +} + +# MIME types +MIME_TYPES = { + "pdf": "application/pdf", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", +} + + +def get_file_hash(file_path: Path, algorithm: str = "sha256") -> str: + """ + Calculate hash of a file. + + Args: + file_path: Path to file + algorithm: Hash algorithm (default: sha256) + + Returns: + Hex digest of file hash + + Raises: + FileNotFoundError: If file doesn't exist + ValueError: If algorithm not supported + + Example: + >>> hash_value = get_file_hash(Path("document.pdf")) + >>> len(hash_value) + 64 + """ + if not file_path.exists(): + raise FileNotFoundError(f"File not found: {file_path}") + + if algorithm not in hashlib.algorithms_available: + raise ValueError(f"Hash algorithm '{algorithm}' not supported") + + hash_obj = hashlib.new(algorithm) + + # Read file in chunks to handle large files + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + hash_obj.update(chunk) + + return hash_obj.hexdigest() + + +def get_magic_bytes(file_path: Path, num_bytes: int = 4) -> bytes: + """ + Read the first N bytes of a file (magic bytes). + + Args: + file_path: Path to file + num_bytes: Number of bytes to read (default: 4) + + Returns: + First N bytes of file + + Raises: + FileNotFoundError: If file doesn't exist + + Example: + >>> magic = get_magic_bytes(Path("document.pdf")) + >>> magic + b'%PDF' + """ + if not file_path.exists(): + raise FileNotFoundError(f"File not found: {file_path}") + + with open(file_path, "rb") as f: + return f.read(num_bytes) + + +def detect_file_type(file_path: Path) -> Optional[str]: + """ + Detect file type from magic bytes. + + Args: + file_path: Path to file + + Returns: + File type ('pdf' or 'docx') or None if unknown + + Example: + >>> detect_file_type(Path("document.pdf")) + 'pdf' + """ + try: + magic = get_magic_bytes(file_path, num_bytes=4) + + for file_type, signature in MAGIC_BYTES.items(): + if magic.startswith(signature): + return file_type + + return None + except Exception as e: + logger.warning(f"Failed to detect file type for {file_path}: {e}") + return None + + +def is_valid_pdf(file_path: Path) -> Tuple[bool, Optional[str]]: + """ + Validate PDF file structure. + + Checks: + - Magic bytes (%PDF) + - File ends with %%EOF + - Minimum file size + + Args: + file_path: Path to PDF file + + Returns: + Tuple of (is_valid, error_message) + + Example: + >>> is_valid, error = is_valid_pdf(Path("document.pdf")) + >>> is_valid + True + """ + try: + # Check file exists and has minimum size + if not file_path.exists(): + return False, "File does not exist" + + file_size = file_path.stat().st_size + if file_size < 100: # Minimum PDF size + return False, "File too small to be a valid PDF" + + # Check magic bytes + magic = get_magic_bytes(file_path, num_bytes=4) + if not magic.startswith(b"%PDF"): + return False, "Invalid PDF magic bytes" + + # Check EOF marker (read last 1024 bytes) + with open(file_path, "rb") as f: + f.seek(max(0, file_size - 1024)) + tail = f.read() + if b"%%EOF" not in tail: + return False, "Missing PDF EOF marker" + + return True, None + + except Exception as e: + return False, f"PDF validation error: {str(e)}" + + +def is_valid_docx(file_path: Path) -> Tuple[bool, Optional[str]]: + """ + Validate DOCX file structure. + + Checks: + - Magic bytes (PK - ZIP signature) + - Contains required DOCX files (word/document.xml) + - Minimum file size + + Args: + file_path: Path to DOCX file + + Returns: + Tuple of (is_valid, error_message) + + Example: + >>> is_valid, error = is_valid_docx(Path("document.docx")) + >>> is_valid + True + """ + try: + # Check file exists and has minimum size + if not file_path.exists(): + return False, "File does not exist" + + file_size = file_path.stat().st_size + if file_size < 1000: # Minimum DOCX size + return False, "File too small to be a valid DOCX" + + # Check magic bytes (ZIP signature) + magic = get_magic_bytes(file_path, num_bytes=4) + if not magic.startswith(b"PK\x03\x04"): + return False, "Invalid DOCX magic bytes (not a ZIP file)" + + # Try to open as ZIP and check for required DOCX structure + import zipfile + + if not zipfile.is_zipfile(file_path): + return False, "File is not a valid ZIP archive" + + with zipfile.ZipFile(file_path, "r") as zip_file: + # Check for required DOCX files + required_files = ["[Content_Types].xml", "word/document.xml"] + zip_contents = zip_file.namelist() + + for required_file in required_files: + if required_file not in zip_contents: + return False, f"Missing required DOCX file: {required_file}" + + return True, None + + except zipfile.BadZipFile: + return False, "Corrupted ZIP/DOCX file" + except Exception as e: + return False, f"DOCX validation error: {str(e)}" + + +def generate_unique_filename(original_filename: str, preserve_extension: bool = True) -> str: + """ + Generate a unique filename using UUID. + + Args: + original_filename: Original filename + preserve_extension: Whether to keep original extension (default: True) + + Returns: + Unique filename (UUID-based) + + Example: + >>> filename = generate_unique_filename("document.pdf") + >>> filename.endswith(".pdf") + True + >>> len(filename.split(".")[0]) + 36 # UUID length + """ + unique_id = str(uuid.uuid4()) + + if preserve_extension: + # Extract extension from original filename + extension = Path(original_filename).suffix + return f"{unique_id}{extension}" + else: + return unique_id + + +def ensure_storage_directory(path: Path) -> None: + """ + Ensure storage directory exists, create if necessary. + + Args: + path: Directory path + + Raises: + PermissionError: If directory cannot be created + OSError: If path exists but is not a directory + + Example: + >>> ensure_storage_directory(Path("/tmp/uploads")) + """ + if path.exists(): + if not path.is_dir(): + raise OSError(f"Path exists but is not a directory: {path}") + logger.debug(f"Storage directory already exists: {path}") + else: + path.mkdir(parents=True, exist_ok=True) + logger.info(f"Created storage directory: {path}") + + +def get_file_size_mb(file_path: Path) -> float: + """ + Get file size in megabytes. + + Args: + file_path: Path to file + + Returns: + File size in MB (rounded to 2 decimals) + + Example: + >>> size_mb = get_file_size_mb(Path("document.pdf")) + >>> size_mb + 2.5 + """ + if not file_path.exists(): + raise FileNotFoundError(f"File not found: {file_path}") + + size_bytes = file_path.stat().st_size + size_mb = size_bytes / (1024 * 1024) + return round(size_mb, 2) + + +def is_file_size_valid(file_path: Path, max_size_mb: int = 50) -> Tuple[bool, Optional[str]]: + """ + Check if file size is within allowed limit. + + Args: + file_path: Path to file + max_size_mb: Maximum allowed size in MB (default: 50) + + Returns: + Tuple of (is_valid, error_message) + + Example: + >>> is_valid, error = is_file_size_valid(Path("document.pdf"), max_size_mb=50) + >>> is_valid + True + """ + try: + size_mb = get_file_size_mb(file_path) + + if size_mb > max_size_mb: + return False, f"File size {size_mb} MB exceeds maximum {max_size_mb} MB" + + if size_mb == 0: + return False, "File is empty (0 bytes)" + + return True, None + + except Exception as e: + return False, f"Size validation error: {str(e)}" + + +def get_mime_type(file_type: str) -> str: + """ + Get MIME type for file type. + + Args: + file_type: File type ('pdf' or 'docx') + + Returns: + MIME type string + + Raises: + ValueError: If file type not supported + + Example: + >>> get_mime_type("pdf") + 'application/pdf' + """ + if file_type not in MIME_TYPES: + raise ValueError(f"Unsupported file type: {file_type}") + + return MIME_TYPES[file_type] diff --git a/app/utils/ocr_utils.py b/app/utils/ocr_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2d9228b8f04d5b2c545d7bdf9c94e91ddc71e31d --- /dev/null +++ b/app/utils/ocr_utils.py @@ -0,0 +1,454 @@ +""" +Utility functions for OCR operations. + +Provides helpers for: +- PDF text detection +- PDF to image conversion +- OCR text cleaning and post-processing +- Text quality assessment + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +import logging +import re +from pathlib import Path +from typing import List, Tuple + +logger = logging.getLogger(__name__) + + +def has_embedded_text(pdf_path: Path, min_text_length: int = 50) -> Tuple[bool, float]: + """ + Check if PDF has embedded text (native PDF). + + Args: + pdf_path: Path to PDF file + min_text_length: Minimum text length to consider as "has text" + + Returns: + Tuple of (has_text, text_coverage_ratio) + - has_text: True if PDF has meaningful embedded text + - text_coverage_ratio: Ratio of pages with text (0.0-1.0) + + Example: + >>> has_text, coverage = has_embedded_text(Path("doc.pdf")) + >>> has_text + True + >>> coverage + 0.95 + """ + try: + from pypdf import PdfReader + + reader = PdfReader(pdf_path) + total_pages = len(reader.pages) + pages_with_text = 0 + total_text_length = 0 + + for page in reader.pages: + text = page.extract_text() + text_length = len(text.strip()) + total_text_length += text_length + + if text_length >= min_text_length: + pages_with_text += 1 + + text_coverage = pages_with_text / total_pages if total_pages > 0 else 0.0 + has_text = total_text_length >= min_text_length + + return has_text, text_coverage + + except Exception as e: + logger.warning(f"Failed to check embedded text: {e}") + return False, 0.0 + + +def has_images(pdf_path: Path) -> bool: + """ + Check if PDF contains images. + + Args: + pdf_path: Path to PDF file + + Returns: + True if PDF contains images + + Example: + >>> has_images(Path("scanned.pdf")) + True + """ + try: + from pypdf import PdfReader + + reader = PdfReader(pdf_path) + + # Check first few pages for images + pages_to_check = min(3, len(reader.pages)) + + for i in range(pages_to_check): + page = reader.pages[i] + if "/XObject" in page.get("/Resources", {}): + x_object = page["/Resources"]["/XObject"].get_object() + for obj in x_object: + if x_object[obj]["/Subtype"] == "/Image": + return True + + return False + + except Exception as e: + logger.warning(f"Failed to check for images: {e}") + return False + + +def convert_pdf_to_images(pdf_path: Path, dpi: int = 300) -> List: + """ + Convert PDF pages to images for OCR processing. + + Args: + pdf_path: Path to PDF file + dpi: Resolution for conversion (default: 300) + + Returns: + List of PIL Image objects + + Raises: + ImportError: If pdf2image not available + Exception: If conversion fails + + Example: + >>> images = convert_pdf_to_images(Path("doc.pdf"), dpi=300) + >>> len(images) + 45 + """ + try: + from pdf2image import convert_from_path + + images = convert_from_path( + str(pdf_path), + dpi=dpi, + fmt="png", + thread_count=2, # Parallel processing + ) + + logger.info(f"Converted {len(images)} pages to images at {dpi} DPI") + return images + + except (ImportError, Exception) as e: + logger.warning(f"pdf2image failed ({e}), trying pypdf image extraction fallback...") + try: + return convert_pdf_to_images_pypdf(pdf_path) + except Exception as fallback_error: + logger.error(f"Fallback image extraction failed: {fallback_error}") + raise e + +def convert_pdf_to_images_pypdf(pdf_path: Path) -> List: + """ + Extract images from PDF using pypdf (Fallback if poppler missing). + Best for scanned PDFs where each page is an image. + """ + from pypdf import PdfReader + from PIL import Image + import io + + reader = PdfReader(pdf_path) + images = [] + + for i, page in enumerate(reader.pages): + count = 0 + for image_file_object in page.images: + try: + # pypdf returns ImageFile objects containing data + image_data = image_file_object.data + image = Image.open(io.BytesIO(image_data)) + images.append(image) + count += 1 + except Exception as img_err: + logger.warning(f"Failed to extract image {count} from page {i}: {img_err}") + + if not images: + raise Exception("No images found in PDF using pypdf extraction") + + logger.info(f"Extracted {len(images)} images from PDF using pypdf") + return images + + +def clean_ocr_text(text: str) -> str: + """ + Clean and post-process OCR text for RAG indexing. + + Removes stamps, signatures, headers, page numbers, watermarks, + and formatting artifacts. + + Args: + text: Raw OCR text + + Returns: + Clean text optimized for RAG indexing + """ + if not text: + return "" + + # 1. Normalize whitespace (preserve newlines for structure) + text = re.sub(r"[ \t]+", " ", text) + + # 2. Fix OCR character replacements + text = _fix_ocr_characters(text) + + # 3. Remove polluting content (stamps, headers, signatures, dates) + text = _remove_administrative_content(text) + + # 4. Remove page markers, watermarks, reference numbers + text = _remove_page_markers_and_watermarks(text) + + # 5. Fix formatting + text = _fix_ocr_formatting(text) + + + return text + + +def _fix_ocr_characters(text: str) -> str: + """Fix common OCR character misreplacements.""" + replacements = { + "\u2019": "'", "\u2018": "'", + "\u00ab": '"', "\u00bb": '"', + "\u2014": "-", "\u2013": "-", "\u2026": "...", + "\ufb01": "fi", "\ufb02": "fl", + } + for old, new in replacements.items(): + text = text.replace(old, new) + return text + + +def _remove_administrative_content(text: str) -> str: + """Remove stamps, headers, signatures, and dates from OCR text.""" + stamp_patterns = [ + r"COPIE\s+CERTIFIEE\s+CONFORME", r"POUR\s+COPIE\s+CONFORME", + r"COPIE\s+CONFORME", r"CERTIFIE\s+CONFORME", + r"AMPLIATIONS?\s*:", r"AMPLIATION", + r"CERTIFIED\s+TRUE\s+COPY", r"CERTIFIED\s+COPY", + r"TRUE\s+COPY", r"FOR\s+CERTIFIED\s+COPY", + ] + header_patterns = [ + r"PRESIDENCE\s+DE\s+LA\s+REPUBLIQUE", r"PRESIDENCY\s+OF\s+THE\s+REPUBLIC", + r"SECRETARIAT\s+GENERAL", + r"SERVICE\s+DU\s+FICHIER\s+LEGISLATIF\s+ET\s+REGLEMENTAIRE", + r"LEGISLATIVE\s+AND\s+STATUTORY\s+AFFAIRS\s+CARD\s+INDEX\s+SERVICE", + r"REPUBLIQUE\s+DU\s+CAMEROUN", r"REPUBLIC\s+OF\s+CAMEROON", + r"PAIX\s*[-\u2013]\s*TRAVAIL\s*[-\u2013]\s*PATRIE", + r"PEACE\s*[-\u2013]\s*WORK\s*[-\u2013]\s*FATHERLAND", + r"JOURNAL\s+OFFICIEL", r"OFFICIAL\s+GAZETTE", + ] + signature_patterns = [ + r"Le\s+Pr[e\u00e9]sident\s+de\s+la\s+R[e\u00e9]publique[^.]*\.?", + r"Le\s+Premier\s+Ministre[^.]*\.?", + r"Le\s+Ministre\s+d['\u2019]?[Ee\u00c9\u00e9]tat[^.]*\.?", + r"Le\s+Ministre[^.]*\.?", + r"Le\s+Secr[e\u00e9]taire\s+G[e\u00e9]n[e\u00e9]ral[^.]*\.?", + r"Le\s+Directeur[^.]*\.?", + r"LE\s+PRESIDENT\s+DE\s+LA\s+REPUBLIQUE[^.]*\.?", + r"(?:Sign[e\u00e9]|Signature)\s*:?\s*[^.\\\n]*", + r"\(se?\)\s+[A-Z][a-z\u00c0-\u00ff]+(?:\s+[A-Z][a-z\u00c0-\u00ff]+)*", + ] + date_patterns = [ + r"Yaound[e\u00e9],?\s+le\s+\d{1,2}\s+\w+\s+\d{4}", + r"Fait\s+[\u00e0a]\s+Yaound[e\u00e9][^.]*\.", + r"Fait\s+le\s+\d{1,2}\s+\w+\s+\d{4}", + r"Done\s+at\s+Yaound[e\u00e9][^.]*\.", + ] + for pattern in stamp_patterns + header_patterns + signature_patterns + date_patterns: + text = re.sub(pattern, "", text, flags=re.IGNORECASE) + return text + + +def _remove_page_markers_and_watermarks(text: str) -> str: + """Remove page numbers, watermarks, and reference numbers.""" + page_patterns = [ + r"^\s*[-\u2013]\s*\d+\s*[-\u2013]\s*$", + r"^\s*Page\s+\d+\s*(?:sur|of|/)?\s*\d*\s*$", + r"^\s*\d+\s*/\s*\d+\s*$", + r"^\s*\d{1,3}\s*$", + ] + for pattern in page_patterns: + text = re.sub(pattern, "", text, flags=re.MULTILINE | re.IGNORECASE) + + watermark_patterns = [ + r"CONFIDENTIEL", r"CONFIDENTIAL", + r"NE\s+PAS\s+DIFFUSER", r"DO\s+NOT\s+DISTRIBUTE", + r"DRAFT", r"BROUILLON", + r"INTERNAL\s+USE\s+ONLY", r"USAGE\s+INTERNE", + ] + ref_patterns = [ + r"R[e\u00e9]f[.:]?\s*N[\u00b0o]?\s*[\d/-]+", + r"N[\u00b0o]?\s*[\d]+/[A-Z]+/[A-Z]+", + r"Dossier\s+N[\u00b0o]?\s*:?\s*[\w/-]+", + ] + for pattern in watermark_patterns + ref_patterns: + text = re.sub(pattern, "", text, flags=re.IGNORECASE) + return text + + +def _fix_ocr_formatting(text: str) -> str: + """Fix spacing, line breaks, and trailing whitespace.""" + text = re.sub(r" +([.,;:!?])", r"\1", text) + text = re.sub(r"([.,;:!?])([A-Za-z\u00c0-\u00ff])", r"\1 \2", text) + text = re.sub(r"\n{3,}", "\n\n", text) + text = re.sub(r"^\s+$", "", text, flags=re.MULTILINE) + text = re.sub(r"\n{3,}", "\n\n", text) + text = re.sub(r" +$", "", text, flags=re.MULTILINE) + return text.strip() + + + + + +def merge_text_blocks(blocks: List[str]) -> str: + """ + Merge multiple text blocks (e.g., from different pages). + + Args: + blocks: List of text blocks + + Returns: + Merged text with proper spacing + + Example: + >>> blocks = ["Article 1. Text...", "Article 2. More text..."] + >>> merge_text_blocks(blocks) + 'Article 1. Text...\\n\\nArticle 2. More text...' + """ + if not blocks: + return "" + + # Clean each block + cleaned_blocks = [clean_ocr_text(block) for block in blocks if block.strip()] + + # Join with double newline + merged = "\n\n".join(cleaned_blocks) + + return merged + + +def assess_text_quality(text: str) -> float: + """ + Assess quality of OCR text. + + Heuristics: + - Ratio of alphabetic characters + - Presence of common words + - Sentence structure + + Args: + text: OCR text to assess + + Returns: + Quality score (0.0-1.0) + + Example: + >>> assess_text_quality("Article 1. La présente loi...") + 0.95 + >>> assess_text_quality("@#$% garbage !!!") + 0.1 + """ + if not text or len(text) < 10: + return 0.0 + + score = 0.0 + + # 1. Alphabetic character ratio (40% weight) + alpha_chars = sum(c.isalpha() for c in text) + total_chars = len(text) + alpha_ratio = alpha_chars / total_chars if total_chars > 0 else 0 + score += alpha_ratio * 0.4 + + # 2. Word ratio (30% weight) + words = text.split() + if words: + # Check for reasonable word lengths + reasonable_words = sum(1 for w in words if 2 <= len(w) <= 20) + word_ratio = reasonable_words / len(words) + score += word_ratio * 0.3 + + # 3. Common French/English words (30% weight) + common_words = { + "le", + "la", + "les", + "de", + "et", + "un", + "une", + "the", + "a", + "an", + "of", + "and", + "article", + "loi", + "law", + } + text_lower = text.lower() + found_common = sum(1 for word in common_words if word in text_lower) + common_ratio = min(found_common / 5, 1.0) # At least 5 common words = 1.0 + score += common_ratio * 0.3 + + return min(score, 1.0) + + +def estimate_ocr_time(page_count: int, dpi: int = 300) -> int: + """ + Estimate OCR processing time. + + Args: + page_count: Number of pages + dpi: DPI setting + + Returns: + Estimated time in seconds + + Example: + >>> estimate_ocr_time(45, dpi=300) + 18 # ~0.4s per page + """ + # Base time per page (seconds) + base_time_per_page = 0.4 + + # DPI multiplier + dpi_multiplier = dpi / 300 + + # Total estimate + estimated_seconds = int(page_count * base_time_per_page * dpi_multiplier) + + return max(estimated_seconds, 1) + + +def split_text_into_pages(text: str, page_markers: List[str] = None) -> List[str]: + """ + Split OCR text back into pages if markers present. + + Args: + text: Full OCR text + page_markers: List of page marker patterns + + Returns: + List of text per page + + Example: + >>> text = "Page 1 text\\n--- PAGE BREAK ---\\nPage 2 text" + >>> split_text_into_pages(text, ["--- PAGE BREAK ---"]) + ['Page 1 text', 'Page 2 text'] + """ + if not page_markers: + # Default markers + page_markers = ["\f", "--- PAGE BREAK ---", "\n\n\n"] + + pages = [text] + + for marker in page_markers: + if marker in text: + pages = text.split(marker) + break + + return [page.strip() for page in pages if page.strip()] diff --git a/app/utils/text_chunker.py b/app/utils/text_chunker.py new file mode 100644 index 0000000000000000000000000000000000000000..ee8df3f35808dc7c21dc7753a6ceb56d62d20fce --- /dev/null +++ b/app/utils/text_chunker.py @@ -0,0 +1,673 @@ +""" +Text chunking utilities for legal document processing. + +Extracts articles from Cameroonian legal documents following +common patterns: Article X, Art. X, Section X, etc. +""" + +import re +from typing import List, Dict, Any, Optional, Tuple +from dataclasses import dataclass +import logging + +logger = logging.getLogger(__name__) + + +class ArticleExtractionError(Exception): + """Raised when article extraction fails.""" + pass + + +@dataclass +class ExtractedArticle: + """Represents a single extracted article.""" + number: str + title: Optional[str] + content: str + position: int + parent_id: Optional[str] + section: Optional[str] + word_count: int + char_count: int + page_number: Optional[int] = None # PDF page number (1-indexed) for navigation + + +# Pattern constants - COMPREHENSIVE support for French and English variants +# French variants: Article 1, Article 1er, Article premier, Article première, Article deuxième, etc. +# English variants: Section 1, Section one, Section first, Article 1, Article one, etc. +ARTICLE_PATTERNS = [ + # === FRENCH PATTERNS === + # Article + number (1, 2, 3...) OR ordinals (1er, 1ère, 2ème, 3ème...) OR words (premier, première, deuxième...) + r'(?:^|\n)\s*Article\s+(?:' + + r'(\d+)(?:er|ère|ème)?(?:\.\d+)*' + # Article 1, Article 1er, Article 2ème, Article 1.1 + r'|' + + r'(premier|première|deuxième|second|seconde|troisième|quatrième|cinquième|sixième|septième|huitième|neuvième|dixième)' + # Article premier, Article deuxième... + r')\s*[.:\-–]?\s*', + + # Art. (abbreviation) + number OR ordinals + r'(?:^|\n)\s*Art\.?\s+(?:' + + r'(\d+)(?:er|ère|ème)?(?:\.\d+)*' + # Art. 1, Art. 1er, Art. 2ème + r'|' + + r'(premier|première|deuxième|second|seconde|troisième|quatrième|cinquième|sixième|septième|huitième|neuvième|dixième)' + # Art. premier + r')\s*[.:\-–]?\s*', + + # === ENGLISH PATTERNS === + # Section + number OR words (one, first, second, third...) + r'(?:^|\n)\s*Section\s+(?:' + + r'(\d+)(?:\.\d+)*' + # Section 1, Section 1.1 + r'|' + + r'(one|first|two|second|three|third|four|fourth|five|fifth|six|sixth|seven|seventh|eight|eighth|nine|ninth|ten|tenth)' + # Section one, Section first + r')\s*[.:\-–]?\s*', + + # Article (English style) + number OR words + r'(?:^|\n)\s*Article\s+(?:' + + r'(\d+)(?:\.\d+)*' + # Article 1 (English) + r'|' + + r'(one|first|two|second|three|third|four|fourth|five|fifth|six|sixth|seven|seventh|eight|eighth|nine|ninth|ten|tenth)' + # Article one (English) + r')\s*[.:\-–]?\s*', + + # Sec. (abbreviation, English) + number + r'(?:^|\n)\s*Sec\.?\s+(\d+)(?:\.\d+)*\s*[.:\-–]?\s*', +] + +# Pattern for préambule/preamble detection - multiple patterns for flexibility +PREAMBLE_PATTERNS = [ + r'(?:^|\n)\s*(PRÉAMBULE|PREAMBULE|PREAMBLE)\s*[.:]?\s*', + r'(?:^|\n)\s*Le\s+peuple\s+camerounais', # Constitution camerounaise + r'(?:^|\n)\s*The\s+people\s+of\s+Cameroon', # English version + r'(?:^|\n)\s*Nous,\s+peuple', # Other constitutions + r'(?:^|\n)\s*We,\s+the\s+people', # English generic +] + +# Patterns indicating end of legal basis / start of substantive content +LEGAL_BASIS_END_PATTERNS = [ + r'(?:^|\n)\s*(PRÉAMBULE|PREAMBULE|PREAMBLE)\s*[.:]?', + r'(?:^|\n)\s*Le\s+peuple\s+camerounais', + r'(?:^|\n)\s*The\s+people\s+of\s+Cameroon', + r'(?:^|\n)\s*(DÉCIDE|DECIDE|DÉCRÈTE|DECRETE|ARRÊTE|ARRETE)\s*[.:]?', + r'(?:^|\n)\s*(TITRE\s+(?:PREMIER|I|1)|TITLE\s+(?:ONE|I|1))\s*[.:\-–]?', +] + +TITLE_PATTERN = r'(?:^|\n)\s*Article\s+\d+\s*[.:]?\s*([^\n]+?)(?:\n|$)' + +# Section patterns - TITRE and CHAPITRE that appear between articles +# These define the section/chapter for subsequent articles +SECTION_PATTERNS = [ + # TITRE PREMIER - DE L'ÉTAT, TITRE I, TITRE 1, etc. + r'(?:^|\n)\s*(TITRE\s+(?:PREMIER|PREMI[ÈE]RE|DEUXI[ÈE]ME|TROISI[ÈE]ME|QUATRI[ÈE]ME|CINQUI[ÈE]ME|SIXI[ÈE]ME|SEPTI[ÈE]ME|HUITI[ÈE]ME|NEUVI[ÈE]ME|DIXI[ÈE]ME|I{1,3}V?|IV|V?I{0,3}|\d+)\s*[.:\-–]?\s*[^\n]*)', + # CHAPITRE PREMIER, CHAPITRE I, CHAPITRE 1, etc. + r'(?:^|\n)\s*(CHAPITRE\s+(?:PREMIER|PREMI[ÈE]RE|DEUXI[ÈE]ME|TROISI[ÈE]ME|QUATRI[ÈE]ME|CINQUI[ÈE]ME|SIXI[ÈE]ME|SEPTI[ÈE]ME|HUITI[ÈE]ME|NEUVI[ÈE]ME|DIXI[ÈE]ME|I{1,3}V?|IV|V?I{0,3}|\d+)\s*[.:\-–]?\s*[^\n]*)', + # PART ONE, PART I, PART 1 (English) + r'(?:^|\n)\s*(PART\s+(?:ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|I{1,3}V?|IV|V?I{0,3}|\d+)\s*[.:\-–]?\s*[^\n]*)', + # CHAPTER ONE, CHAPTER I, CHAPTER 1 (English) + r'(?:^|\n)\s*(CHAPTER\s+(?:ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|I{1,3}V?|IV|V?I{0,3}|\d+)\s*[.:\-–]?\s*[^\n]*)', +] + + +def normalize_article_number(number: str) -> str: + """ + Normalize article/section numbers to standard format. + + Converts ALL variants to numeric format: + - French: 'premier', 'première' -> '1', 'deuxième', 'second' -> '2', etc. + - English: 'one', 'first' -> '1', 'two', 'second' -> '2', etc. + - Ordinals: '1er', '1ère', '2ème' -> '1', '2', etc. + - Special: 'PRÉAMBULE', 'PREAMBULE', 'PREAMBLE' -> 'PREAMBULE' + """ + if not number: + return number + + lower = number.lower().strip() + + # === FRENCH WORD-TO-NUMBER MAPPING === + french_numbers = { + 'premier': '1', 'première': '1', + 'deuxième': '2', 'second': '2', 'seconde': '2', + 'troisième': '3', + 'quatrième': '4', + 'cinquième': '5', + 'sixième': '6', + 'septième': '7', + 'huitième': '8', + 'neuvième': '9', + 'dixième': '10', + } + + # === ENGLISH WORD-TO-NUMBER MAPPING === + english_numbers = { + 'one': '1', 'first': '1', + 'two': '2', 'second': '2', + 'three': '3', 'third': '3', + 'four': '4', 'fourth': '4', + 'five': '5', 'fifth': '5', + 'six': '6', 'sixth': '6', + 'seven': '7', 'seventh': '7', + 'eight': '8', 'eighth': '8', + 'nine': '9', 'ninth': '9', + 'ten': '10', 'tenth': '10', + } + + # Check French word numbers + if lower in french_numbers: + return french_numbers[lower] + + # Check English word numbers + if lower in english_numbers: + return english_numbers[lower] + + # Handle preamble + if lower in ['préambule', 'preambule', 'preamble']: + return 'PREAMBULE' + + # Remove French ordinal suffixes (1er, 1ère, 2ème, 3ème...) + if lower.endswith(('er', 'ère', 'ème')): + # Extract just the number part + number_clean = re.sub(r'(er|ère|ème)$', '', lower) + if number_clean.replace('.', '').isdigit(): + return number_clean + + # Return as-is for numeric values (1, 2, 3, 1.1, 2.3, etc.) + return number + + +def _make_chunk( + number: str, title: Optional[str], content: str, + position: int, section: Optional[str], + page_number: int, parent_id: Optional[str] = None, +) -> Dict[str, Any]: + """Create a standardized chunk dict. Centralizes chunk creation logic.""" + assert content, "Chunk content must not be empty" + assert isinstance(position, int) and position >= 0, "Position must be non-negative int" + return { + 'number': number, + 'title': title, + 'content': content, + 'position': position, + 'parent_id': parent_id, + 'section': section, + 'word_count': len(content.split()), + 'char_count': len(content), + 'page_number': page_number, + } + + +def _update_page(text: str, page_marker_pattern: re.Pattern, current_page: int) -> int: + """Extract page number from text if present, else return current page.""" + page_match = page_marker_pattern.search(text) + if page_match: + return int(page_match.group(1)) + return current_page + + +def _extract_pre_article_chunks( + processed_text: str, pattern: re.Pattern, + page_marker_pattern: re.Pattern, current_page: int, +) -> Tuple[List[Dict[str, Any]], int, int]: + """ + Extract legal basis and preamble chunks from pre-article text. + + Returns: + Tuple of (chunks, next_position, current_page) + """ + assert processed_text, "Processed text must not be empty" + assert pattern is not None, "Article pattern must be provided" + + chunks: List[Dict[str, Any]] = [] + position = 0 + matches = list(pattern.finditer(processed_text)) + + if not matches: + return chunks, position, current_page + + pre_article_text = processed_text[:matches[0].start()].strip() + if not pre_article_text: + return chunks, position, current_page + + # Find earliest preamble marker + preamble_match = _find_earliest_preamble(pre_article_text) + + if preamble_match: + chunks, position, current_page = _split_legal_basis_and_preamble( + pre_article_text, preamble_match, page_marker_pattern, current_page + ) + else: + chunks, position, current_page = _classify_pre_article_text( + pre_article_text, page_marker_pattern, current_page + ) + + return chunks, position, current_page + + +def _find_earliest_preamble(pre_article_text: str) -> Optional[re.Match]: + """Find the earliest preamble pattern match in pre-article text.""" + best_match = None + best_pos = None + for preamble_pattern in PREAMBLE_PATTERNS: + match = re.search(preamble_pattern, pre_article_text, re.IGNORECASE | re.MULTILINE) + if match and (best_pos is None or match.start() < best_pos): + best_pos = match.start() + best_match = match + return best_match + + +def _split_legal_basis_and_preamble( + pre_article_text: str, preamble_match: re.Match, + page_marker_pattern: re.Pattern, current_page: int, +) -> Tuple[List[Dict[str, Any]], int, int]: + """Split pre-article text into legal basis and preamble chunks.""" + chunks: List[Dict[str, Any]] = [] + position = 0 + + legal_basis_text = pre_article_text[:preamble_match.start()].strip() + preamble_text = pre_article_text[preamble_match.start():].strip() + + # Add legal basis chunk (if substantial) + if legal_basis_text and len(legal_basis_text) > 20: + current_page = _update_page(legal_basis_text, page_marker_pattern, current_page) + clean = page_marker_pattern.sub('', legal_basis_text).strip() + chunks.append(_make_chunk('LEGAL_BASIS', 'Base légale', clean, position, None, current_page)) + position += 1 + + # Add preamble chunk + if preamble_text: + current_page = _update_page(preamble_text, page_marker_pattern, current_page) + clean = page_marker_pattern.sub('', preamble_text).strip() + chunks.append(_make_chunk('PREAMBULE', 'Préambule', clean, position, None, current_page)) + position += 1 + + return chunks, position, current_page + + +def _classify_pre_article_text( + pre_article_text: str, page_marker_pattern: re.Pattern, current_page: int, +) -> Tuple[List[Dict[str, Any]], int, int]: + """Classify pre-article text as either preamble or legal basis.""" + is_preamble = any( + re.match(p, pre_article_text.strip(), re.IGNORECASE) + for p in PREAMBLE_PATTERNS[1:] + ) + + current_page = _update_page(pre_article_text, page_marker_pattern, current_page) + clean = page_marker_pattern.sub('', pre_article_text).strip() + + number = 'PREAMBULE' if is_preamble else 'LEGAL_BASIS' + title = 'Préambule' if is_preamble else 'Base légale' + + chunk = _make_chunk(number, title, clean, 0, None, current_page) + return [chunk], 1, current_page + + +def _extract_article_chunks( + processed_text: str, pattern: re.Pattern, + page_marker_pattern: re.Pattern, start_position: int, + current_page: int, min_article_length: int, +) -> Tuple[List[Dict[str, Any]], int, int]: + """ + Extract article chunks from text using detected pattern. + + Returns: + Tuple of (chunks, next_position, current_page) + """ + assert processed_text, "Text must not be empty" + assert min_article_length >= 0, "min_article_length must be non-negative" + + chunks: List[Dict[str, Any]] = [] + position = start_position + raw_articles = _split_by_pattern_with_sections(processed_text, pattern) + + for number, content, section in raw_articles: + current_page = _update_page(content, page_marker_pattern, current_page) + clean_content = page_marker_pattern.sub('', content).strip() + normalized_number = normalize_article_number(number) + parent_id = _get_parent_id(normalized_number) + char_count = len(clean_content) + + if char_count >= min_article_length: + chunks.append(_make_chunk( + normalized_number, None, clean_content, + position, section, current_page, parent_id, + )) + position += 1 + else: + logger.warning( + f"⚠️ Article {normalized_number} trop court " + f"({char_count} chars), ignoré" + ) + + return chunks, position, current_page + + +def _extract_paragraph_chunks( + processed_text: str, page_marker_pattern: re.Pattern, + current_page: int, min_article_length: int, +) -> List[Dict[str, Any]]: + """ + Extract paragraph chunks from documents without articles. + + Returns: + List of chunk dicts + """ + assert processed_text, "Text must not be empty" + assert min_article_length >= 0, "min_article_length must be non-negative" + + chunks: List[Dict[str, Any]] = [] + position = 0 + paragraphs = re.split(r'\n\s*\n', processed_text) + paragraphs = [p.strip() for p in paragraphs if p.strip()] + + if len(paragraphs) > 1: + for i, paragraph in enumerate(paragraphs, start=1): + current_page = _update_page(paragraph, page_marker_pattern, current_page) + clean_para = page_marker_pattern.sub('', paragraph).strip() + if len(clean_para) >= min_article_length: + chunks.append(_make_chunk( + f'PARA_{i}', f'Paragraphe {i}', clean_para, + position, None, current_page, + )) + position += 1 + else: + logger.info("📄 Texte continu - stockage en un seul chunk") + current_page = _update_page(processed_text, page_marker_pattern, current_page) + clean_text = page_marker_pattern.sub('', processed_text).strip() + chunks.append(_make_chunk( + 'FULL_TEXT', 'Document complet', clean_text, + 0, None, current_page, + )) + + return chunks + + +def extract_articles( + text: str, + min_article_length: int = 10, + preserve_formatting: bool = False, + strict: bool = True +) -> List[Dict[str, Any]]: + """ + Extract articles from legal document text with COMPLETE content preservation. + + Delegates to helper functions for each extraction phase: + 1. Pre-article content (legal basis, preamble) + 2. Articles (with section tracking) + 3. Paragraphs (for non-article documents) + + Args: + text: Full legal document text + min_article_length: Minimum characters per chunk + preserve_formatting: Keep original whitespace/formatting + strict: Raise errors vs warnings for validation failures + + Returns: + List of chunk dicts with standard keys (number, title, content, etc.) + + Raises: + ValueError: If text is empty or too large + ArticleExtractionError: If no content could be extracted + """ + # 1. Validate input + assert isinstance(text, str), "text must be a string" + if not text or not text.strip(): + raise ValueError("Le texte ne peut pas être vide") + if len(text) > 5_000_000: + raise ValueError(f"Texte trop volumineux ({len(text)} chars, max 5M)") + + # 2. Preprocess and detect pattern + processed_text = _preprocess_text(text, preserve_formatting) + pattern = _detect_article_pattern(processed_text) + page_marker_pattern = re.compile(r'<>') + + # 3. Extract chunks based on document structure + if pattern: + logger.info("📋 Document avec articles détecté") + pre_chunks, position, page = _extract_pre_article_chunks( + processed_text, pattern, page_marker_pattern, 1 + ) + article_chunks, _, _ = _extract_article_chunks( + processed_text, pattern, page_marker_pattern, + position, page, min_article_length + ) + chunks = pre_chunks + article_chunks + else: + logger.info("📄 Document sans articles - extraction par paragraphes") + chunks = _extract_paragraph_chunks( + processed_text, page_marker_pattern, 1, min_article_length + ) + + # 4. Final validation + if not chunks: + raise ArticleExtractionError("Aucun contenu extrait du document") + + logger.info(f"✅ Extracted {len(chunks)} chunks (complete content preservation)") + return chunks + + +def _validate_input(text: str, strict: bool) -> None: + """Validate input text.""" + if not text or not text.strip(): + raise ValueError("Le texte ne peut pas être vide") + + # Removed minimum length requirement to support smaller documents + # if len(text) < 200: + # raise ValueError(f"Texte trop court ({len(text)} chars, minimum 200)") + + if len(text) > 5_000_000: + raise ValueError( + f"Texte trop volumineux ({len(text)} chars, max 5M)" + ) + + +def _preprocess_text(text: str, preserve_formatting: bool) -> str: + """Preprocess text for article extraction.""" + if preserve_formatting: + return text + + # Normalize line endings + text = text.replace('\r\n', '\n').replace('\r', '\n') + + # Remove excessive blank lines (keep structure) + text = re.sub(r'\n{3,}', '\n\n', text) + + # Normalize spaces (but preserve line breaks) + lines = text.split('\n') + lines = [re.sub(r' {2,}', ' ', line.strip()) for line in lines] + text = '\n'.join(lines) + + return text + + +def _detect_article_pattern(text: str) -> re.Pattern: + """Detect which article pattern is most common.""" + pattern_counts = {} + + for pattern_str in ARTICLE_PATTERNS: + pattern = re.compile(pattern_str, re.IGNORECASE | re.MULTILINE) + matches = pattern.findall(text) + pattern_counts[pattern_str] = len(matches) + + # Use most common pattern + best_pattern_str = max(pattern_counts, key=pattern_counts.get) + + if pattern_counts[best_pattern_str] == 0: + logger.warning("⚠️ Aucun pattern d'article détecté.") + return None + + logger.info( + f"📋 Detected pattern: {best_pattern_str} " + f"({pattern_counts[best_pattern_str]} matches)" + ) + + return re.compile(best_pattern_str, re.IGNORECASE | re.MULTILINE) + + + +def _split_by_pattern(text: str, pattern: re.Pattern) -> List[Tuple[str, str]]: + """Split text by article pattern (legacy, without section tracking).""" + articles = [] + matches = list(pattern.finditer(text)) + + for i, match in enumerate(matches): + number = None + for group_idx in range(1, len(match.groups()) + 1): + group_value = match.group(group_idx) + if group_value is not None: + number = group_value + break + + if number is None: + continue + + start = match.end() + end = matches[i + 1].start() if i + 1 < len(matches) else len(text) + content = text[start:end].strip() + articles.append((number, content)) + + return articles + + +def _split_by_pattern_with_sections(text: str, pattern: re.Pattern) -> List[Tuple[str, str, Optional[str]]]: + """ + Split text by article pattern WITH section tracking. + + Detects TITRE and CHAPITRE headers between articles and associates + each article with its current section. + + Returns: + List of tuples: (article_number, content, section_header) + """ + articles = [] + + # Compile section patterns + section_pattern = re.compile( + '|'.join(f'({p})' for p in SECTION_PATTERNS), + re.IGNORECASE | re.MULTILINE + ) + + # Find all article matches + article_matches = list(pattern.finditer(text)) + + # Track current section + current_section = None + + for i, match in enumerate(article_matches): + # Extract article number + number = None + for group_idx in range(1, len(match.groups()) + 1): + group_value = match.group(group_idx) + if group_value is not None: + number = group_value + break + + if number is None: + logger.warning(f"⚠️ Could not extract article number from match: {match.group(0)}") + continue + + # Check for section headers BEFORE this article + # Look in the text between previous article end and this article start + if i == 0: + # First article - look from start of text + search_start = 0 + else: + # Look from end of previous article + search_start = article_matches[i - 1].end() + + search_end = match.start() + between_text = text[search_start:search_end] + + # Find section headers in between text + section_matches = list(section_pattern.finditer(between_text)) + if section_matches: + # Use the LAST section header found (closest to this article) + last_section_match = section_matches[-1] + # Extract the matched section text (first non-None group) + for group_idx in range(1, len(last_section_match.groups()) + 1): + group_value = last_section_match.group(group_idx) + if group_value: + current_section = group_value.strip() + logger.info(f"📑 Section détectée: {current_section}") + break + + # Article content (from this match to next match or end) + start = match.end() + end = article_matches[i + 1].start() if i + 1 < len(article_matches) else len(text) + content = text[start:end].strip() + + # Strip section headers from content (they belong between articles, not in content) + # Find first section header in content and cut before it + section_in_content = section_pattern.search(content) + if section_in_content: + # Cut content before the section header + content = content[:section_in_content.start()].strip() + + articles.append((number, content, current_section)) + + return articles + + +def _extract_title(content: str) -> Optional[str]: + """ + Extract article title if present. + + Titles are typically on first line after article number. + Example: "Article 1. Dispositions générales\nLa présente loi..." + + Returns None if: + - First line starts with paragraph number like (1), (2) + - First line is too long (>100 chars) or too short (<5 chars) + - First line doesn't start with uppercase + """ + lines = content.split('\n', 1) + if len(lines) == 0: + return None + + first_line = lines[0].strip() + + # Ignore if line starts with paragraph number like (1), (2), 1), 2), etc. + if re.match(r'^\(?\d+\)?[.\s)]', first_line): + return None + + # Ignore if line starts with dash, bullet, or other list markers + if re.match(r'^[-•*–—]\s*', first_line): + return None + + # Ignore if line looks like a sentence continuation (starts with lowercase) + if first_line and first_line[0].islower(): + return None + + # If first line is short (<100 chars) and not empty, likely title + if 5 <= len(first_line) <= 100: + # Check if it looks like a title (short, capitalized) + if first_line[0].isupper(): + return first_line.rstrip('.') + + return None + + +def _clean_article_content(content: str, preserve_formatting: bool, title: Optional[str]) -> str: + """Clean article content.""" + if preserve_formatting: + return content.strip() + + # Remove title if extracted separately + if title: + # Remove first line if it matches the title + lines = content.split('\n') + if len(lines) > 1: + first_line = lines[0].strip().rstrip('.') + if first_line == title: + content = '\n'.join(lines[1:]) + + return content.strip() + + +def _get_parent_id(number: str) -> Optional[str]: + """Determine parent article for hierarchical numbering.""" + # Example: "1.1" → parent="1", "1.2.3" → parent="1.2" + + if '.' not in number: + return None + + parts = number.split('.') + if len(parts) > 1: + return '.'.join(parts[:-1]) + + return None diff --git a/app/utils/validation_utils.py b/app/utils/validation_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e8b36dd341fa0d27e25b2d55aa42ca80375e97a4 --- /dev/null +++ b/app/utils/validation_utils.py @@ -0,0 +1,391 @@ +""" +Utility functions for input validation and sanitization. + +Provides helpers for: +- SQL injection prevention +- XSS prevention +- Input sanitization +- Pattern validation +- Length validation + +Author: JuriX Development Team +Date: 2026-01-11 +""" + +import re +from typing import Optional, Tuple +from urllib.parse import urlparse + + +# SQL injection dangerous keywords +SQL_DANGEROUS_KEYWORDS = [ + "SELECT", "INSERT", "UPDATE", "DELETE", "DROP", "CREATE", "ALTER", + "EXEC", "EXECUTE", "UNION", "DECLARE", "CAST", "CONVERT", + "--", "/*", "*/", "xp_", "sp_", "INFORMATION_SCHEMA" +] + +# XSS dangerous patterns +XSS_PATTERNS = [ + r"]*>.*?", + r"javascript:", + r"on\w+\s*=", # Event handlers like onclick= + r" Tuple[bool, Optional[str]]: + """ + Check if input is safe from SQL injection. + + Args: + input_str: Input string to check + strict: If True, apply strict checking + + Returns: + Tuple of (is_safe, reason) + + Example: + >>> is_safe_sql_input("John Doe") + (True, None) + >>> is_safe_sql_input("'; DROP TABLE users; --") + (False, "Contains SQL keyword: DROP") + """ + if not input_str: + return True, None + + # Check for dangerous SQL keywords + input_upper = input_str.upper() + for keyword in SQL_DANGEROUS_KEYWORDS: + if keyword in input_upper: + return False, f"Contains SQL keyword: {keyword}" + + # Check for SQL comment patterns + if "--" in input_str or "/*" in input_str or "*/" in input_str: + return False, "Contains SQL comment pattern" + + # Check for multiple statements (semicolon) + if strict and ";" in input_str: + return False, "Contains statement separator (;)" + + # Check for quote escaping attempts + if strict and ("\\'" in input_str or '\\"' in input_str): + return False, "Contains quote escaping" + + return True, None + + +def sanitize_sql_input(input_str: str) -> str: + """ + Sanitize input for SQL queries. + + Note: This is a defense-in-depth measure. + Always use parameterized queries as primary defense. + + Args: + input_str: Input to sanitize + + Returns: + Sanitized string + + Example: + >>> sanitize_sql_input("John'; DROP TABLE") + "John DROP TABLE" + """ + if not input_str: + return "" + + # Remove SQL keywords + sanitized = input_str + for keyword in SQL_DANGEROUS_KEYWORDS: + sanitized = re.sub( + rf"\b{keyword}\b", + "", + sanitized, + flags=re.IGNORECASE + ) + + # Remove SQL comment patterns + sanitized = sanitized.replace("--", "") + sanitized = sanitized.replace("/*", "") + sanitized = sanitized.replace("*/", "") + + # Remove semicolons + sanitized = sanitized.replace(";", "") + + # Remove extra whitespace + sanitized = " ".join(sanitized.split()) + + return sanitized.strip() + + +def contains_xss(input_str: str) -> Tuple[bool, Optional[str]]: + """ + Check if input contains XSS patterns. + + Args: + input_str: Input string to check + + Returns: + Tuple of (contains_xss, pattern_found) + + Example: + >>> contains_xss("") + (True, "script tag") + >>> contains_xss("Hello World") + (False, None) + """ + if not input_str: + return False, None + + input_lower = input_str.lower() + + # Check for script tags + if " str: + """ + Sanitize HTML to prevent XSS. + + Args: + html: HTML string to sanitize + allowed_tags: List of allowed HTML tags (default: basic formatting) + + Returns: + Sanitized HTML + + Example: + >>> sanitize_html("

Safe

") + "

Safe

" + """ + if not html: + return "" + + try: + import bleach + + if allowed_tags is None: + allowed_tags = ["p", "br", "strong", "em", "u", "a", "ul", "ol", "li"] + + allowed_attrs = { + "a": ["href", "title"], + } + + cleaned = bleach.clean( + html, + tags=allowed_tags, + attributes=allowed_attrs, + strip=True + ) + + return cleaned + + except ImportError: + # Fallback: strip all HTML tags + return re.sub(r"<[^>]+>", "", html) + + +def validate_email(email: str) -> bool: + """ + Validate email format. + + Args: + email: Email address to validate + + Returns: + True if valid email format + + Example: + >>> validate_email("user@example.com") + True + >>> validate_email("invalid-email") + False + """ + if not email: + return False + + # RFC 5322 simplified pattern + pattern = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$" + return bool(re.match(pattern, email)) + + +def validate_url(url: str, allowed_schemes: Optional[list] = None) -> Tuple[bool, Optional[str]]: + """ + Validate URL format and scheme. + + Args: + url: URL to validate + allowed_schemes: List of allowed schemes (default: http, https) + + Returns: + Tuple of (is_valid, reason) + + Example: + >>> validate_url("https://example.com") + (True, None) + >>> validate_url("javascript:alert('XSS')") + (False, "Invalid scheme: javascript") + """ + if not url: + return False, "Empty URL" + + if allowed_schemes is None: + allowed_schemes = ["http", "https"] + + try: + parsed = urlparse(url) + + # Check scheme + if parsed.scheme not in allowed_schemes: + return False, f"Invalid scheme: {parsed.scheme}" + + # Check netloc (domain) + if not parsed.netloc: + return False, "Missing domain" + + return True, None + + except Exception as e: + return False, f"Invalid URL format: {str(e)}" + + +def check_length_limits( + input_str: str, + min_length: Optional[int] = None, + max_length: Optional[int] = None +) -> Tuple[bool, Optional[str]]: + """ + Check if string length is within limits. + + Args: + input_str: String to check + min_length: Minimum length (optional) + max_length: Maximum length (optional) + + Returns: + Tuple of (is_valid, reason) + + Example: + >>> check_length_limits("Hello", min_length=3, max_length=10) + (True, None) + >>> check_length_limits("Hi", min_length=3) + (False, "Too short: 2 < 3") + """ + length = len(input_str) + + if min_length is not None and length < min_length: + return False, f"Too short: {length} < {min_length}" + + if max_length is not None and length > max_length: + return False, f"Too long: {length} > {max_length}" + + return True, None + + +def validate_pattern(input_str: str, pattern: str, pattern_name: str = "pattern") -> Tuple[bool, Optional[str]]: + """ + Validate string against regex pattern. + + Args: + input_str: String to validate + pattern: Regex pattern + pattern_name: Name of pattern for error message + + Returns: + Tuple of (is_valid, reason) + + Example: + >>> validate_pattern("ABC123", r"^[A-Z]{3}\\d{3}$", "code") + (True, None) + >>> validate_pattern("abc123", r"^[A-Z]{3}\\d{3}$", "code") + (False, "Does not match code pattern") + """ + if not re.match(pattern, input_str): + return False, f"Does not match {pattern_name} pattern" + + return True, None + + +def sanitize_filename(filename: str) -> str: + """ + Sanitize filename to prevent directory traversal. + + Args: + filename: Filename to sanitize + + Returns: + Sanitized filename + + Example: + >>> sanitize_filename("../../etc/passwd") + "etc_passwd" + >>> sanitize_filename("file") + + assert has_xss is True + assert "script" in pattern.lower() + + def test_detect_event_handler(self, validation_service): + """Test détection event handlers.""" + has_xss, pattern = validation_service.check_xss("") + + assert has_xss is True + assert "event" in pattern.lower() + + def test_sanitize_html_removes_script(self, validation_service): + """Test que sanitize_html retire les scripts.""" + dirty = "

Safe content

" + clean = validation_service.sanitize_html(dirty) + + assert "

Safe content

" in clean + assert "", + check_xss=True + ) + + assert is_valid is False + assert len(errors) > 0 + assert any("XSS" in error for error in errors) + + +# ==================== UTILITY TESTS ==================== + + +class TestUtilities: + """Tests des utilitaires.""" + + def test_sanitize_filename(self, validation_service): + """Test sanitization nom de fichier.""" + dangerous = "../../etc/passwd" + safe = validation_service.sanitize_filename(dangerous) + + assert ".." not in safe + assert "/" not in safe + assert "\\" not in safe + + def test_normalize_text(self, validation_service): + """Test normalisation texte.""" + messy = "Hello World\n\n\nTest" + clean = validation_service.normalize_text(messy) + + assert " " not in clean # No multiple spaces + assert "\n\n\n" not in clean # No triple newlines + + +# ==================== HEALTH CHECK TESTS ==================== + + +class TestHealthCheck: + """Tests du health check.""" + + def test_health_check_returns_status(self, validation_service): + """Test que le health check retourne un statut.""" + health = validation_service.health_check() + + assert "service" in health + assert "status" in health + assert "features" in health + + assert health["service"] == "ValidationService" + assert health["status"] == "healthy" + + def test_health_check_features(self, validation_service): + """Test que toutes les features sont listées.""" + health = validation_service.health_check() + + features = health["features"] + assert features["sql_injection_prevention"] is True + assert features["xss_prevention"] is True + assert features["rate_limiting"] is True + assert features["format_validation"] is True diff --git a/tests/test_tasks/__init__.py b/tests/test_tasks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/test_utils/test_text_chunker.py b/tests/test_utils/test_text_chunker.py new file mode 100644 index 0000000000000000000000000000000000000000..31709d1c06b6a99a59f424f2b0c035286430835b --- /dev/null +++ b/tests/test_utils/test_text_chunker.py @@ -0,0 +1,435 @@ +""" +Test suite for text_chunker utility module. + +Tests article extraction from legal documents with various patterns, +edge cases, and validation scenarios. +""" + +import pytest +from app.utils.text_chunker import ( + extract_articles, + ArticleExtractionError, + ExtractedArticle +) + + +class TestBasicExtraction: + """Test basic article extraction.""" + + def test_extract_simple_articles(self): + """Test extraction with 'Article X.' pattern.""" + text = """ + Article 1. Dispositions générales + La présente loi régit les sociétés commerciales. + + Article 2. Champ d'application + Les dispositions s'appliquent à toutes les sociétés. + + Article 3. Définitions + Au sens de la présente loi, on entend par société... + """ + + articles = extract_articles(text) + + assert len(articles) == 3 + assert articles[0]['number'] == '1' + assert articles[0]['title'] == 'Dispositions générales' + assert 'sociétés commerciales' in articles[0]['content'] + assert articles[0]['position'] == 0 + assert articles[0]['parent_id'] is None + assert articles[1]['number'] == '2' + assert articles[2]['number'] == '3' + + def test_extract_art_abbreviation(self): + """Test extraction with 'Art.' abbreviation.""" + text = """ + Art. 1. Premier article + Contenu de l'article premier avec suffisamment de texte. + + Art. 2. Deuxième article + Contenu de l'article deux avec suffisamment de texte. + + Art. 3. Troisième article + Contenu de l'article trois avec suffisamment de texte. + """ + + articles = extract_articles(text) + + assert len(articles) == 3 + assert all(a['number'] in ['1', '2', '3'] for a in articles) + + def test_extract_article_premier(self): + """Test extraction with 'Article premier'.""" + text = """ + Article premier. Objet de la loi + La présente loi a pour objet de régir les activités commerciales au Cameroun + et de protéger les intérêts des consommateurs conformément aux normes en vigueur. + + Article 2. Dispositions générales applicables + Les dispositions suivantes s'appliquent à tous les cas prévus par la présente loi + et concernent l'ensemble des acteurs économiques et commerciaux du territoire. + + Article 3. Finalités et objectifs de la loi + La finalité de cette loi est de protéger les consommateurs et de réguler + les activités commerciales selon les standards internationaux reconnus. + """ + + articles = extract_articles(text) + + assert len(articles) == 3 + assert articles[0]['number'] == 'premier' + assert articles[1]['number'] == '2' + assert articles[2]['number'] == '3' + + +class TestHierarchicalNumbering: + """Test hierarchical article numbering.""" + + def test_extract_nested_articles(self): + """Test extraction with nested numbering (1.1, 1.2).""" + text = """ + Article 1. Titre principal + Contenu de l'article 1 avec suffisamment de texte pour validation. + + Article 1.1. Sous-article 1 + Contenu du sous-article 1.1 avec suffisamment de texte. + + Article 1.2. Sous-article 2 + Contenu du sous-article 1.2 avec suffisamment de texte. + + Article 2. Deuxième article + Contenu de l'article 2 avec suffisamment de texte. + """ + + articles = extract_articles(text) + + assert len(articles) == 4 + assert articles[0]['number'] == '1' + assert articles[0]['parent_id'] is None + assert articles[1]['number'] == '1.1' + assert articles[1]['parent_id'] == '1' + assert articles[2]['number'] == '1.2' + assert articles[2]['parent_id'] == '1' + assert articles[3]['number'] == '2' + assert articles[3]['parent_id'] is None + + def test_extract_deeply_nested_articles(self): + """Test extraction with deeply nested numbering (1.2.3).""" + text = """ + Article 1. Titre niveau 1 + Contenu de l'article 1 avec suffisamment de texte. + + Article 1.1. Titre niveau 2 + Contenu du sous-article 1.1 avec suffisamment de texte. + + Article 1.1.1. Titre niveau 3 + Contenu du sous-article 1.1.1 avec suffisamment de texte. + + Article 2. Autre article + Contenu final avec suffisamment de texte pour validation. + """ + + articles = extract_articles(text) + + assert len(articles) == 4 + assert articles[2]['number'] == '1.1.1' + assert articles[2]['parent_id'] == '1.1' + + +class TestValidation: + """Test input validation.""" + + def test_empty_text_raises_error(self): + """Test that empty text raises ValueError.""" + with pytest.raises(ValueError, match="ne peut pas être vide"): + extract_articles("") + + def test_whitespace_only_text_raises_error(self): + """Test that whitespace-only text raises ValueError.""" + with pytest.raises(ValueError, match="ne peut pas être vide"): + extract_articles(" \n\n \t\t ") + + def test_text_too_short_raises_error(self): + """Test that short text raises ValueError.""" + short_text = "Article 1. Test." + + with pytest.raises(ValueError, match="trop court"): + extract_articles(short_text) + + def test_no_articles_raises_error(self): + """Test that text without articles raises error.""" + text = "This is a long text without any article markers. " * 20 + + with pytest.raises(ArticleExtractionError, match="Aucun pattern"): + extract_articles(text) + + def test_less_than_3_articles_raises_error(self): + """Test that <3 articles raises error in strict mode.""" + text = """ + Article 1. Premier article + Contenu du premier article avec assez de texte pour dépasser 200 caractères minimum requis. + + Article 2. Deuxième article + Contenu du deuxième article avec assez de texte pour validation et dépasser le seuil. + """ + + with pytest.raises(ArticleExtractionError, match="Minimum 3 articles"): + extract_articles(text, strict=True) + + def test_less_than_3_articles_warning_non_strict(self): + """Test that <3 articles returns result in non-strict mode.""" + text = """ + Article 1. Premier article avec suffisamment de contenu + Contenu du premier article avec assez de texte pour validation. + + Article 2. Deuxième article avec contenu + Contenu du deuxième article avec assez de texte. + """ + + # Should not raise in non-strict mode + articles = extract_articles(text, strict=False) + assert len(articles) == 2 + + def test_text_too_large_raises_error(self): + """Test that text >5MB raises ValueError.""" + # Create a very large text (>5MB) + large_text = "Article 1. " + ("x" * 6_000_000) + + with pytest.raises(ValueError, match="trop volumineux"): + extract_articles(large_text) + + +class TestEdgeCases: + """Test edge cases.""" + + def test_article_with_colon_separator(self): + """Test articles with ':' separator.""" + text = """ + Article 1: Dispositions générales + Contenu avec deux-points et suffisamment de texte. + + Article 2: Champ d'application + Autre contenu avec suffisamment de texte pour validation. + + Article 3: Définitions légales + Dernières dispositions avec suffisamment de texte. + """ + + articles = extract_articles(text) + assert len(articles) == 3 + assert articles[0]['title'] == 'Dispositions générales' + + def test_mixed_french_english(self): + """Test mixed French/English article markers.""" + text = """ + Article 1. Dispositions en français + Contenu en français avec suffisamment de texte. + + Article 2. More provisions in English + English content here with enough text for validation. + + Article 3. Dispositions finales + Contenu final avec suffisamment de texte. + """ + + articles = extract_articles(text) + assert len(articles) == 3 + + def test_preserve_formatting(self): + """Test that formatting is preserved when requested.""" + text = """ + Article 1. Avec espaces multiples + Et des espaces dans le contenu. + + Article 2. Normal article + Contenu normal avec suffisamment de texte. + + Article 3. Final article + Contenu final avec suffisamment de texte. + """ + + articles = extract_articles(text, preserve_formatting=True) + + # Formatting should be preserved + assert ' des espaces dans' in articles[0]['content'] + + def test_very_long_article(self): + """Test extraction of very long articles.""" + long_content = "Contenu répété. " * 1000 # ~16KB + text = f""" + Article 1. Long article + {long_content} + + Article 2. Normal article + Contenu normal avec suffisamment de texte. + + Article 3. Final article + Contenu final avec suffisamment de texte. + """ + + articles = extract_articles(text) + + assert len(articles) == 3 + assert articles[0]['char_count'] > 10000 + assert articles[0]['word_count'] > 1500 + + def test_article_without_title(self): + """Test articles without explicit title.""" + text = """ + Article 1. + Contenu direct sans titre. La présente loi régit les sociétés commerciales au Cameroun + et établit les règles applicables. + + Article 2. + Contenu sans titre. Les dispositions s'appliquent à toutes les sociétés + commerciales et industrielles. + + Article 3. + Pas de titre ici. Au sens de la présente loi, on entend par société + toute entité juridique constituée. + """ + + articles = extract_articles(text) + + assert len(articles) == 3 + # These should not be detected as titles (too long) + assert articles[0]['title'] is None or len(articles[0]['title']) < 100 + assert articles[1]['title'] is None or len(articles[1]['title']) < 100 + assert articles[2]['title'] is None or len(articles[2]['title']) < 100 + + +class TestStatistics: + """Test article statistics.""" + + def test_word_count_accurate(self): + """Test that word count is accurate.""" + text = """ + Article 1. Test words + Un deux trois quatre cinq six sept huit neuf dix onze douze treize quatorze quinze. + Suffisamment de contenu pour valider le minimum de caractères requis. + + Article 2. Test content + Contenu court texte avec suffisamment de mots pour validation. + + Article 3. Test final + Autre contenu texte avec suffisamment de mots également. + """ + + articles = extract_articles(text) + + # First article should have 15 words in first sentence + assert articles[0]['word_count'] >= 15 + + def test_char_count_accurate(self): + """Test that character count is accurate.""" + text = """ + Article 1. Test chars + Exactement cinquante caracteres ici pour tester la longueur du contenu. + Ajout de texte supplémentaire pour validation complète. + + Article 2. Test autre + Autre texte avec suffisamment de caractères pour validation. + + Article 3. Test final + Final texte avec suffisamment de caractères également. + """ + + articles = extract_articles(text) + + # First article content should have specific character count + assert articles[0]['char_count'] == len(articles[0]['content']) + assert articles[0]['char_count'] > 0 + + +class TestPositioning: + """Test article positioning.""" + + def test_position_sequential(self): + """Test that positions are sequential.""" + text = """ + Article 1. Premier + Contenu premier avec suffisamment de texte. + + Article 2. Deuxième + Contenu deuxième avec suffisamment de texte. + + Article 3. Troisième + Contenu troisième avec suffisamment de texte. + + Article 4. Quatrième + Contenu quatrième avec suffisamment de texte. + """ + + articles = extract_articles(text) + + positions = [a['position'] for a in articles] + assert positions == [0, 1, 2, 3] + + +class TestMinimumArticleLength: + """Test minimum article length filtering.""" + + def test_short_articles_filtered_out(self): + """Test that articles below min_article_length are filtered.""" + text = """ + Article 1. Short + x + + Article 2. Normal length article + This article has enough content to pass the minimum length requirement. + + Article 3. Another normal + This article also has enough content to pass validation checks. + + Article 4. Very short + y + """ + + # Default min_article_length=10 + articles = extract_articles(text, strict=False) + + # Only articles 2 and 3 should pass (articles 1 and 4 are too short) + assert len(articles) == 2 + assert articles[0]['number'] == '2' + assert articles[1]['number'] == '3' + + def test_custom_min_article_length(self): + """Test custom min_article_length parameter.""" + text = """ + Article 1. Test content + Short text here with some additional words for validation purposes. + + Article 2. Test other + Another short text with additional content for validation. + + Article 3. Test final + Final short text with sufficient content for validation. + """ + + # Set very low minimum + articles = extract_articles(text, min_article_length=5, strict=True) + + assert len(articles) == 3 + + +class TestMultiplePatterns: + """Test detection of different patterns in same document.""" + + def test_mixed_article_and_art_patterns(self): + """Test that most common pattern is selected.""" + # Predominantly "Article X" pattern + text = """ + Article 1. First article with content + Content with sufficient text for validation purposes and regulatory compliance. + + Article 2. Second article with content + More content with sufficient text for validation and additional requirements. + + Article 3. Third article with content + Final content with sufficient text for validation and complete coverage. + """ + + articles = extract_articles(text) + + # Should extract all 3 (pattern detection picks most common) + assert len(articles) == 3