Spaces:
Running
Running
| #!/usr/bin/env python3 | |
| """ | |
| CLI d'ingestion : construit l'index (Parquet) à partir de `knowledge_base/` | |
| et, si `HF_DATASET_REPO` est configuré, le pousse vers le dataset privé | |
| Hugging Face. | |
| Usage : | |
| python scripts/ingest.py | |
| python scripts/ingest.py --kb-dir chemin/vers/mes_documents --no-push | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | |
| from core.ingestion.indexer import ( # noqa: E402 | |
| build_chunk_records, | |
| push_index_to_hub, | |
| save_index_locally, | |
| ) | |
| from config import settings # noqa: E402 | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description="Ingestion PolyglotRAG") | |
| parser.add_argument("--kb-dir", default="knowledge_base", help="Dossier source") | |
| parser.add_argument( | |
| "--out-dir", default=settings.local_index_dir, help="Dossier de sortie local" | |
| ) | |
| parser.add_argument( | |
| "--no-push", action="store_true", help="Ne pas pousser vers le Hub HF" | |
| ) | |
| args = parser.parse_args() | |
| errors = settings.validate() | |
| if errors: | |
| print("⚠️ " + "\n⚠️ ".join(errors)) | |
| kb_dir = Path(args.kb_dir) | |
| if not kb_dir.exists(): | |
| raise SystemExit(f"Dossier introuvable : {kb_dir}") | |
| def progress(file_name: str, lang: str) -> None: | |
| print(f" ✓ {lang}/{file_name}") | |
| print(f"📚 Ingestion de '{kb_dir}'…") | |
| records = build_chunk_records(kb_dir, progress_cb=progress) | |
| print(f"✅ {len(records)} chunks générés.") | |
| out_dir = Path(args.out_dir) | |
| parquet_path = save_index_locally(records, out_dir) | |
| print(f"💾 Index local sauvegardé : {parquet_path}") | |
| if args.no_push: | |
| print("⏭️ Push vers le Hub désactivé (--no-push).") | |
| return | |
| if not settings.hf_dataset_repo: | |
| print( | |
| "ℹ️ HF_DATASET_REPO non défini : l'index reste local uniquement. " | |
| "Définissez HF_DATASET_REPO dans .env pour le publier sur le Hub." | |
| ) | |
| return | |
| print(f"☁️ Publication vers le dataset privé '{settings.hf_dataset_repo}'…") | |
| url = push_index_to_hub(out_dir, settings.hf_dataset_repo, settings.hf_token) | |
| print(f"✅ Dataset publié : {url}") | |
| if __name__ == "__main__": | |
| main() | |