main_cli.py

import argparse
import sys
import os
import json

if sys.stdout.encoding != 'utf-8':
    try:
        sys.stdout.reconfigure(encoding='utf-8')
    except Exception:
        pass

from nlp_core.language_detector import LanguageDetector
from nlp_core.tokenizer import BilingualTokenizer
from models.extractive.textrank import TextRankSummarizer
from models.extractive.lsa import LSASummarizer
from models.extractive.hybrid_scorer import HybridSummarizer
from models.abstractive.seq2seq_model import Seq2SeqSummarizer
from models.abstractive.transformer_wrap import TransformerSummarizer
from evaluation.metrics_manager import MetricsManager

def main():
    parser = argparse.ArgumentParser(
        description="Bilingual Arabic-English Natural Language Processing System for Automatic Text Summarization"
    )
    parser.add_argument("--file", type=str, help="Path to text document file to summarize")
    parser.add_argument("--text", type=str, help="Direct raw input text string to summarize")
    parser.add_argument("--mode", type=str, default="extractive", choices=["extractive", "abstractive", "hybrid"], help="Summarization paradigm")
    parser.add_argument("--method", type=str, default="textrank", choices=["textrank", "lsa", "hybrid", "seq2seq", "transformer"], help="Specific summarization algorithm")
    parser.add_argument("--lang", type=str, default="auto", choices=["auto", "ar", "en"], help="Target language (auto-detected if 'auto')")
    parser.add_argument("--sentences", type=int, default=3, help="Number of sentences for extractive summary")
    parser.add_argument("--ratio", type=float, default=None, help="Compression ratio (e.g. 0.3 for 30%% length)")
    parser.add_argument("--beam", type=int, default=3, help="Beam size for Seq2Seq abstractive decoding")
    parser.add_argument("--ref", type=str, default=None, help="Reference/Gold standard summary string for evaluation")
    parser.add_argument("--ref-file", type=str, default=None, help="Path to reference summary file")
    parser.add_argument("--json", action="store_true", help="Output results formatted in JSON")

    args = parser.parse_args()

    # Read input text
    input_text = ""
    if args.file:
        if not os.path.exists(args.file):
            print(f"Error: File not found: {args.file}", file=sys.stderr)
            sys.exit(1)
        with open(args.file, "r", encoding="utf-8") as f:
            input_text = f.read()
    elif args.text:
        input_text = args.text
    else:
        print("Error: Please provide --file <filepath> or --text '<raw text>'", file=sys.stderr)
        parser.print_help()
        sys.exit(1)

    # Detect Language
    lang_detector = LanguageDetector()
    detected_lang = lang_detector.detect_language(input_text)
    active_lang = detected_lang if args.lang == "auto" else args.lang
    lang_confidence = lang_detector.get_language_confidence(input_text)

    # Read reference text if provided
    ref_text = args.ref
    if args.ref_file and os.path.exists(args.ref_file):
        with open(args.ref_file, "r", encoding="utf-8") as f:
            ref_text = f.read()

    summary_text = ""
    extra_details = {}

    # Run Summarizer
    if args.mode == "extractive" or args.method in ["textrank", "lsa", "hybrid"]:
        if args.method == "lsa":
            summarizer = LSASummarizer()
        elif args.method == "hybrid":
            summarizer = HybridSummarizer()
        else:
            summarizer = TextRankSummarizer()

        result = summarizer.summarize(
            input_text,
            num_sentences=args.sentences,
            ratio=args.ratio,
            lang=active_lang
        )
        summary_text = result["summary"]
        extra_details["selected_indices"] = result["selected_indices"]
        extra_details["sentence_scores"] = result["sentence_scores"]

    elif args.mode == "abstractive" or args.method in ["seq2seq", "transformer"]:
        if args.method == "transformer":
            transformer_wrap = TransformerSummarizer(lang=active_lang)
            summary = transformer_wrap.summarize(input_text)
            if summary:
                summary_text = summary
            else:
                # Fallback to extractive if transformer weights not available locally
                fallback = TextRankSummarizer().summarize(input_text, num_sentences=args.sentences, lang=active_lang)
                summary_text = fallback["summary"]
        else:
            # Custom Seq2Seq from scratch
            ckpt_path = f"checkpoints/seq2seq_{'arabic' if active_lang == 'ar' else 'english'}.pt"
            if os.path.exists(ckpt_path):
                model = Seq2SeqSummarizer.load_checkpoint(ckpt_path)
                tokenizer = BilingualTokenizer()
                tokens = tokenizer.tokenize_words(input_text, lang=active_lang)
                summary_tokens = model.summarize_beam(tokens, beam_width=args.beam, max_len=60)
                summary_text = ' '.join(summary_tokens)
            else:
                # Fallback to TextRank
                fallback = TextRankSummarizer().summarize(input_text, num_sentences=args.sentences, lang=active_lang)
                summary_text = fallback["summary"]
                extra_details["note"] = f"Trained checkpoint not found at {ckpt_path}. Used TextRank fallback."

    # Compute Metrics
    metrics_mgr = MetricsManager()
    eval_results = metrics_mgr.evaluate_summary(
        original_text=input_text,
        generated_summary=summary_text,
        reference_summary=ref_text,
        lang=active_lang
    )

    if args.json:
        output_payload = {
            "language": active_lang,
            "language_confidence": lang_confidence,
            "mode": args.mode,
            "method": args.method,
            "summary": summary_text,
            "metrics": eval_results,
            "extra": extra_details
        }
        print(json.dumps(output_payload, ensure_ascii=False, indent=2))
    else:
        print("\n============================================================")
        print(f"📖 Bilingual Summarization System [{active_lang.upper()}] (Detected: {detected_lang})")
        print(f"Method: {args.method.upper()} | Mode: {args.mode.upper()}")
        print("============================================================")
        print("\n--- GENERATED SUMMARY ---")
        print(summary_text)
        print("\n--- SUMMARY METRICS ---")
        print(f"Original Words: {eval_results['original_stats']['word_count']} | Summary Words: {eval_results['summary_stats']['word_count']}")
        print(f"Compression Ratio: {eval_results['compression_ratio']} (Reduction: {eval_results['reduction_percentage']})")
        print(f"Vocabulary Diversity (TTR): {eval_results['summary_stats']['type_token_ratio']}")

        if "rouge" in eval_results:
            print("\n--- EVALUATION BENCHMARKS ---")
            r = eval_results["rouge"]
            b = eval_results["bleu"]
            print(f"ROUGE-1: F1 = {r['rouge-1']['f1']} (P: {r['rouge-1']['precision']}, R: {r['rouge-1']['recall']})")
            print(f"ROUGE-2: F1 = {r['rouge-2']['f1']} (P: {r['rouge-2']['precision']}, R: {r['rouge-2']['recall']})")
            print(f"ROUGE-L: F1 = {r['rouge-l']['f1']} (P: {r['rouge-l']['precision']}, R: {r['rouge-l']['recall']})")
            print(f"BLEU Cumulative: {b['bleu_cumulative']} (BLEU-1: {b['bleu-1']}, BLEU-2: {b['bleu-2']})")
        print("============================================================\n")

if __name__ == "__main__":
    main()