import argparse import sys import os import json if sys.stdout.encoding != 'utf-8': try: sys.stdout.reconfigure(encoding='utf-8') except Exception: pass from nlp_core.language_detector import LanguageDetector from nlp_core.tokenizer import BilingualTokenizer from models.extractive.textrank import TextRankSummarizer from models.extractive.lsa import LSASummarizer from models.extractive.hybrid_scorer import HybridSummarizer from models.abstractive.seq2seq_model import Seq2SeqSummarizer from models.abstractive.transformer_wrap import TransformerSummarizer from evaluation.metrics_manager import MetricsManager def main(): parser = argparse.ArgumentParser( description="Bilingual Arabic-English Natural Language Processing System for Automatic Text Summarization" ) parser.add_argument("--file", type=str, help="Path to text document file to summarize") parser.add_argument("--text", type=str, help="Direct raw input text string to summarize") parser.add_argument("--mode", type=str, default="extractive", choices=["extractive", "abstractive", "hybrid"], help="Summarization paradigm") parser.add_argument("--method", type=str, default="textrank", choices=["textrank", "lsa", "hybrid", "seq2seq", "transformer"], help="Specific summarization algorithm") parser.add_argument("--lang", type=str, default="auto", choices=["auto", "ar", "en"], help="Target language (auto-detected if 'auto')") parser.add_argument("--sentences", type=int, default=3, help="Number of sentences for extractive summary") parser.add_argument("--ratio", type=float, default=None, help="Compression ratio (e.g. 0.3 for 30%% length)") parser.add_argument("--beam", type=int, default=3, help="Beam size for Seq2Seq abstractive decoding") parser.add_argument("--ref", type=str, default=None, help="Reference/Gold standard summary string for evaluation") parser.add_argument("--ref-file", type=str, default=None, help="Path to reference summary file") parser.add_argument("--json", action="store_true", help="Output results formatted in JSON") args = parser.parse_args() # Read input text input_text = "" if args.file: if not os.path.exists(args.file): print(f"Error: File not found: {args.file}", file=sys.stderr) sys.exit(1) with open(args.file, "r", encoding="utf-8") as f: input_text = f.read() elif args.text: input_text = args.text else: print("Error: Please provide --file or --text ''", file=sys.stderr) parser.print_help() sys.exit(1) # Detect Language lang_detector = LanguageDetector() detected_lang = lang_detector.detect_language(input_text) active_lang = detected_lang if args.lang == "auto" else args.lang lang_confidence = lang_detector.get_language_confidence(input_text) # Read reference text if provided ref_text = args.ref if args.ref_file and os.path.exists(args.ref_file): with open(args.ref_file, "r", encoding="utf-8") as f: ref_text = f.read() summary_text = "" extra_details = {} # Run Summarizer if args.mode == "extractive" or args.method in ["textrank", "lsa", "hybrid"]: if args.method == "lsa": summarizer = LSASummarizer() elif args.method == "hybrid": summarizer = HybridSummarizer() else: summarizer = TextRankSummarizer() result = summarizer.summarize( input_text, num_sentences=args.sentences, ratio=args.ratio, lang=active_lang ) summary_text = result["summary"] extra_details["selected_indices"] = result["selected_indices"] extra_details["sentence_scores"] = result["sentence_scores"] elif args.mode == "abstractive" or args.method in ["seq2seq", "transformer"]: if args.method == "transformer": transformer_wrap = TransformerSummarizer(lang=active_lang) summary = transformer_wrap.summarize(input_text) if summary: summary_text = summary else: # Fallback to extractive if transformer weights not available locally fallback = TextRankSummarizer().summarize(input_text, num_sentences=args.sentences, lang=active_lang) summary_text = fallback["summary"] else: # Custom Seq2Seq from scratch ckpt_path = f"checkpoints/seq2seq_{'arabic' if active_lang == 'ar' else 'english'}.pt" if os.path.exists(ckpt_path): model = Seq2SeqSummarizer.load_checkpoint(ckpt_path) tokenizer = BilingualTokenizer() tokens = tokenizer.tokenize_words(input_text, lang=active_lang) summary_tokens = model.summarize_beam(tokens, beam_width=args.beam, max_len=60) summary_text = ' '.join(summary_tokens) else: # Fallback to TextRank fallback = TextRankSummarizer().summarize(input_text, num_sentences=args.sentences, lang=active_lang) summary_text = fallback["summary"] extra_details["note"] = f"Trained checkpoint not found at {ckpt_path}. Used TextRank fallback." # Compute Metrics metrics_mgr = MetricsManager() eval_results = metrics_mgr.evaluate_summary( original_text=input_text, generated_summary=summary_text, reference_summary=ref_text, lang=active_lang ) if args.json: output_payload = { "language": active_lang, "language_confidence": lang_confidence, "mode": args.mode, "method": args.method, "summary": summary_text, "metrics": eval_results, "extra": extra_details } print(json.dumps(output_payload, ensure_ascii=False, indent=2)) else: print("\n============================================================") print(f"📖 Bilingual Summarization System [{active_lang.upper()}] (Detected: {detected_lang})") print(f"Method: {args.method.upper()} | Mode: {args.mode.upper()}") print("============================================================") print("\n--- GENERATED SUMMARY ---") print(summary_text) print("\n--- SUMMARY METRICS ---") print(f"Original Words: {eval_results['original_stats']['word_count']} | Summary Words: {eval_results['summary_stats']['word_count']}") print(f"Compression Ratio: {eval_results['compression_ratio']} (Reduction: {eval_results['reduction_percentage']})") print(f"Vocabulary Diversity (TTR): {eval_results['summary_stats']['type_token_ratio']}") if "rouge" in eval_results: print("\n--- EVALUATION BENCHMARKS ---") r = eval_results["rouge"] b = eval_results["bleu"] print(f"ROUGE-1: F1 = {r['rouge-1']['f1']} (P: {r['rouge-1']['precision']}, R: {r['rouge-1']['recall']})") print(f"ROUGE-2: F1 = {r['rouge-2']['f1']} (P: {r['rouge-2']['precision']}, R: {r['rouge-2']['recall']})") print(f"ROUGE-L: F1 = {r['rouge-l']['f1']} (P: {r['rouge-l']['precision']}, R: {r['rouge-l']['recall']})") print(f"BLEU Cumulative: {b['bleu_cumulative']} (BLEU-1: {b['bleu-1']}, BLEU-2: {b['bleu-2']})") print("============================================================\n") if __name__ == "__main__": main()