Upload 2 files

app.py

@@ -0,0 +1,105 @@
+import numpy as np
+import streamlit as st
+from transformers import BartTokenizer, BartForConditionalGeneration
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.cluster import KMeans
+from nltk.tokenize import sent_tokenize
+import nltk
+nltk.download('punkt')
+
+# Load BART model and tokenizer
+@st.cache_resource
+def load_model():
+    tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn')
+    model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn')
+    return tokenizer, model
+
+tokenizer, model = load_model()
+
+# Function for single document summarization
+def summarize(text, max_length=150, min_length=50):
+    inputs = tokenizer([text], max_length=1024, truncation=True, return_tensors='pt')
+    summary_ids = model.generate(
+        inputs['input_ids'], 
+        num_beams=4, 
+        max_length=max_length, 
+        min_length=min_length, 
+        early_stopping=True
+    )
+    return tokenizer.decode(summary_ids[0], skip_special_tokens=True)
+
+# Function for clustering documents
+def cluster_documents(documents, n_clusters=3):
+    vectorizer = TfidfVectorizer(stop_words='english')
+    X = vectorizer.fit_transform(documents)
+    kmeans = KMeans(n_clusters=n_clusters, random_state=42).fit(X)
+    return kmeans.labels_
+
+# Function for multi-document summarization
+def multi_document_summarize(documents):
+    # Cluster documents
+    if len(documents) > 3:
+        n_clusters = min(3, len(documents)//2)
+        clusters = cluster_documents(documents, n_clusters=n_clusters)
+    else:
+        clusters = np.zeros(len(documents))
+    
+    # Summarize each cluster
+    summaries = []
+    for cluster_id in np.unique(clusters):
+        cluster_docs = [doc for doc, c in zip(documents, clusters) if c == cluster_id]
+        combined_text = " ".join(cluster_docs)
+        
+        # Split into chunks if too long for BART
+        max_chunk_length = 1000
+        if len(combined_text.split()) > max_chunk_length:
+            sentences = sent_tokenize(combined_text)
+            current_chunk = []
+            chunks = []
+            for sentence in sentences:
+                if len(" ".join(current_chunk + [sentence]).split()) <= max_chunk_length:
+                    current_chunk.append(sentence)
+                else:
+                    chunks.append(" ".join(current_chunk))
+                    current_chunk = [sentence]
+            if current_chunk:
+                chunks.append(" ".join(current_chunk))
+        else:
+            chunks = [combined_text]
+        
+        # Summarize each chunk and combine
+        cluster_summary = " ".join([summarize(chunk) for chunk in chunks])
+        summaries.append(cluster_summary)
+    
+    # Create final summary
+    final_summary = summarize(" ".join(summaries))
+    return final_summary
+
+# Streamlit UI
+st.title("Multi-Document Summarization")
+st.write("This system summarizes multiple related documents using clustering and BART.")
+
+uploaded_files = st.file_uploader(
+    "Upload text files (one document per file)", 
+    type=['txt'], 
+    accept_multiple_files=True
+)
+
+if uploaded_files:
+    documents = []
+    for file in uploaded_files:
+        documents.append(file.read().decode("utf-8"))
+    
+    if st.button("Generate Summary"):
+        with st.spinner("Processing documents..."):
+            summary = multi_document_summarize(documents)
+        
+        st.subheader("Final Summary")
+        st.write(summary)
+        
+        st.subheader("Input Documents Preview")
+        for i, doc in enumerate(documents):
+            st.write(f"### Document {i+1} (First 200 chars)")
+            st.write(doc[:200] + "...")
+else:
+    st.info("Please upload some text files to get started.")

requirements.txt

@@ -0,0 +1,6 @@
+streamlit
+transformers
+scikit-learn
+nltk
+torch
+numpy