Spaces:
Runtime error
Runtime error
File size: 5,662 Bytes
7e12425 3ccf82a 7e12425 3ccf82a 7e12425 3ccf82a 7e12425 3ccf82a 7e12425 3ccf82a 7e12425 3ccf82a 7e12425 3ccf82a 7e12425 df535ff | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | import numpy as np
import streamlit as st
from transformers import BartTokenizer, BartForConditionalGeneration
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from nltk.tokenize import sent_tokenize
import nltk
# Download required NLTK data
@st.cache_resource
def download_nltk_data():
try:
nltk.data.find('tokenizers/punkt')
except LookupError:
nltk.download('punkt', quiet=True)
nltk.download('punkt_tab', quiet=True)
download_nltk_data()
# Load BART model and tokenizer
@st.cache_resource
def load_model():
tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn')
model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn')
return tokenizer, model
tokenizer, model = load_model()
# Function for single document summarization
def summarize(text, max_length=150, min_length=50):
inputs = tokenizer([text], max_length=1024, truncation=True, return_tensors='pt')
summary_ids = model.generate(
inputs['input_ids'],
num_beams=4,
max_length=max_length,
min_length=min_length,
early_stopping=True
)
return tokenizer.decode(summary_ids[0], skip_special_tokens=True)
# Function for clustering documents
def cluster_documents(documents, n_clusters=3):
vectorizer = TfidfVectorizer(stop_words='english')
X = vectorizer.fit_transform(documents)
kmeans = KMeans(n_clusters=n_clusters, random_state=42).fit(X)
return kmeans.labels_
# Function to split long text into chunks
def chunk_text(text, max_words=1000):
if len(text.split()) <= max_words:
return [text]
sentences = sent_tokenize(text)
current_chunk = []
chunks = []
current_word_count = 0
for sentence in sentences:
sentence_words = len(sentence.split())
if current_word_count + sentence_words <= max_words:
current_chunk.append(sentence)
current_word_count += sentence_words
else:
chunks.append(" ".join(current_chunk))
current_chunk = [sentence]
current_word_count = sentence_words
if current_chunk:
chunks.append(" ".join(current_chunk))
return chunks
# Function for multi-document summarization
def multi_document_summarize(documents):
results = {
'individual_summaries': [],
'cluster_summaries': [],
'final_summary': None
}
# Generate individual document summaries
for doc in documents:
chunks = chunk_text(doc)
doc_summary = " ".join([summarize(chunk) for chunk in chunks])
results['individual_summaries'].append(doc_summary)
# Cluster documents if we have multiple
if len(documents) > 1:
n_clusters = min(3, len(documents))
clusters = cluster_documents(documents, n_clusters=n_clusters)
# Generate cluster summaries
for cluster_id in np.unique(clusters):
cluster_docs = [doc for doc, c in zip(documents, clusters) if c == cluster_id]
combined_text = " ".join(cluster_docs)
chunks = chunk_text(combined_text)
cluster_summary = " ".join([summarize(chunk) for chunk in chunks])
results['cluster_summaries'].append({
'doc_indices': [i for i, c in enumerate(clusters) if c == cluster_id],
'summary': cluster_summary
})
# Generate final comprehensive summary
all_summaries = results['individual_summaries'] + [cs['summary'] for cs in results['cluster_summaries']]
results['final_summary'] = summarize(" ".join(all_summaries), max_length=200, min_length=100)
else:
results['final_summary'] = results['individual_summaries'][0]
return results
# Streamlit UI
st.title("Multi-Document Summarization")
st.write("This system summarizes multiple related documents using clustering and BART.")
uploaded_files = st.file_uploader(
"Upload text files (one document per file)",
type=['txt'],
accept_multiple_files=True
)
if uploaded_files:
documents = []
for file in uploaded_files:
documents.append(file.read().decode("utf-8"))
if st.button("Generate Summary"):
with st.spinner("Processing documents..."):
try:
results = multi_document_summarize(documents)
# Display individual document summaries
st.subheader("Individual Document Summaries")
for i, (doc, summary) in enumerate(zip(documents, results['individual_summaries'])):
with st.expander(f"Document {i+1} Summary (First 200 chars: {doc[:200]}...)"):
st.write(summary)
# Display cluster summaries if we have multiple documents
if len(documents) > 1:
st.subheader("Cluster Summaries")
for i, cluster in enumerate(results['cluster_summaries']):
with st.expander(f"Cluster {i+1} Summary (Documents: {', '.join(str(x+1) for x in cluster['doc_indices'])}))"):
st.write(cluster['summary'])
# Display final summary
st.subheader("Final Comprehensive Summary")
st.write(results['final_summary'])
except Exception as e:
st.error(f"An error occurred: {str(e)}")
st.info("Please try again or upload different documents.")
else:
st.info("Please upload some text files to get started.") |