Sumayyea commited on
Commit
7e12425
·
verified ·
1 Parent(s): df535ff

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +117 -104
app.py CHANGED
@@ -1,105 +1,118 @@
1
- import numpy as np
2
- import streamlit as st
3
- from transformers import BartTokenizer, BartForConditionalGeneration
4
- from sklearn.feature_extraction.text import TfidfVectorizer
5
- from sklearn.cluster import KMeans
6
- from nltk.tokenize import sent_tokenize
7
- import nltk
8
- nltk.download('punkt')
9
-
10
- # Load BART model and tokenizer
11
- @st.cache_resource
12
- def load_model():
13
- tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn')
14
- model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn')
15
- return tokenizer, model
16
-
17
- tokenizer, model = load_model()
18
-
19
- # Function for single document summarization
20
- def summarize(text, max_length=150, min_length=50):
21
- inputs = tokenizer([text], max_length=1024, truncation=True, return_tensors='pt')
22
- summary_ids = model.generate(
23
- inputs['input_ids'],
24
- num_beams=4,
25
- max_length=max_length,
26
- min_length=min_length,
27
- early_stopping=True
28
- )
29
- return tokenizer.decode(summary_ids[0], skip_special_tokens=True)
30
-
31
- # Function for clustering documents
32
- def cluster_documents(documents, n_clusters=3):
33
- vectorizer = TfidfVectorizer(stop_words='english')
34
- X = vectorizer.fit_transform(documents)
35
- kmeans = KMeans(n_clusters=n_clusters, random_state=42).fit(X)
36
- return kmeans.labels_
37
-
38
- # Function for multi-document summarization
39
- def multi_document_summarize(documents):
40
- # Cluster documents
41
- if len(documents) > 3:
42
- n_clusters = min(3, len(documents)//2)
43
- clusters = cluster_documents(documents, n_clusters=n_clusters)
44
- else:
45
- clusters = np.zeros(len(documents))
46
-
47
- # Summarize each cluster
48
- summaries = []
49
- for cluster_id in np.unique(clusters):
50
- cluster_docs = [doc for doc, c in zip(documents, clusters) if c == cluster_id]
51
- combined_text = " ".join(cluster_docs)
52
-
53
- # Split into chunks if too long for BART
54
- max_chunk_length = 1000
55
- if len(combined_text.split()) > max_chunk_length:
56
- sentences = sent_tokenize(combined_text)
57
- current_chunk = []
58
- chunks = []
59
- for sentence in sentences:
60
- if len(" ".join(current_chunk + [sentence]).split()) <= max_chunk_length:
61
- current_chunk.append(sentence)
62
- else:
63
- chunks.append(" ".join(current_chunk))
64
- current_chunk = [sentence]
65
- if current_chunk:
66
- chunks.append(" ".join(current_chunk))
67
- else:
68
- chunks = [combined_text]
69
-
70
- # Summarize each chunk and combine
71
- cluster_summary = " ".join([summarize(chunk) for chunk in chunks])
72
- summaries.append(cluster_summary)
73
-
74
- # Create final summary
75
- final_summary = summarize(" ".join(summaries))
76
- return final_summary
77
-
78
- # Streamlit UI
79
- st.title("Multi-Document Summarization")
80
- st.write("This system summarizes multiple related documents using clustering and BART.")
81
-
82
- uploaded_files = st.file_uploader(
83
- "Upload text files (one document per file)",
84
- type=['txt'],
85
- accept_multiple_files=True
86
- )
87
-
88
- if uploaded_files:
89
- documents = []
90
- for file in uploaded_files:
91
- documents.append(file.read().decode("utf-8"))
92
-
93
- if st.button("Generate Summary"):
94
- with st.spinner("Processing documents..."):
95
- summary = multi_document_summarize(documents)
96
-
97
- st.subheader("Final Summary")
98
- st.write(summary)
99
-
100
- st.subheader("Input Documents Preview")
101
- for i, doc in enumerate(documents):
102
- st.write(f"### Document {i+1} (First 200 chars)")
103
- st.write(doc[:200] + "...")
104
- else:
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  st.info("Please upload some text files to get started.")
 
1
+ import numpy as np
2
+ import streamlit as st
3
+ from transformers import BartTokenizer, BartForConditionalGeneration
4
+ from sklearn.feature_extraction.text import TfidfVectorizer
5
+ from sklearn.cluster import KMeans
6
+ from nltk.tokenize import sent_tokenize
7
+ import nltk
8
+
9
+ # Download required NLTK data
10
+ @st.cache_resource
11
+ def download_nltk_data():
12
+ try:
13
+ nltk.data.find('tokenizers/punkt')
14
+ except LookupError:
15
+ nltk.download('punkt', quiet=True)
16
+ nltk.download('punkt_tab', quiet=True)
17
+
18
+ download_nltk_data()
19
+
20
+ # Load BART model and tokenizer
21
+ @st.cache_resource
22
+ def load_model():
23
+ tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn')
24
+ model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn')
25
+ return tokenizer, model
26
+
27
+ tokenizer, model = load_model()
28
+
29
+ # Function for single document summarization
30
+ def summarize(text, max_length=150, min_length=50):
31
+ inputs = tokenizer([text], max_length=1024, truncation=True, return_tensors='pt')
32
+ summary_ids = model.generate(
33
+ inputs['input_ids'],
34
+ num_beams=4,
35
+ max_length=max_length,
36
+ min_length=min_length,
37
+ early_stopping=True
38
+ )
39
+ return tokenizer.decode(summary_ids[0], skip_special_tokens=True)
40
+
41
+ # Function for clustering documents
42
+ def cluster_documents(documents, n_clusters=3):
43
+ vectorizer = TfidfVectorizer(stop_words='english')
44
+ X = vectorizer.fit_transform(documents)
45
+ kmeans = KMeans(n_clusters=n_clusters, random_state=42).fit(X)
46
+ return kmeans.labels_
47
+
48
+ # Function for multi-document summarization
49
+ def multi_document_summarize(documents):
50
+ # Cluster documents
51
+ if len(documents) > 3:
52
+ n_clusters = min(3, len(documents)//2)
53
+ clusters = cluster_documents(documents, n_clusters=n_clusters)
54
+ else:
55
+ clusters = np.zeros(len(documents))
56
+
57
+ # Summarize each cluster
58
+ summaries = []
59
+ for cluster_id in np.unique(clusters):
60
+ cluster_docs = [doc for doc, c in zip(documents, clusters) if c == cluster_id]
61
+ combined_text = " ".join(cluster_docs)
62
+
63
+ # Split into chunks if too long for BART
64
+ max_chunk_length = 1000
65
+ if len(combined_text.split()) > max_chunk_length:
66
+ sentences = sent_tokenize(combined_text)
67
+ current_chunk = []
68
+ chunks = []
69
+ for sentence in sentences:
70
+ if len(" ".join(current_chunk + [sentence]).split()) <= max_chunk_length:
71
+ current_chunk.append(sentence)
72
+ else:
73
+ chunks.append(" ".join(current_chunk))
74
+ current_chunk = [sentence]
75
+ if current_chunk:
76
+ chunks.append(" ".join(current_chunk))
77
+ else:
78
+ chunks = [combined_text]
79
+
80
+ # Summarize each chunk and combine
81
+ cluster_summary = " ".join([summarize(chunk) for chunk in chunks])
82
+ summaries.append(cluster_summary)
83
+
84
+ # Create final summary
85
+ final_summary = summarize(" ".join(summaries))
86
+ return final_summary
87
+
88
+ # Streamlit UI
89
+ st.title("Multi-Document Summarization")
90
+ st.write("This system summarizes multiple related documents using clustering and BART.")
91
+
92
+ uploaded_files = st.file_uploader(
93
+ "Upload text files (one document per file)",
94
+ type=['txt'],
95
+ accept_multiple_files=True
96
+ )
97
+
98
+ if uploaded_files:
99
+ documents = []
100
+ for file in uploaded_files:
101
+ documents.append(file.read().decode("utf-8"))
102
+
103
+ if st.button("Generate Summary"):
104
+ with st.spinner("Processing documents..."):
105
+ try:
106
+ summary = multi_document_summarize(documents)
107
+ st.subheader("Final Summary")
108
+ st.write(summary)
109
+
110
+ st.subheader("Input Documents Preview")
111
+ for i, doc in enumerate(documents):
112
+ st.write(f"### Document {i+1} (First 200 chars)")
113
+ st.write(doc[:200] + "...")
114
+ except Exception as e:
115
+ st.error(f"An error occurred: {str(e)}")
116
+ st.info("Please try again or upload different documents.")
117
+ else:
118
  st.info("Please upload some text files to get started.")