Sumayyea commited on
Commit
3ccf82a
·
verified ·
1 Parent(s): 7e12425

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +74 -40
app.py CHANGED
@@ -45,45 +45,68 @@ def cluster_documents(documents, n_clusters=3):
45
  kmeans = KMeans(n_clusters=n_clusters, random_state=42).fit(X)
46
  return kmeans.labels_
47
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  # Function for multi-document summarization
49
  def multi_document_summarize(documents):
50
- # Cluster documents
51
- if len(documents) > 3:
52
- n_clusters = min(3, len(documents)//2)
53
- clusters = cluster_documents(documents, n_clusters=n_clusters)
54
- else:
55
- clusters = np.zeros(len(documents))
56
 
57
- # Summarize each cluster
58
- summaries = []
59
- for cluster_id in np.unique(clusters):
60
- cluster_docs = [doc for doc, c in zip(documents, clusters) if c == cluster_id]
61
- combined_text = " ".join(cluster_docs)
 
 
 
 
 
62
 
63
- # Split into chunks if too long for BART
64
- max_chunk_length = 1000
65
- if len(combined_text.split()) > max_chunk_length:
66
- sentences = sent_tokenize(combined_text)
67
- current_chunk = []
68
- chunks = []
69
- for sentence in sentences:
70
- if len(" ".join(current_chunk + [sentence]).split()) <= max_chunk_length:
71
- current_chunk.append(sentence)
72
- else:
73
- chunks.append(" ".join(current_chunk))
74
- current_chunk = [sentence]
75
- if current_chunk:
76
- chunks.append(" ".join(current_chunk))
77
- else:
78
- chunks = [combined_text]
79
 
80
- # Summarize each chunk and combine
81
- cluster_summary = " ".join([summarize(chunk) for chunk in chunks])
82
- summaries.append(cluster_summary)
 
 
83
 
84
- # Create final summary
85
- final_summary = summarize(" ".join(summaries))
86
- return final_summary
87
 
88
  # Streamlit UI
89
  st.title("Multi-Document Summarization")
@@ -103,14 +126,25 @@ if uploaded_files:
103
  if st.button("Generate Summary"):
104
  with st.spinner("Processing documents..."):
105
  try:
106
- summary = multi_document_summarize(documents)
107
- st.subheader("Final Summary")
108
- st.write(summary)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
109
 
110
- st.subheader("Input Documents Preview")
111
- for i, doc in enumerate(documents):
112
- st.write(f"### Document {i+1} (First 200 chars)")
113
- st.write(doc[:200] + "...")
114
  except Exception as e:
115
  st.error(f"An error occurred: {str(e)}")
116
  st.info("Please try again or upload different documents.")
 
45
  kmeans = KMeans(n_clusters=n_clusters, random_state=42).fit(X)
46
  return kmeans.labels_
47
 
48
+ # Function to split long text into chunks
49
+ def chunk_text(text, max_words=1000):
50
+ if len(text.split()) <= max_words:
51
+ return [text]
52
+
53
+ sentences = sent_tokenize(text)
54
+ current_chunk = []
55
+ chunks = []
56
+ current_word_count = 0
57
+
58
+ for sentence in sentences:
59
+ sentence_words = len(sentence.split())
60
+ if current_word_count + sentence_words <= max_words:
61
+ current_chunk.append(sentence)
62
+ current_word_count += sentence_words
63
+ else:
64
+ chunks.append(" ".join(current_chunk))
65
+ current_chunk = [sentence]
66
+ current_word_count = sentence_words
67
+
68
+ if current_chunk:
69
+ chunks.append(" ".join(current_chunk))
70
+
71
+ return chunks
72
+
73
  # Function for multi-document summarization
74
  def multi_document_summarize(documents):
75
+ results = {
76
+ 'individual_summaries': [],
77
+ 'cluster_summaries': [],
78
+ 'final_summary': None
79
+ }
 
80
 
81
+ # Generate individual document summaries
82
+ for doc in documents:
83
+ chunks = chunk_text(doc)
84
+ doc_summary = " ".join([summarize(chunk) for chunk in chunks])
85
+ results['individual_summaries'].append(doc_summary)
86
+
87
+ # Cluster documents if we have multiple
88
+ if len(documents) > 1:
89
+ n_clusters = min(3, len(documents))
90
+ clusters = cluster_documents(documents, n_clusters=n_clusters)
91
 
92
+ # Generate cluster summaries
93
+ for cluster_id in np.unique(clusters):
94
+ cluster_docs = [doc for doc, c in zip(documents, clusters) if c == cluster_id]
95
+ combined_text = " ".join(cluster_docs)
96
+ chunks = chunk_text(combined_text)
97
+ cluster_summary = " ".join([summarize(chunk) for chunk in chunks])
98
+ results['cluster_summaries'].append({
99
+ 'doc_indices': [i for i, c in enumerate(clusters) if c == cluster_id],
100
+ 'summary': cluster_summary
101
+ })
 
 
 
 
 
 
102
 
103
+ # Generate final comprehensive summary
104
+ all_summaries = results['individual_summaries'] + [cs['summary'] for cs in results['cluster_summaries']]
105
+ results['final_summary'] = summarize(" ".join(all_summaries), max_length=200, min_length=100)
106
+ else:
107
+ results['final_summary'] = results['individual_summaries'][0]
108
 
109
+ return results
 
 
110
 
111
  # Streamlit UI
112
  st.title("Multi-Document Summarization")
 
126
  if st.button("Generate Summary"):
127
  with st.spinner("Processing documents..."):
128
  try:
129
+ results = multi_document_summarize(documents)
130
+
131
+ # Display individual document summaries
132
+ st.subheader("Individual Document Summaries")
133
+ for i, (doc, summary) in enumerate(zip(documents, results['individual_summaries'])):
134
+ with st.expander(f"Document {i+1} Summary (First 200 chars: {doc[:200]}...)"):
135
+ st.write(summary)
136
+
137
+ # Display cluster summaries if we have multiple documents
138
+ if len(documents) > 1:
139
+ st.subheader("Cluster Summaries")
140
+ for i, cluster in enumerate(results['cluster_summaries']):
141
+ with st.expander(f"Cluster {i+1} Summary (Documents: {', '.join(str(x+1) for x in cluster['doc_indices'])}))"):
142
+ st.write(cluster['summary'])
143
+
144
+ # Display final summary
145
+ st.subheader("Final Comprehensive Summary")
146
+ st.write(results['final_summary'])
147
 
 
 
 
 
148
  except Exception as e:
149
  st.error(f"An error occurred: {str(e)}")
150
  st.info("Please try again or upload different documents.")