File size: 9,748 Bytes
86257ee
41d2040
86257ee
 
 
41d2040
86257ee
41d2040
 
 
932a73b
41d2040
86257ee
 
 
 
 
 
41d2040
86257ee
 
 
 
 
 
41d2040
86257ee
 
 
 
 
 
 
 
 
41d2040
4fc36e2
c538137
86257ee
4fc36e2
c538137
 
86257ee
 
41d2040
 
86257ee
41d2040
 
86257ee
41d2040
 
 
f1bfcb1
41d2040
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86257ee
41d2040
 
86257ee
41d2040
 
86257ee
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41d2040
 
 
86257ee
 
41d2040
 
86257ee
 
41d2040
86257ee
41d2040
 
86257ee
 
 
 
41d2040
 
 
86257ee
41d2040
86257ee
41d2040
86257ee
 
 
41d2040
 
 
 
86257ee
 
41d2040
86257ee
 
 
 
 
 
 
41d2040
86257ee
 
 
 
41d2040
86257ee
 
 
41d2040
86257ee
 
 
41d2040
86257ee
41d2040
86257ee
 
 
41d2040
86257ee
 
41d2040
86257ee
 
 
 
 
41d2040
86257ee
 
41d2040
 
86257ee
 
 
 
 
41d2040
 
86257ee
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41d2040
86257ee
 
c538137
86257ee
 
 
 
41d2040
 
86257ee
 
 
 
 
 
 
 
 
 
 
 
41d2040
86257ee
 
 
 
 
 
 
41d2040
86257ee
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
import gradio as gr
import logging
from openai import OpenAI
import os
from langgraph.graph import StateGraph, END
from typing import Dict, List, TypedDict
from langgraph.checkpoint.memory import MemorySaver
import requests
from bs4 import BeautifulSoup
import re

# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

# Initialize OpenAI client
api_key = os.getenv("OPENAI_API_KEY")
if not api_key:
    raise ValueError("OPENAI_API_KEY environment variable is not set.")
client = OpenAI(api_key=api_key)

# Define the state for LangGraph
class SummarizerState(TypedDict):
    urls: List[str]
    titles: List[str]
    contents: List[str]
    summaries: List[str]
    master_summary: str
    errors: List[str]
    master_output: str
    individual_output: str
    error_output: str

# Instructions for the front-end
INSTRUCTIONS = """
# Web Article Summarizer

Get key insights from up to 3 web articles in concise 2-4 bullet point summaries, so you don’t have to read multiple documents to learn! Powered by an agentic LangGraph framework, it delivers individual and master summaries effortlessly.

## Instructions
1. Enter a web article URL and click **"Add URL"** to include it (up to 3 articles). View added URLs and their titles below. Click **"Clear"** to start over.
2. Click **"Summarize"** to generate a master summary (combining insights from all articles) and individual summaries for each article.
"""

# Function to fetch article title and content
def fetch_article(url: str) -> tuple[str, str, str]:
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        }
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, 'html.parser')
        
        # Extract title
        title = soup.title.string if soup.title else "Untitled Article"
        title = title.strip()
        
        # Extract main content (heuristic-based)
        content = ""
        for tag in soup.find_all(['p', 'article', 'div']):
            if tag.name == 'p' or ('article' in tag.get('class', []) or 'content' in tag.get('class', [])):
                text = tag.get_text(strip=True)
                if text and len(text) > 50:  # Ignore short snippets
                    content += text + " "
        
        # Clean content
        content = re.sub(r'\s+', ' ', content).strip()
        if not content:
            return title, "", f"Error: No meaningful content found at {url}."
        
        logger.info(f"Fetched article: {title} from {url}")
        return title, content[:10000], None  # Limit content length
    except Exception as e:
        logger.error(f"Error fetching article {url}: {str(e)}")
        return "Unknown Title", "", f"Error fetching article {url}: {str(e)}"

# Function to summarize text using ChatGPT 4o-mini
def summarize_text(text: str, prompt_prefix: str = "Summarize the following article in 2-4 bullet points:") -> str:
    try:
        response = client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[
                {"role": "system", "content": "You are a helpful assistant that summarizes text concisely in bullet point format."},
                {"role": "user", "content": f"{prompt_prefix}\n\n{text}"}
            ],
            max_tokens=300,
            temperature=0.7
        )
        summary = response.choices[0].message.content.strip()
        logger.info(f"Generated summary: {summary}")
        return summary
    except Exception as e:
        logger.error(f"Error summarizing text: {str(e)}")
        return f"Error summarizing text: {str(e)}"

# LangGraph Nodes
def fetch_contents(state: SummarizerState) -> SummarizerState:
    state["contents"] = []
    state["titles"] = []
    state["errors"] = []
    for url in state["urls"]:
        title, content, error = fetch_article(url)
        state["titles"].append(title)
        if error:
            state["errors"].append(error)
            state["contents"].append("")
        else:
            state["contents"].append(content)
    logger.info(f"Fetched contents for {len(state['contents'])} articles")
    return state

def generate_individual_summaries(state: SummarizerState) -> SummarizerState:
    state["summaries"] = []
    for url, title, content in zip(state["urls"], state["titles"], state["contents"]):
        if not content:
            state["summaries"].append(f"**Summary for '{title}' ({url})**:\nNo content available.")
        else:
            summary = summarize_text(content, "Summarize the following article in 2-4 bullet points:")
            state["summaries"].append(f"**Summary for '{title}' ({url})**:\n{summary}")
    logger.info(f"Generated {len(state['summaries'])} individual summaries")
    return state

def generate_master_summary(state: SummarizerState) -> SummarizerState:
    valid_contents = [c for c in state["contents"] if c]
    if valid_contents:
        combined_text = " ".join(valid_contents)
        master_prompt = "Provide a concise summary in 2-4 bullet points that combines key insights from the following articles, highlighting common themes or differences:"
        state["master_summary"] = summarize_text(combined_text, master_prompt)
    else:
        state["master_summary"] = "No valid content available for master summary."
    logger.info(f"Master summary: {state['master_summary']}")
    return state

def format_output(state: SummarizerState) -> SummarizerState:
    state["master_output"] = f"**Master Summary**:\n{state['master_summary']}"
    state["individual_output"] = "\n\n".join(state["summaries"])
    state["error_output"] = "\n\n".join(state["errors"]) if state["errors"] else "All summaries generated successfully."
    logger.info(f"Formatted output prepared")
    return state

# Build the LangGraph workflow
workflow = StateGraph(SummarizerState)
workflow.add_node("fetch_contents", fetch_contents)
workflow.add_node("generate_individual_summaries", generate_individual_summaries)
workflow.add_node("generate_master_summary", generate_master_summary)
workflow.add_node("format_output", format_output)
workflow.add_edge("fetch_contents", "generate_individual_summaries")
workflow.add_edge("generate_individual_summaries", "generate_master_summary")
workflow.add_edge("generate_master_summary", "format_output")
workflow.add_edge("format_output", END)
workflow.set_entry_point("fetch_contents")

# Compile the graph
memory = MemorySaver()
app = workflow.compile(checkpointer=memory)

# Function to add a URL
def add_url(url: str, state: Dict) -> tuple[Dict, str, str, str]:
    if not url:
        return state, "Please enter a valid URL.", "", ""
    if len(state["urls"]) >= 3:
        return state, "Maximum limit of 3 URLs reached.", "", ""
    if url in state["urls"]:
        return state, "This URL has already been added.", "", ""
    state["urls"].append(url)
    url_list = "\n".join([f"{url}" for url in state["urls"]])
    return state, f"Added URL: {url}", url_list, ""

# Function to summarize articles
def summarize_articles(state: Dict) -> tuple[Dict, str, str, str]:
    if not state["urls"]:
        return state, "No URLs added yet.", "", ""

    initial_state = SummarizerState(
        urls=state["urls"],
        titles=[],
        contents=[],
        summaries=[],
        master_summary="",
        errors=[],
        master_output="",
        individual_output="",
        error_output=""
    )

    thread_id = "summarizer_thread"
    final_state = app.invoke(initial_state, config={"configurable": {"thread_id": thread_id}})

    state["summaries"] = final_state["summaries"]
    state["master_summary"] = final_state["master_summary"]

    return (
        state,
        final_state["error_output"],
        final_state["master_output"],
        final_state["individual_output"]
    )

# Function to clear the state
def clear_state() -> tuple[Dict, str, str, str]:
    return {"urls": [], "summaries": [], "master_summary": ""}, "Cleared all URLs and summaries.", "", ""

# Gradio interface
with gr.Blocks(theme=gr.themes.Soft(), css="body, .gradio-container, .block { font-size: 14px !important; }") as demo:
    gr.Markdown(INSTRUCTIONS)

    with gr.Row():
        with gr.Column(scale=1):
            gr.Markdown("### Add Article URLs")
            url_input = gr.Textbox(label="Enter Article URL", placeholder="https://www.bbc.com/news/...")
            add_button = gr.Button("Add URL")
            url_list = gr.Textbox(label="Added URLs", interactive=False, lines=5)
            with gr.Row():
                summarize_button = gr.Button("Summarize")
                clear_button = gr.Button("Clear")
            status_output = gr.Textbox(label="Status")

        with gr.Column(scale=2):
            gr.Markdown("### Summaries")
            master_summary = gr.Markdown(label="Master Summary", min_height=200)
            individual_summaries = gr.Markdown(label="Individual Summaries", min_height=300)

    state = gr.State(value={"urls": [], "summaries": [], "master_summary": ""})

    add_button.click(
        fn=add_url,
        inputs=[url_input, state],
        outputs=[state, status_output, url_list, url_input]
    )
    summarize_button.click(
        fn=summarize_articles,
        inputs=[state],
        outputs=[state, status_output, master_summary, individual_summaries]
    )
    clear_button.click(
        fn=clear_state,
        inputs=[],
        outputs=[state, status_output, master_summary, individual_summaries]
    )

# Launch the app
demo.launch(share=False)