Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| import requests | |
| from bs4 import BeautifulSoup | |
| import json | |
| import re | |
| from urllib.parse import urljoin, urlparse | |
| import pandas as pd | |
| from io import StringIO | |
| class WebScraper: | |
| def __init__(self): | |
| self.session = requests.Session() | |
| self.session.headers.update({ | |
| 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' | |
| }) | |
| def scrape_page(self, url, scrape_type='all'): | |
| try: | |
| response = self.session.get(url, timeout=15) | |
| response.raise_for_status() | |
| soup = BeautifulSoup(response.text, 'html.parser') | |
| results = {'url': url, 'status': 'success', 'data': {}} | |
| if scrape_type in ['all', 'text']: | |
| text_content = soup.get_text(separator=' ', strip=True) | |
| results['data']['text'] = text_content[:10000] | |
| if scrape_type in ['all', 'links']: | |
| links = [] | |
| for link in soup.find_all('a', href=True): | |
| href = link['href'] | |
| full_url = urljoin(url, href) | |
| links.append({'text': link.get_text(strip=True), 'url': full_url}) | |
| results['data']['links'] = links[:50] | |
| if scrape_type in ['all', 'images']: | |
| images = [] | |
| for img in soup.find_all('img'): | |
| src = img.get('src') or img.get('data-src', '') | |
| if src: | |
| images.append({'src': urljoin(url, src), 'alt': img.get('alt', '')}) | |
| results['data']['images'] = images[:30] | |
| if scrape_type in ['all', 'tables']: | |
| tables = [] | |
| for table in soup.find_all('table'): | |
| rows = [] | |
| for tr in table.find_all('tr'): | |
| cells = [td.get_text(strip=True) for td in tr.find_all(['td', 'th'])] | |
| if cells: | |
| rows.append(cells) | |
| if rows: | |
| tables.append(rows) | |
| results['data']['tables'] = tables[:5] | |
| if scrape_type in ['all', 'headings']: | |
| headings = {} | |
| for level in range(1, 7): | |
| tags = soup.find_all(f'h{level}') | |
| headings[f'h{level}'] = [h.get_text(strip=True) for h in tags] | |
| results['data']['headings'] = headings | |
| if scrape_type in ['all', 'meta']: | |
| meta = {} | |
| for tag in soup.find_all('meta'): | |
| name = tag.get('name') or tag.get('property', '') | |
| content = tag.get('content', '') | |
| if name and content: | |
| meta[name] = content | |
| results['data']['meta'] = meta | |
| return json.dumps(results, indent=2, ensure_ascii=False) | |
| except Exception as e: | |
| return json.dumps({'url': url, 'status': 'error', 'error': str(e)}, indent=2) | |
| def scrape_multiple(self, urls_text, scrape_type='all'): | |
| urls = [u.strip() for u in urls_text.split('\n') if u.strip()] | |
| results = [] | |
| for url in urls[:10]: | |
| result = self.scrape_page(url, scrape_type) | |
| results.append(result) | |
| return '\n\n---\n\n'.join(results) | |
| def export_to_csv(self, url, scrape_type='tables'): | |
| try: | |
| response = self.session.get(url, timeout=15) | |
| soup = BeautifulSoup(response.text, 'html.parser') | |
| tables = soup.find_all('table') | |
| if not tables: | |
| return 'No tables found on this page.' | |
| csv_output = [] | |
| for i, table in enumerate(tables[:3]): | |
| csv_output.append(f'### Table {i+1}\n') | |
| df = pd.read_html(StringIO(str(table)))[0] | |
| csv_output.append(df.to_csv(index=False)) | |
| return '\n'.join(csv_output) | |
| except Exception as e: | |
| return f'Error: {str(e)}' | |
| def scrape_website(url, scrape_type, multiple_urls): | |
| scraper = WebScraper() | |
| if multiple_urls: | |
| return scraper.scrape_multiple(url, scrape_type) | |
| return scraper.scrape_page(url, scrape_type) | |
| def export_data(url, scrape_type): | |
| scraper = WebScraper() | |
| return scraper.export_to_csv(url, scrape_type) | |
| with gr.Blocks(theme=gr.themes.Soft(), title='Web Scraper Pro') as demo: | |
| gr.Markdown('# 🕷️ Web Scraper Pro\n\nExtract data from any website with ease. Supports text, links, images, tables, and more.') | |
| with gr.Row(): | |
| with gr.Column(scale=2): | |
| url_input = gr.Textbox( | |
| label='Website URL(s)', | |
| placeholder='https://example.com\n\nFor multiple URLs, enter one per line', | |
| lines=3 | |
| ) | |
| with gr.Row(): | |
| scrape_type = gr.Dropdown( | |
| choices=['all', 'text', 'links', 'images', 'tables', 'headings', 'meta'], | |
| value='all', | |
| label='Scrape Type' | |
| ) | |
| multiple_urls = gr.Checkbox(label='Multiple URLs Mode', value=False) | |
| scrape_btn = gr.Button('🚀 Start Scraping', variant='primary', size='lg') | |
| export_btn = gr.Button('📥 Export Tables to CSV', variant='secondary') | |
| with gr.Column(scale=3): | |
| output = gr.Textbox( | |
| label='Scraped Results', | |
| lines=20, | |
| max_lines=50 | |
| ) | |
| gr.Markdown('### 📊 Quick Stats') | |
| with gr.Row(): | |
| stat_text = gr.Number(label='Characters Extracted', value=0) | |
| stat_links = gr.Number(label='Links Found', value=0) | |
| stat_images = gr.Number(label='Images Found', value=0) | |
| gr.Markdown('### 💡 Tips\n- Use **Multiple URLs Mode** to scrape several pages at once\n- Select specific **Scrape Type** for faster results\n- **Export Tables** downloads structured data as CSV') | |
| scrape_btn.click( | |
| fn=scrape_website, | |
| inputs=[url_input, scrape_type, multiple_urls], | |
| outputs=output | |
| ) | |
| export_btn.click( | |
| fn=export_data, | |
| inputs=[url_input, scrape_type], | |
| outputs=output | |
| ) | |
| if __name__ == '__main__': | |
| demo.launch(server_name='0.0.0.0', server_port=7860, share=True) |