Download app.py from Juna190825/seleniumwebscrapping: direct link, hf CLI and curl.
- Browser
- Download file 9.47 kB
-
https://huggingface.co/spaces/Juna190825/seleniumwebscrapping/resolve/main/app.py
- Command line
-
hf download hf://spaces/Juna190825/seleniumwebscrapping/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Juna190825/seleniumwebscrapping/resolve/main/app.py
9.47 kB
| import os | |
| import time | |
| import tempfile | |
| import shutil | |
| import logging | |
| import subprocess | |
| import socket | |
| import random | |
| from selenium import webdriver | |
| from selenium.webdriver.common.by import By | |
| from selenium.webdriver.support.ui import WebDriverWait | |
| from selenium.webdriver.support import expected_conditions as EC | |
| from selenium.webdriver.chrome.options import Options | |
| from selenium.webdriver.chrome.service import Service | |
| import gradio as gr | |
| # Use these specific ports | |
| WEBDRIVER_PORT = random.randint(4444, 4544) | |
| # Configure logging | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format='%(asctime)s - %(levelname)s - %(message)s', | |
| handlers=[ | |
| logging.FileHandler('/tmp/webscraper.log'), | |
| logging.StreamHandler() | |
| ] | |
| ) | |
| logger = logging.getLogger(__name__) | |
| # Configure environment for matplotlib | |
| os.environ['MPLCONFIGDIR'] = tempfile.mkdtemp() | |
| logger.info(f"Matplotlib config directory set to: {os.environ['MPLCONFIGDIR']}") | |
| def verify_chrome_installation(): | |
| try: | |
| result = subprocess.run(['google-chrome-stable', '--version'], | |
| capture_output=True, text=True) | |
| logger.info(f"Chrome version: {result.stdout}") | |
| return True | |
| except: | |
| logger.error("Chrome verification failed") | |
| return False | |
| def verify_chromedriver(): | |
| chromedriver_path = '/usr/local/bin/chromedriver' | |
| try: | |
| result = subprocess.run([chromedriver_path, '--version'], | |
| capture_output=True, text=True) | |
| logger.info(f"ChromeDriver version: {result.stdout}") | |
| st = os.stat(chromedriver_path) | |
| if not st.st_mode & 0o111: # Check executable bit | |
| logger.warning("ChromeDriver not executable, fixing permissions...") | |
| os.chmod(chromedriver_path, 0o755) | |
| logger.info("Permissions updated") | |
| return True | |
| except: | |
| logger.error("ChromeDriver verification failed") | |
| return False | |
| def create_service(): | |
| # 1. Verify chromedriver | |
| chromedriver_path = shutil.which('chromedriver') or '/usr/local/bin/chromedriver' | |
| if not os.path.exists(chromedriver_path): | |
| raise FileNotFoundError(f"ChromeDriver missing at {chromedriver_path}") | |
| # 2. Check port availability | |
| port = WEBDRIVER_PORT | |
| s = socket.socket() | |
| if s.connect_ex(('127.0.0.1', port)) == 0: | |
| port += 1 | |
| s.close() | |
| # 3. Ensure log directory | |
| log_dir = tempfile.gettempdir() | |
| if not os.access(log_dir, os.W_OK): | |
| log_dir = '/tmp' | |
| # 4. Create service | |
| try: | |
| return Service( | |
| executable_path=chromedriver_path, | |
| port=port, | |
| service_args=[ | |
| '--verbose', | |
| '--log-path=/tmp/chromedriver.log' | |
| ] | |
| ) | |
| except Exception as e: | |
| print(f"Service creation failed: {str(e)}") | |
| return Service() # Fallback to simplest config | |
| def verify_critical_path(): | |
| print(f"ChromeDriver exists: {os.path.exists('/usr/local/bin/chromedriver')}") | |
| print(f"Chrome exists: {os.path.exists('/usr/bin/google-chrome-stable')}") | |
| print(f"ChromeDriver executable: {os.access('/usr/local/bin/chromedriver', os.X_OK)}") | |
| print(f"Chrome executable: {os.access('/usr/bin/google-chrome-stable', os.X_OK)}") | |
| try: | |
| subprocess.run(["/usr/local/bin/chromedriver", "--version"], check=True) | |
| subprocess.run(["/usr/bin/google-chrome-stable", "--version"], check=True) | |
| except subprocess.CalledProcessError as e: | |
| print(f"Binary test failed: {e}") | |
| def setup_selenium(): | |
| try: | |
| logger.info("Initializing Selenium Chrome driver...") | |
| options = Options() | |
| options.add_argument("--headless=new") | |
| options.add_argument("--no-sandbox") | |
| options.add_argument("--disable-dev-shm-usage") | |
| options.add_argument("--disable-gpu") | |
| options.add_argument("--window-size=1920,1080") | |
| options.add_argument("--start-maximized") | |
| options.add_argument("--disable-blink-features=AutomationControlled") | |
| options.add_argument("--disable-notifications") | |
| options.add_argument("--remote-debugging-port=9222") | |
| # Disable automation flag | |
| options.add_experimental_option("excludeSwitches", ["enable-automation"]) | |
| options.add_experimental_option("useAutomationExtension", False) | |
| options.add_experimental_option("prefs", { | |
| "profile.default_content_setting_values.notifications": 2 # Disable notifications | |
| }) | |
| # Mimic a real browser's user agent | |
| options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") | |
| logger.info("Verifying ChromeDriver...") | |
| verifying_chromedriver = verify_chromedriver() | |
| logger.info(f"ChromeDriver verification: {verifying_chromedriver}") | |
| chromedriver_path = "/usr/local/bin/chromedriver" | |
| # os.chmod(chromedriver_path, 0o755) # Make sure it's executable | |
| logger.info(f"Setting executable permission for {chromedriver_path}") | |
| service = create_service() | |
| logger.info("Service created") | |
| verify_critical_path() | |
| try: | |
| logger.info("Attempting Chrome initialization...") | |
| driver = webdriver.Chrome( | |
| options=options, | |
| service=service | |
| ) | |
| logger.info("Successfully initialized Chrome driver!") | |
| return driver | |
| except Exception as e: | |
| logger.error(f"Initialization failed completely: {str(e)}") | |
| try: | |
| from subprocess import run | |
| run([chromedriver_path, "--version"], check=True) | |
| run(["/usr/bin/google-chrome-stable", "--version"], check=True) | |
| except Exception as sub_e: | |
| logger.error(f"Subprocess check failed: {str(sub_e)}") | |
| raise | |
| except Exception as e: | |
| logger.error("Possible causes:") | |
| logger.error("- Missing or incompatible ChromeDriver") | |
| logger.error("- Chrome binary not found") | |
| logger.error("- Insufficient permissions") | |
| logger.error("- Missing system dependencies") | |
| raise RuntimeError(f"Failed to initialize WebDriver: {str(e)}") | |
| def check_selenium_environment(): | |
| try: | |
| driver = setup_selenium() | |
| driver.quit() | |
| return True | |
| except: | |
| return False | |
| if not check_selenium_environment(): | |
| logger.critical("Selenium environment check failed!") | |
| exit(1) | |
| def form_input_text(word): | |
| logger.info(f"Generating input text for word: {word}") | |
| return f"use the word '{word}' based on the following sentences types, give output in a table format, no quotes around sentences, no repeating the same sentence: Simple Sentence, Compound Sentence" | |
| def fetch_sentences(input_text): | |
| driver = None | |
| generated_sentences = [] | |
| try: | |
| logger.info("Starting sentence fetching process...") | |
| logger.debug(f"Input text: {input_text[:100]}...") | |
| driver = setup_selenium() | |
| driver.implicitly_wait(5) | |
| logger.info("Navigating to target URL...") | |
| driver.get("https://copilot.microsoft.com/chat") | |
| print("...after: driver.get('https://copilot.microsoft.com/chat')") | |
| logger.info("Waiting for page to load...") | |
| WebDriverWait(driver, 50).until(EC.presence_of_element_located((By.TAG_NAME, "body"))) | |
| time.sleep(10) | |
| textarea = WebDriverWait(driver, 20).until( | |
| EC.presence_of_element_located((By.ID, "userInput")) | |
| ) | |
| textarea.clear() | |
| textarea.send_keys(input_text) | |
| print("...after: sending") | |
| time.sleep(20) | |
| button = driver.find_element(By.XPATH, "//button[@title='Submit message']") | |
| button.click() | |
| print("...after: button.click()") | |
| print("Text inserted and button pressed successfully!") | |
| time.sleep(20) | |
| table = WebDriverWait(driver, 10).until( | |
| EC.presence_of_element_located((By.XPATH, "//table")) | |
| ) | |
| rows = table.find_elements(By.TAG_NAME, "tr") | |
| print(len(rows)) | |
| for row in rows: | |
| row_cells = row.find_elements(By.TAG_NAME, "td") | |
| generated_sentences.append([r.text for r in row_cells]) | |
| time.sleep(10) | |
| finally: | |
| if driver: | |
| driver.quit() | |
| generated_sentences = [item for item in generated_sentences if len(item) > 0] if len(generated_sentences) > 1 else generated_sentences | |
| return generated_sentences | |
| def scrape_website(url): | |
| try: | |
| if not verify_chrome_installation(): | |
| return "Error: Chrome not properly installed" | |
| current_word = 'go' | |
| return fetch_sentences(form_input_text(current_word)) | |
| except RuntimeError as e: | |
| logger.error(f"Runtime error: {str(e)}") | |
| return f"System error: {str(e)}" | |
| except Exception as e: | |
| logger.error(f"Unexpected error: {str(e)}") | |
| return f"Unexpected error occurred: {str(e)}" | |
| iface = gr.Interface( | |
| fn=scrape_website, | |
| inputs=[ | |
| gr.Textbox(label="URL to scrape", placeholder="https://example.com"), | |
| ], | |
| outputs=gr.Textbox(label="Scraped Content"), | |
| title="Web Scraper with Selenium & Chrome", | |
| description="Enter a URL to scrape its content using headless Chrome browser." | |
| ) | |
| iface.queue().launch( | |
| server_name="0.0.0.0", | |
| server_port=7860, | |
| share=False | |
| ) | |