Spaces:
Paused
Paused
| import os | |
| import json | |
| from bs4 import BeautifulSoup | |
| import glob | |
| INPUT_DIR = "pages" | |
| OUTPUT_FILE = "videos_dataset.json" | |
| def extract_data_from_html(file_path): | |
| """Parses an HTML file and extracts video data.""" | |
| extracted_data = [] | |
| with open(file_path, "r", encoding="utf-8") as f: | |
| soup = BeautifulSoup(f, "html.parser") | |
| # Find all video links | |
| video_elements = soup.find_all("a", class_="video") | |
| for video in video_elements: | |
| try: | |
| url = video.get("href", "").strip() | |
| thumbnail = video.get("data-bg", "").strip() | |
| # Title can be in the title attribute or the h2 tag | |
| title = video.get("title", "") | |
| if not title: | |
| title_elem = video.find("h2", class_="vtitle") | |
| if title_elem: | |
| title = title_elem.text.strip() | |
| # Time and duration | |
| duration_elem = video.find("span", class_="time clock") | |
| duration = duration_elem.text.strip() if duration_elem else "" | |
| ago_elem = video.find("span", class_="ago") | |
| uploaded_ago = ago_elem.text.strip() if ago_elem else "" | |
| # Only add if we have at least a URL and title | |
| if url and title: | |
| extracted_data.append({ | |
| "url": url, | |
| "thumbnail": thumbnail, | |
| "title": title, | |
| "duration": duration, | |
| "uploaded_ago": uploaded_ago | |
| }) | |
| except Exception as e: | |
| print(f"Error extracting an item in {file_path}: {e}") | |
| return extracted_data | |
| def main(): | |
| print(f"Starting Phase 2: Data Extraction from '{INPUT_DIR}' directory...") | |
| # Check if input directory exists | |
| if not os.path.exists(INPUT_DIR): | |
| print(f"Error: Directory '{INPUT_DIR}' not found. Please run phase 1 first.") | |
| return | |
| all_videos = [] | |
| # Get all HTML files in the pages directory | |
| html_files = glob.glob(os.path.join(INPUT_DIR, "*.html")) | |
| if not html_files: | |
| print(f"No HTML files found in '{INPUT_DIR}'.") | |
| return | |
| print(f"Found {len(html_files)} HTML files. Processing...") | |
| # Process each file | |
| for file_path in html_files: | |
| videos_in_file = extract_data_from_html(file_path) | |
| all_videos.extend(videos_in_file) | |
| # Optional progress output | |
| # print(f"Extracted {len(videos_in_file)} videos from {os.path.basename(file_path)}") | |
| print(f"Extraction complete! Extracted {len(all_videos)} total videos.") | |
| # Remove duplicates based on URL just in case | |
| unique_videos = {v['url']: v for v in all_videos}.values() | |
| final_dataset = list(unique_videos) | |
| print(f"Dataset contains {len(final_dataset)} unique videos.") | |
| # Save to JSON | |
| with open(OUTPUT_FILE, "w", encoding="utf-8") as f: | |
| json.dump(final_dataset, f, indent=4, ensure_ascii=False) | |
| print(f"Successfully saved dataset to {OUTPUT_FILE}") | |
| if __name__ == "__main__": | |
| main() | |