from bs4 import BeautifulSoup as bs, NavigableString import requests from urllib.parse import urljoin import json from datetime import datetime import os import argparse # For command-line arguments # --- Configuration --- # If tesseract is not in your PATH, you might need to specify its location # For example, on Windows: # pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe' CIRCULAR_DATA_FILE = "circular-data.json" HEADERS = { 'Host': 'www.epfindia.gov.in', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:55.0) Gecko/20100101 Firefox/55.0', 'Accept': 'text/html, */*; q=0.01', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate, br', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'X-Requested-With': 'XMLHttpRequest', 'Referer': 'https://www.epfindia.gov.in/site_en/Contact_office_wise.php?id=MHEM', 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1' } def generate_year_params(): """Dynamically generates financial year parameters up to the current date.""" current_date = datetime.now() current_year = current_date.year current_month = current_date.month # Financial year in India starts in April if current_month >= 4: latest_start_year = current_year else: latest_start_year = current_year - 1 params = [] # Oldest explicitly listed year before 'Old Circulars' is 2009-2010 for year in range(latest_start_year, 2008, -1): params.append(f"yr={year}-{year+1}") params.append("yr=Old+Circulars") return params YEAR_PARAMS = generate_year_params() # --- Utility Functions --- def load_json_file(filepath): """Loads a JSON file if it exists, otherwise returns an empty dictionary.""" if os.path.exists(filepath): try: with open(filepath, 'r', encoding='utf-8') as f: return json.load(f) except json.JSONDecodeError: print(f"Warning: Could not decode JSON from {filepath}. Starting fresh.") return {} return {} def save_json_file(data, filepath): """Saves data to a JSON file.""" with open(filepath, 'w', encoding='utf-8') as f: json.dump(data, f, ensure_ascii=False, indent=4) print(f"Data saved to {filepath}") # --- Main Data Fetching Logic --- def fetch_circular_metadata(): """Fetches circular metadata from EPFO website and saves to circular-data.json.""" parsed_circulars_data = [] print("Starting to fetch circular metadata...") for q_param in YEAR_PARAMS: current_page_url = f'https://www.epfindia.gov.in/site_en/get_cir_content.php?{q_param}' print(f" Requesting URL: {current_page_url}") try: r = requests.get(current_page_url, headers=HEADERS, timeout=20) r.raise_for_status() soup = bs(r.text, 'html.parser') print(f" Status Code: {r.status_code} for {q_param}") table_rows = soup.find_all('tr') data_rows = table_rows[1:] if table_rows and table_rows[0].find('th') else table_rows for row_idx, row in enumerate(data_rows): cells = row.find_all('td') if len(cells) < 4: # print(f" Skipping row {row_idx+1} in {q_param} due to insufficient cells ({len(cells)}).") continue serial_no = cells[0].get_text(strip=True) subject_cell = cells[1] title_parts = [content.strip() for content in subject_cell.contents if isinstance(content, NavigableString) and content.strip()] title = " ".join(title_parts).split('Circular No.')[0].split('No.')[0].strip() # Basic title cleaning circular_no_date_raw = "" after_first_br = False temp_circular_parts = [] for content in subject_cell.contents: if content.name == 'br': if not after_first_br: after_first_br = True continue else: # Second br or end of relevant part break if after_first_br: if isinstance(content, NavigableString): text_content = content.strip() if text_content: temp_circular_parts.append(text_content) elif content.name == 'a' and temp_circular_parts: # Link after some text break circular_no_date_raw = " ".join(filter(None, temp_circular_parts)) circular_no = "" date_of_circular = "" delimiter_dated = " dated " delimiter_date = " date " # some entries use "date" instead of "dated" actual_delimiter = None if delimiter_dated in circular_no_date_raw.lower(): # Check lower case actual_delimiter = delimiter_dated elif delimiter_date in circular_no_date_raw.lower(): actual_delimiter = delimiter_date if actual_delimiter: # Find the actual delimiter with original casing for split delimiter_pos = circular_no_date_raw.lower().find(actual_delimiter) original_delimiter = circular_no_date_raw[delimiter_pos : delimiter_pos + len(actual_delimiter)] parts = circular_no_date_raw.split(original_delimiter, 1) circular_no = parts[0].strip() if len(parts) > 1: date_of_circular = parts[1].strip() else: circular_no = circular_no_date_raw.strip() # Further clean title from circular number if any residue if circular_no and title.endswith(circular_no): # simple check title = title[:-len(circular_no)].strip() def get_pdf_link(cell, base_url): link_tag = cell.find('a') if link_tag and link_tag.has_attr('href'): relative_link = link_tag['href'] return urljoin(base_url, relative_link) return None hindi_pdf_link = get_pdf_link(cells[2], current_page_url) english_pdf_link = get_pdf_link(cells[3], current_page_url) circular_data = { "serial_no": serial_no, "title": title, "circular_no": circular_no, "date": date_of_circular, "hindi_pdf_link": hindi_pdf_link, "english_pdf_link": english_pdf_link } parsed_circulars_data.append(circular_data) print(f" Successfully processed {q_param}") except requests.exceptions.RequestException as e: print(f" Error fetching {q_param}: {e}") except Exception as e: print(f" An error occurred during parsing for {q_param}: {e}") if parsed_circulars_data: save_json_file(parsed_circulars_data, CIRCULAR_DATA_FILE) else: print("No circular metadata was extracted.") print("Finished fetching circular metadata.") # --- Main Execution --- if __name__ == "__main__": parser = argparse.ArgumentParser(description="Fetch EPFO circular metadata.") parser.add_argument( "--action", choices=['fetch', 'all'], default='fetch', help="'fetch' updates circular-data.json. 'all' is kept as a backwards-compatible alias." ) args = parser.parse_args() if args.action in {'fetch', 'all'}: fetch_circular_metadata() print("\nScript finished.")