from fastapi import FastAPI,HTTPException from bs4 import BeautifulSoup import time import requests from fastapi.responses import HTMLResponse import firebase_etf from starlette.responses import FileResponse from index_data import IndexEnum,INDEX_MAP from typing import Dict, List from pydantic import BaseModel from models import CommodityEnum, PsxMarketResponse,PsxStock,IndexPrice,IndexPriceFormate,DataIndex,Metal from threading import Thread from datetime import datetime,timedelta import re from psx_scraper import PsxScraper from metals_price import MetalsPrice, Commodity import os import pytz from currency_exchange import CurrencyExchange from ticker_detail import PSXScraper,build_ticker from index_detail import PSXIndicesScraper,convert_indices_to_tickers import httpx from cache_ticker import CacheTicker from tickers import PsxIndexAndTickers CACHE = { "dividends": None, "gainers": None, "last_updated": None, "circuit_breaker": None, "gold_price": None, "silver_price": None, "palladium_price": None, "copper_price": None, "platinum_price": None, 'ticker_details': None } BASE_URL = "https://www.psx.com.pk" api_key_metal = os.getenv("metail_api") app = FastAPI( title="PSX web scraper", docs_url="/", description="Scrape data from PSX website" ) def parse_dividend(dividend_str): """Convert dividend like '20%(i) (D)' into structured object""" if not dividend_str or dividend_str.strip() in ["-", "No dividend"]: return "-" match = re.search(r"(\d+(?:\.\d+)?)%(?:\(([ivx]+)\))?\s*\((\w)\)", dividend_str) if match: percentage = match.group(1) installment = match.group(2) code = match.group(3) type_map = { "D": "cash", "B": "bonus", "R": "right", "F": "final", "I": "interim" } roman_map = { "i": "1st", "ii": "2nd", "iii": "3rd", "iv": "4th", "v": "5th" } div_type = type_map.get(code, "unknown") installment_str = f" ({roman_map.get(installment, installment)})" if installment else "" return f"{percentage}% {div_type.title()} Dividend{installment_str}" return "-" def parse_board_meeting(bm_str): """Convert board meeting like '13/05/2026(EOGM)' into readable string""" if not bm_str or bm_str.strip() == "-": return "-" match = re.search(r"(\d{2}/\d{2}/\d{4})(?:\((\w+)\))?", bm_str) if match: date = match.group(1) meeting_type = match.group(2) type_map = { "EOGM": "Extra General", "AGM": "Annual General", "EGM": "Emergency General", } if meeting_type: return f"{date} — {type_map.get(meeting_type, meeting_type)}" return date return "-" def parse_year_ended(ye_str): """Convert '31/03/2026(IIIQ)' into readable string""" if not ye_str or ye_str.strip() == "-": return "-" match = re.search(r"(\d{2}/\d{2}/\d{4})\((\w+)\)", ye_str) if match: date = match.group(1) code = match.group(2) period_map = { "IQ": "1st Quarter", "IIQ": "2nd Quarter", "IIIQ": "3rd Quarter", "IVQ": "4th Quarter", "HYR": "Half Year", "YR": "Full Year", } period = period_map.get(code, code) return f"{period} ended {date}" return "-" def parse_profit_loss(pl_str): """Convert profit/loss like '33,738.884 (UCS)' into structured object""" if not pl_str or pl_str.strip() == "-": return "-" match = re.search(r"^\(?([\d,]+\.?\d*)\)?(?:\s*\((CS|UCS)\))?", pl_str.strip()) if match: value_str = match.group(1).replace(",", "") consolidated = match.group(2) is_loss = pl_str.strip().startswith("(") value = float(value_str) if is_loss: value = -value type_map = { "CS": "consolidated", "UCS": "unconsolidated", None: "standalone" } label = type_map.get(consolidated, 'standalone') formatted_value = f"{value:,.3f}" return f"{formatted_value} {label}" # return { # "value": value, # "type": type_map.get(consolidated, "standalone"), # "formatted": f"{'('+str(value_str)+')' if is_loss else value_str} {type_map.get(consolidated, '')}" # } return "-" def simple_dividend_extraction(): """ Simple function to extract dividend data """ r = requests.get('https://www.psx.com.pk/psx/announcement/financial-announcements') print("request done") soup = BeautifulSoup(r.text, 'html.parser') table = soup.find('table') _table = soup.select('list') results = [] if table: print("in table") #rows = table.find('tbody').findChildren('tr') rows = table.find_all('tr')[1:] for row in rows: if not hasattr(row, 'find_all'): print("no data") continue cells = row.find_all('td') if len(cells) >= 6: company_name = cells[0].get_text(strip=True) dividend_amount = cells[3].get_text(strip=True) or "No dividend" dividend_date = cells[8].get_text(strip=True) or "No date" board_meeting = cells[7].get_text(strip=True) or "No meeting" eps = cells[6].get_text(strip=True) or "No eps" profit_loss_before_tax = cells[4].get_text(strip=True) or "No profit/loss" profit_loss_after_tax = cells[5].get_text(strip=True) or "No profit/loss" year_ended = cells[2].get_text(strip=True) or "No profit/loss" results.append({ 'Company': company_name, 'Dividend': parse_dividend(dividend_amount), 'Date': dividend_date, 'BoardMeeting':parse_board_meeting(board_meeting), "Eps":eps, 'profitLossBeforeTax': parse_profit_loss(profit_loss_before_tax), 'profitLossAfterTax': parse_profit_loss(profit_loss_after_tax), "yearEnded": parse_year_ended(year_ended) }) return results def get_active_gainers() -> PsxMarketResponse: r = requests.get('https://www.psx.com.pk/market-summary/') print("request done") """Extract all market data from the HTML tables""" # Parse the HTML soup = BeautifulSoup(r.text, 'html.parser') main_div = soup.find('div', {'id': 'marketmainboard'}) if not main_div: print("Main div not found") return {} all_data = {} # Find all tables directly (not nested) tables = main_div.find_all('table') for i, table in enumerate(tables): # Get the sector/category name from the table header header = table.find('th') if header: h4_tag = header.find('h4') if h4_tag: sector_name = h4_tag.text.strip() else: sector_name = header.text.strip() else: sector_name = f"Sector_{i+1}" # Skip if it's not a real sector (too short or just column headers) if len(sector_name) < 5 or sector_name in ['SCRIP', 'LDCP', 'SYMBOL']: continue # Find the tbody - using find() not find_all() tbody = table.find('tbody') if not tbody: print(f" No tbody found, trying direct rows...") # Try to get rows directly from table rows = table.find_all('tr') else: rows = tbody.find_all('tr') sector_data = [] row_count = 0 # Process rows for row_idx, row in enumerate(rows): # Skip header rows row_text = row.get_text().lower() if any(x in row_text for x in ['scrip', 'ldcp', 'open', 'high', 'low', 'current', 'change', 'volume']): continue columns = row.find_all('td') # Check if this is a data row (should have 8 columns) if len(columns) >= 8: row_count += 1 try: # Extract script name and code script_name = "" script_code = "" first_col = columns[0] # Try to find dataportal class for script code dataportal_cell = first_col.find(class_='dataportal') if dataportal_cell: if dataportal_cell.has_attr('data-srip'): script_code = dataportal_cell['data-srip'] script_name = dataportal_cell.get_text(strip=True) else: script_name = first_col.get_text(strip=True) # Clean numeric values def clean_numeric(value): # Remove commas and non-numeric chars except decimal and minus cleaned = ''.join(c for c in str(value) if c.isdigit() or c in '.-') return cleaned if cleaned else "0" # Extract change value (might have span inside) change_cell = columns[6] change_text = change_cell.get_text(strip=True) # Determine trend trend = 'increase' if 'red-text-td' in row.get('class', []): trend = 'decrease' elif 'green-text-td' in row.get('class', []): trend = 'increase' stock = PsxStock( script_code=script_code, script_name=script_name, ldcp=clean_numeric(columns[1].get_text(strip=True)), open=clean_numeric(columns[2].get_text(strip=True)), high=clean_numeric(columns[3].get_text(strip=True)), low=clean_numeric(columns[4].get_text(strip=True)), current=clean_numeric(columns[5].get_text(strip=True)), change=clean_numeric(columns[6].get_text(strip=True)), volume=clean_numeric(columns[7].get_text(strip=True)), trend=trend ) sector_data.append(stock) except Exception as e: print(f" Error processing row {row_idx}: {e}") print(f" Row columns: {[col.get_text(strip=True)[:20] for col in columns]}") continue if sector_data: all_data[sector_name] = sector_data else: print(f" No data extracted from {sector_name}") return PsxMarketResponse(sectors=all_data) def get_announcements_scrap(): """ Scrape announcements from HTML content or URL Parameters: - html_content: Raw HTML string (if provided) - url: URL to fetch HTML from (if html_content not provided) """ r = requests.get('https://dps.psx.com.pk/announcements/companies') print("request done") """Extract all market data from the HTML tables""" # Parse the HTML soup = BeautifulSoup(r.text, 'html.parser') # Find the announcements table table = soup.find('table', {'id': 'announcementsTable'}) if not table: print("ERROR: Table with id 'announcementsTable' not found!") return [] print(f"✓ Found announcements table") # Extract announcements from table rows announcements = [] rows = table.find('tbody').find_all('tr') print(f"Found {len(rows)} announcement rows") for i, row in enumerate(rows, 1): try: cells = row.find_all('td') if len(cells) < 6: print(f" Row {i}: Only {len(cells)} cells, skipping...") continue # Extract date date = cells[0].get_text(strip=True) # Extract time time = cells[1].get_text(strip=True) # Extract symbol (from link if available) symbol_cell = cells[2] symbol_link = symbol_cell.find('a') symbol = symbol_link.get_text(strip=True) if symbol_link else symbol_cell.get_text(strip=True) symbol_url = symbol_link.get('href', '') if symbol_link else '' # Extract company name (from link if available) name_cell = cells[3] name_link = name_cell.find('a') name = name_link.get_text(strip=True) if name_link else name_cell.get_text(strip=True) name_url = name_link.get('href', '') if name_link else '' # Extract title title = cells[4].get_text(strip=True) # Extract links from the last cell links_cell = cells[5] # Find view link (data-images attribute) view_link = '' view_anchor = links_cell.find('a', href=lambda x: x and 'javascript:' in str(x)) if view_anchor: view_images = view_anchor.get('data-images', '') if view_images: view_link = view_images # Find PDF link pdf_link = '' pdf_anchor = links_cell.find('a', href=lambda x: x and '.pdf' in str(x)) if pdf_anchor: pdf_link = pdf_anchor.get('href', '') # Extract document ID from PDF link document_id = '' if pdf_link: match = re.search(r'/(\d+)\.pdf$', pdf_link) if match: document_id = match.group(1) elif view_link: # Try to extract from view link if no PDF link match = re.search(r'(\d+)-\d+\.', view_link) if match: document_id = match.group(1) # Create announcement dictionary announcement = { 'Row_Number': i, 'Date': date, 'Time': time, 'Symbol': symbol, 'Symbol_URL': symbol_url, 'Company_Name': name, 'Company_URL': name_url, 'Title': title, 'View_Images': view_link, 'PDF_Link': pdf_link, 'Document_ID': document_id, 'Scraped_At': datetime.now().strftime('%Y-%m-%d %H:%M:%S') } announcements.append(announcement) # Print progress for first few rows if i <= 3: print(f" Processed row {i}: {symbol} - {title[:50]}...") except Exception as e: print(f" Error processing row {i}: {e}") continue return announcements def get_metal_data(metal:str): BASE_URL = "https://api.gold-api.com/history" # Initialize the class metals_api = MetalsPrice(api_key=f"{api_key_metal}", base_url=BASE_URL) # Last 730 days (2 years) end = int(datetime.utcnow().timestamp()) start = int((datetime.utcnow() - timedelta(days=730)).timestamp()) data = metals_api.get_price_history(metal, start, end) return data def get_circuit_breakers(): r = requests.get("https://dps.psx.com.pk/circuit-breakers") scraper = PsxScraper(html_content=r.text) return { "upper": scraper.fetch_circuit_breaker_table("upperCap"), "lower": scraper.fetch_circuit_breaker_table("lowerCap"), } def get_all_tickers(): cache = CacheTicker() result = cache.get_kse_30_ticker_detail() print(result) return result def background_scrap(): while True: try: print("Updating cached data...") pakistan_tz = pytz.timezone('Asia/Karachi') current_time = datetime.now(pakistan_tz) start_time = current_time.replace(hour=9, minute=0, second=0, microsecond=0) end_time = current_time.replace(hour=17, minute=0, second=0, microsecond=0) if start_time <= current_time <= end_time: CACHE["dividends"] = simple_dividend_extraction() CACHE["gainers"] = get_active_gainers() CACHE["last_updated"] = time.time() CACHE["circuit_breaker"] = get_circuit_breakers() CACHE["gold_price"] = get_metal_data("XAU") CACHE["silver_price"] = get_metal_data(metal="XAG") CACHE['platinum_price'] = get_metal_data(metal="XPT") CACHE['palladium_price'] = get_metal_data(metal="XPD") CACHE['copper_price'] = get_metal_data(metal="HG") CACHE['ticker_details'] = get_all_tickers() print(f"Next update after 3 hours delay...") time.sleep(10800) else: next_run = current_time.replace(hour=9, minute=0, second=0, microsecond=0) if current_time > next_run: next_run += timedelta(days=1) sleep_seconds = (next_run - current_time).total_seconds() print(f"Outside working hours. Next update at {next_run.strftime('%Y-%m-%d %H:%M:%S %Z')}") time.sleep(sleep_seconds) except Exception as e: print("Background scrape error:", e) time.sleep(3600) @app.on_event("startup") def start_background_task(): thread = Thread(target=background_scrap,daemon=True) thread.start() @app.get("/hello") def greet_json(): return {"Hello": "World!"} @app.get("/dividend_history") def get_dividend(): return CACHE["dividends"] @app.get("/PrivacyPolicy") def get_privacy_policy(): return FileResponse('text.html') @app.get("/portfolio") def get_portfolio(): return FileResponse('portfolio.html') @app.get("/index/{index_name}") def get_index_companies(index_name: IndexEnum): companies = INDEX_MAP.get(index_name) if companies is None: raise HTTPException(status_code=404, detail="Index not found") return [company.to_dict() for company in companies] @app.get("/gainer_loosers") def get_gainers_loosers(): return CACHE["gainers"] @app.get("/get_symbol_detail{symbol}") def get_symbol_detail(symbol:str): r = requests.get(f'https://dps.psx.com.pk/company/{symbol}') scraper = PsxScraper(html_content=r.text) company_data = scraper.scrape_all_data() return company_data @app.get("/circuit-breakers/all") def get_all_circuit_breakers(): return CACHE["circuit_breaker"] @app.get("/{metal}_price") def get_historical_price(metal: Metal): return CACHE[f"{metal.value}_price"] @app.get("/get_all_currency") def get_all_currency(): currencyExchange = CurrencyExchange() response = currencyExchange.get_all_currency() return response @app.get("/get_index_price/{index_name}") def get_index_price(index_name: str): r = requests.get(f"https://dps.psx.com.pk/timeseries/int/{index_name}") response = IndexPrice.model_validate(r.json()) formatted_data = [ DataIndex( date=item[0], price=item[1], volume=item[2] ) for item in response.data ] formatted_response = IndexPriceFormate( status=response.status, message=response.message, data=formatted_data ) return formatted_response @app.get("/get_all_etfs") def get_all_etf(): etf = firebase_etf return etf.getAllEtf() @app.get("/ticker/{symbol}") def get_ticker_detail(symbol:str): scraper = PSXScraper(symbol) result = scraper.scrape() ticker = build_ticker(result, symbol) return ticker @app.get("/indices") def get_all_indices(): scraper = PSXIndicesScraper() indices_raw = scraper.scrape() ticker_list = convert_indices_to_tickers(indices_raw) # If you want JSON serializable output result = [ticker.model_dump() for ticker in ticker_list] return result @app.get("/get_all_ticker") def get_all_ticker(): return CACHE['ticker_details'] @app.get("/get_all_symbols") def get_all_symbols(): symbols = PsxIndexAndTickers() return symbols.getAllSymbols() @app.get("/get_all_index") def get_all_index(indexName): symbols = PsxIndexAndTickers() return symbols.getIndexTicker(index=indexName) @app.get("/get_all_commodity") def get_all_commodity(): commodity = Commodity() return commodity.get_commodity_list() @app.get("/get_single_commodity/{symbol}") def get_single_commodity(symbol:CommodityEnum): commodity = Commodity() return commodity.get_commodity_price(symbol=symbol)