Download app.py from sarim/pix: direct link, hf CLI and curl.
- Browser
- Download file 21.5 kB
-
https://huggingface.co/spaces/sarim/pix/resolve/main/app.py
- Command line
-
hf download hf://spaces/sarim/pix/app.py
-
curl -L -o app.py https://huggingface.co/spaces/sarim/pix/resolve/main/app.py
21.5 kB
| from fastapi import FastAPI,HTTPException | |
| from bs4 import BeautifulSoup | |
| import time | |
| import requests | |
| from fastapi.responses import HTMLResponse | |
| import firebase_etf | |
| from starlette.responses import FileResponse | |
| from index_data import IndexEnum,INDEX_MAP | |
| from typing import Dict, List | |
| from pydantic import BaseModel | |
| from models import CommodityEnum, PsxMarketResponse,PsxStock,IndexPrice,IndexPriceFormate,DataIndex,Metal | |
| from threading import Thread | |
| from datetime import datetime,timedelta | |
| import re | |
| from psx_scraper import PsxScraper | |
| from metals_price import MetalsPrice, Commodity | |
| import os | |
| import pytz | |
| from currency_exchange import CurrencyExchange | |
| from ticker_detail import PSXScraper,build_ticker | |
| from index_detail import PSXIndicesScraper,convert_indices_to_tickers | |
| import httpx | |
| from cache_ticker import CacheTicker | |
| from tickers import PsxIndexAndTickers | |
| CACHE = { | |
| "dividends": None, | |
| "gainers": None, | |
| "last_updated": None, | |
| "circuit_breaker": None, | |
| "gold_price": None, | |
| "silver_price": None, | |
| "palladium_price": None, | |
| "copper_price": None, | |
| "platinum_price": None, | |
| 'ticker_details': None | |
| } | |
| BASE_URL = "https://www.psx.com.pk" | |
| api_key_metal = os.getenv("metail_api") | |
| app = FastAPI( | |
| title="PSX web scraper", | |
| docs_url="/", | |
| description="Scrape data from PSX website" | |
| ) | |
| def parse_dividend(dividend_str): | |
| """Convert dividend like '20%(i) (D)' into structured object""" | |
| if not dividend_str or dividend_str.strip() in ["-", "No dividend"]: | |
| return "-" | |
| match = re.search(r"(\d+(?:\.\d+)?)%(?:\(([ivx]+)\))?\s*\((\w)\)", dividend_str) | |
| if match: | |
| percentage = match.group(1) | |
| installment = match.group(2) | |
| code = match.group(3) | |
| type_map = { | |
| "D": "cash", | |
| "B": "bonus", | |
| "R": "right", | |
| "F": "final", | |
| "I": "interim" | |
| } | |
| roman_map = { | |
| "i": "1st", | |
| "ii": "2nd", | |
| "iii": "3rd", | |
| "iv": "4th", | |
| "v": "5th" | |
| } | |
| div_type = type_map.get(code, "unknown") | |
| installment_str = f" ({roman_map.get(installment, installment)})" if installment else "" | |
| return f"{percentage}% {div_type.title()} Dividend{installment_str}" | |
| return "-" | |
| def parse_board_meeting(bm_str): | |
| """Convert board meeting like '13/05/2026(EOGM)' into readable string""" | |
| if not bm_str or bm_str.strip() == "-": | |
| return "-" | |
| match = re.search(r"(\d{2}/\d{2}/\d{4})(?:\((\w+)\))?", bm_str) | |
| if match: | |
| date = match.group(1) | |
| meeting_type = match.group(2) | |
| type_map = { | |
| "EOGM": "Extra General", | |
| "AGM": "Annual General", | |
| "EGM": "Emergency General", | |
| } | |
| if meeting_type: | |
| return f"{date} — {type_map.get(meeting_type, meeting_type)}" | |
| return date | |
| return "-" | |
| def parse_year_ended(ye_str): | |
| """Convert '31/03/2026(IIIQ)' into readable string""" | |
| if not ye_str or ye_str.strip() == "-": | |
| return "-" | |
| match = re.search(r"(\d{2}/\d{2}/\d{4})\((\w+)\)", ye_str) | |
| if match: | |
| date = match.group(1) | |
| code = match.group(2) | |
| period_map = { | |
| "IQ": "1st Quarter", | |
| "IIQ": "2nd Quarter", | |
| "IIIQ": "3rd Quarter", | |
| "IVQ": "4th Quarter", | |
| "HYR": "Half Year", | |
| "YR": "Full Year", | |
| } | |
| period = period_map.get(code, code) | |
| return f"{period} ended {date}" | |
| return "-" | |
| def parse_profit_loss(pl_str): | |
| """Convert profit/loss like '33,738.884 (UCS)' into structured object""" | |
| if not pl_str or pl_str.strip() == "-": | |
| return "-" | |
| match = re.search(r"^\(?([\d,]+\.?\d*)\)?(?:\s*\((CS|UCS)\))?", pl_str.strip()) | |
| if match: | |
| value_str = match.group(1).replace(",", "") | |
| consolidated = match.group(2) | |
| is_loss = pl_str.strip().startswith("(") | |
| value = float(value_str) | |
| if is_loss: | |
| value = -value | |
| type_map = { | |
| "CS": "consolidated", | |
| "UCS": "unconsolidated", | |
| None: "standalone" | |
| } | |
| label = type_map.get(consolidated, 'standalone') | |
| formatted_value = f"{value:,.3f}" | |
| return f"{formatted_value} {label}" | |
| # return { | |
| # "value": value, | |
| # "type": type_map.get(consolidated, "standalone"), | |
| # "formatted": f"{'('+str(value_str)+')' if is_loss else value_str} {type_map.get(consolidated, '')}" | |
| # } | |
| return "-" | |
| def simple_dividend_extraction(): | |
| """ | |
| Simple function to extract dividend data | |
| """ | |
| r = requests.get('https://www.psx.com.pk/psx/announcement/financial-announcements') | |
| print("request done") | |
| soup = BeautifulSoup(r.text, 'html.parser') | |
| table = soup.find('table') | |
| _table = soup.select('list') | |
| results = [] | |
| if table: | |
| print("in table") | |
| #rows = table.find('tbody').findChildren('tr') | |
| rows = table.find_all('tr')[1:] | |
| for row in rows: | |
| if not hasattr(row, 'find_all'): | |
| print("no data") | |
| continue | |
| cells = row.find_all('td') | |
| if len(cells) >= 6: | |
| company_name = cells[0].get_text(strip=True) | |
| dividend_amount = cells[3].get_text(strip=True) or "No dividend" | |
| dividend_date = cells[8].get_text(strip=True) or "No date" | |
| board_meeting = cells[7].get_text(strip=True) or "No meeting" | |
| eps = cells[6].get_text(strip=True) or "No eps" | |
| profit_loss_before_tax = cells[4].get_text(strip=True) or "No profit/loss" | |
| profit_loss_after_tax = cells[5].get_text(strip=True) or "No profit/loss" | |
| year_ended = cells[2].get_text(strip=True) or "No profit/loss" | |
| results.append({ | |
| 'Company': company_name, | |
| 'Dividend': parse_dividend(dividend_amount), | |
| 'Date': dividend_date, | |
| 'BoardMeeting':parse_board_meeting(board_meeting), | |
| "Eps":eps, | |
| 'profitLossBeforeTax': parse_profit_loss(profit_loss_before_tax), | |
| 'profitLossAfterTax': parse_profit_loss(profit_loss_after_tax), | |
| "yearEnded": parse_year_ended(year_ended) | |
| }) | |
| return results | |
| def get_active_gainers() -> PsxMarketResponse: | |
| r = requests.get('https://www.psx.com.pk/market-summary/') | |
| print("request done") | |
| """Extract all market data from the HTML tables""" | |
| # Parse the HTML | |
| soup = BeautifulSoup(r.text, 'html.parser') | |
| main_div = soup.find('div', {'id': 'marketmainboard'}) | |
| if not main_div: | |
| print("Main div not found") | |
| return {} | |
| all_data = {} | |
| # Find all tables directly (not nested) | |
| tables = main_div.find_all('table') | |
| for i, table in enumerate(tables): | |
| # Get the sector/category name from the table header | |
| header = table.find('th') | |
| if header: | |
| h4_tag = header.find('h4') | |
| if h4_tag: | |
| sector_name = h4_tag.text.strip() | |
| else: | |
| sector_name = header.text.strip() | |
| else: | |
| sector_name = f"Sector_{i+1}" | |
| # Skip if it's not a real sector (too short or just column headers) | |
| if len(sector_name) < 5 or sector_name in ['SCRIP', 'LDCP', 'SYMBOL']: | |
| continue | |
| # Find the tbody - using find() not find_all() | |
| tbody = table.find('tbody') | |
| if not tbody: | |
| print(f" No tbody found, trying direct rows...") | |
| # Try to get rows directly from table | |
| rows = table.find_all('tr') | |
| else: | |
| rows = tbody.find_all('tr') | |
| sector_data = [] | |
| row_count = 0 | |
| # Process rows | |
| for row_idx, row in enumerate(rows): | |
| # Skip header rows | |
| row_text = row.get_text().lower() | |
| if any(x in row_text for x in ['scrip', 'ldcp', 'open', 'high', 'low', 'current', 'change', 'volume']): | |
| continue | |
| columns = row.find_all('td') | |
| # Check if this is a data row (should have 8 columns) | |
| if len(columns) >= 8: | |
| row_count += 1 | |
| try: | |
| # Extract script name and code | |
| script_name = "" | |
| script_code = "" | |
| first_col = columns[0] | |
| # Try to find dataportal class for script code | |
| dataportal_cell = first_col.find(class_='dataportal') | |
| if dataportal_cell: | |
| if dataportal_cell.has_attr('data-srip'): | |
| script_code = dataportal_cell['data-srip'] | |
| script_name = dataportal_cell.get_text(strip=True) | |
| else: | |
| script_name = first_col.get_text(strip=True) | |
| # Clean numeric values | |
| def clean_numeric(value): | |
| # Remove commas and non-numeric chars except decimal and minus | |
| cleaned = ''.join(c for c in str(value) if c.isdigit() or c in '.-') | |
| return cleaned if cleaned else "0" | |
| # Extract change value (might have span inside) | |
| change_cell = columns[6] | |
| change_text = change_cell.get_text(strip=True) | |
| # Determine trend | |
| trend = 'increase' | |
| if 'red-text-td' in row.get('class', []): | |
| trend = 'decrease' | |
| elif 'green-text-td' in row.get('class', []): | |
| trend = 'increase' | |
| stock = PsxStock( | |
| script_code=script_code, | |
| script_name=script_name, | |
| ldcp=clean_numeric(columns[1].get_text(strip=True)), | |
| open=clean_numeric(columns[2].get_text(strip=True)), | |
| high=clean_numeric(columns[3].get_text(strip=True)), | |
| low=clean_numeric(columns[4].get_text(strip=True)), | |
| current=clean_numeric(columns[5].get_text(strip=True)), | |
| change=clean_numeric(columns[6].get_text(strip=True)), | |
| volume=clean_numeric(columns[7].get_text(strip=True)), | |
| trend=trend | |
| ) | |
| sector_data.append(stock) | |
| except Exception as e: | |
| print(f" Error processing row {row_idx}: {e}") | |
| print(f" Row columns: {[col.get_text(strip=True)[:20] for col in columns]}") | |
| continue | |
| if sector_data: | |
| all_data[sector_name] = sector_data | |
| else: | |
| print(f" No data extracted from {sector_name}") | |
| return PsxMarketResponse(sectors=all_data) | |
| def get_announcements_scrap(): | |
| """ | |
| Scrape announcements from HTML content or URL | |
| Parameters: | |
| - html_content: Raw HTML string (if provided) | |
| - url: URL to fetch HTML from (if html_content not provided) | |
| """ | |
| r = requests.get('https://dps.psx.com.pk/announcements/companies') | |
| print("request done") | |
| """Extract all market data from the HTML tables""" | |
| # Parse the HTML | |
| soup = BeautifulSoup(r.text, 'html.parser') | |
| # Find the announcements table | |
| table = soup.find('table', {'id': 'announcementsTable'}) | |
| if not table: | |
| print("ERROR: Table with id 'announcementsTable' not found!") | |
| return [] | |
| print(f"✓ Found announcements table") | |
| # Extract announcements from table rows | |
| announcements = [] | |
| rows = table.find('tbody').find_all('tr') | |
| print(f"Found {len(rows)} announcement rows") | |
| for i, row in enumerate(rows, 1): | |
| try: | |
| cells = row.find_all('td') | |
| if len(cells) < 6: | |
| print(f" Row {i}: Only {len(cells)} cells, skipping...") | |
| continue | |
| # Extract date | |
| date = cells[0].get_text(strip=True) | |
| # Extract time | |
| time = cells[1].get_text(strip=True) | |
| # Extract symbol (from link if available) | |
| symbol_cell = cells[2] | |
| symbol_link = symbol_cell.find('a') | |
| symbol = symbol_link.get_text(strip=True) if symbol_link else symbol_cell.get_text(strip=True) | |
| symbol_url = symbol_link.get('href', '') if symbol_link else '' | |
| # Extract company name (from link if available) | |
| name_cell = cells[3] | |
| name_link = name_cell.find('a') | |
| name = name_link.get_text(strip=True) if name_link else name_cell.get_text(strip=True) | |
| name_url = name_link.get('href', '') if name_link else '' | |
| # Extract title | |
| title = cells[4].get_text(strip=True) | |
| # Extract links from the last cell | |
| links_cell = cells[5] | |
| # Find view link (data-images attribute) | |
| view_link = '' | |
| view_anchor = links_cell.find('a', href=lambda x: x and 'javascript:' in str(x)) | |
| if view_anchor: | |
| view_images = view_anchor.get('data-images', '') | |
| if view_images: | |
| view_link = view_images | |
| # Find PDF link | |
| pdf_link = '' | |
| pdf_anchor = links_cell.find('a', href=lambda x: x and '.pdf' in str(x)) | |
| if pdf_anchor: | |
| pdf_link = pdf_anchor.get('href', '') | |
| # Extract document ID from PDF link | |
| document_id = '' | |
| if pdf_link: | |
| match = re.search(r'/(\d+)\.pdf$', pdf_link) | |
| if match: | |
| document_id = match.group(1) | |
| elif view_link: | |
| # Try to extract from view link if no PDF link | |
| match = re.search(r'(\d+)-\d+\.', view_link) | |
| if match: | |
| document_id = match.group(1) | |
| # Create announcement dictionary | |
| announcement = { | |
| 'Row_Number': i, | |
| 'Date': date, | |
| 'Time': time, | |
| 'Symbol': symbol, | |
| 'Symbol_URL': symbol_url, | |
| 'Company_Name': name, | |
| 'Company_URL': name_url, | |
| 'Title': title, | |
| 'View_Images': view_link, | |
| 'PDF_Link': pdf_link, | |
| 'Document_ID': document_id, | |
| 'Scraped_At': datetime.now().strftime('%Y-%m-%d %H:%M:%S') | |
| } | |
| announcements.append(announcement) | |
| # Print progress for first few rows | |
| if i <= 3: | |
| print(f" Processed row {i}: {symbol} - {title[:50]}...") | |
| except Exception as e: | |
| print(f" Error processing row {i}: {e}") | |
| continue | |
| return announcements | |
| def get_metal_data(metal:str): | |
| BASE_URL = "https://api.gold-api.com/history" | |
| # Initialize the class | |
| metals_api = MetalsPrice(api_key=f"{api_key_metal}", base_url=BASE_URL) | |
| # Last 730 days (2 years) | |
| end = int(datetime.utcnow().timestamp()) | |
| start = int((datetime.utcnow() - timedelta(days=730)).timestamp()) | |
| data = metals_api.get_price_history(metal, start, end) | |
| return data | |
| def get_circuit_breakers(): | |
| r = requests.get("https://dps.psx.com.pk/circuit-breakers") | |
| scraper = PsxScraper(html_content=r.text) | |
| return { | |
| "upper": scraper.fetch_circuit_breaker_table("upperCap"), | |
| "lower": scraper.fetch_circuit_breaker_table("lowerCap"), | |
| } | |
| def get_all_tickers(): | |
| cache = CacheTicker() | |
| result = cache.get_kse_30_ticker_detail() | |
| print(result) | |
| return result | |
| def background_scrap(): | |
| while True: | |
| try: | |
| print("Updating cached data...") | |
| pakistan_tz = pytz.timezone('Asia/Karachi') | |
| current_time = datetime.now(pakistan_tz) | |
| start_time = current_time.replace(hour=9, minute=0, second=0, microsecond=0) | |
| end_time = current_time.replace(hour=17, minute=0, second=0, microsecond=0) | |
| if start_time <= current_time <= end_time: | |
| CACHE["dividends"] = simple_dividend_extraction() | |
| CACHE["gainers"] = get_active_gainers() | |
| CACHE["last_updated"] = time.time() | |
| CACHE["circuit_breaker"] = get_circuit_breakers() | |
| CACHE["gold_price"] = get_metal_data("XAU") | |
| CACHE["silver_price"] = get_metal_data(metal="XAG") | |
| CACHE['platinum_price'] = get_metal_data(metal="XPT") | |
| CACHE['palladium_price'] = get_metal_data(metal="XPD") | |
| CACHE['copper_price'] = get_metal_data(metal="HG") | |
| CACHE['ticker_details'] = get_all_tickers() | |
| print(f"Next update after 3 hours delay...") | |
| time.sleep(10800) | |
| else: | |
| next_run = current_time.replace(hour=9, minute=0, second=0, microsecond=0) | |
| if current_time > next_run: | |
| next_run += timedelta(days=1) | |
| sleep_seconds = (next_run - current_time).total_seconds() | |
| print(f"Outside working hours. Next update at {next_run.strftime('%Y-%m-%d %H:%M:%S %Z')}") | |
| time.sleep(sleep_seconds) | |
| except Exception as e: | |
| print("Background scrape error:", e) | |
| time.sleep(3600) | |
| def start_background_task(): | |
| thread = Thread(target=background_scrap,daemon=True) | |
| thread.start() | |
| def greet_json(): | |
| return {"Hello": "World!"} | |
| def get_dividend(): | |
| return CACHE["dividends"] | |
| def get_privacy_policy(): | |
| return FileResponse('text.html') | |
| def get_portfolio(): | |
| return FileResponse('portfolio.html') | |
| def get_index_companies(index_name: IndexEnum): | |
| companies = INDEX_MAP.get(index_name) | |
| if companies is None: | |
| raise HTTPException(status_code=404, detail="Index not found") | |
| return [company.to_dict() for company in companies] | |
| def get_gainers_loosers(): | |
| return CACHE["gainers"] | |
| def get_symbol_detail(symbol:str): | |
| r = requests.get(f'https://dps.psx.com.pk/company/{symbol}') | |
| scraper = PsxScraper(html_content=r.text) | |
| company_data = scraper.scrape_all_data() | |
| return company_data | |
| def get_all_circuit_breakers(): | |
| return CACHE["circuit_breaker"] | |
| def get_historical_price(metal: Metal): | |
| return CACHE[f"{metal.value}_price"] | |
| def get_all_currency(): | |
| currencyExchange = CurrencyExchange() | |
| response = currencyExchange.get_all_currency() | |
| return response | |
| def get_index_price(index_name: str): | |
| r = requests.get(f"https://dps.psx.com.pk/timeseries/int/{index_name}") | |
| response = IndexPrice.model_validate(r.json()) | |
| formatted_data = [ | |
| DataIndex( | |
| date=item[0], | |
| price=item[1], | |
| volume=item[2] | |
| ) for item in response.data | |
| ] | |
| formatted_response = IndexPriceFormate( | |
| status=response.status, | |
| message=response.message, | |
| data=formatted_data | |
| ) | |
| return formatted_response | |
| def get_all_etf(): | |
| etf = firebase_etf | |
| return etf.getAllEtf() | |
| def get_ticker_detail(symbol:str): | |
| scraper = PSXScraper(symbol) | |
| result = scraper.scrape() | |
| ticker = build_ticker(result, symbol) | |
| return ticker | |
| def get_all_indices(): | |
| scraper = PSXIndicesScraper() | |
| indices_raw = scraper.scrape() | |
| ticker_list = convert_indices_to_tickers(indices_raw) | |
| # If you want JSON serializable output | |
| result = [ticker.model_dump() for ticker in ticker_list] | |
| return result | |
| def get_all_ticker(): | |
| return CACHE['ticker_details'] | |
| def get_all_symbols(): | |
| symbols = PsxIndexAndTickers() | |
| return symbols.getAllSymbols() | |
| def get_all_index(indexName): | |
| symbols = PsxIndexAndTickers() | |
| return symbols.getIndexTicker(index=indexName) | |
| def get_all_commodity(): | |
| commodity = Commodity() | |
| return commodity.get_commodity_list() | |
| def get_single_commodity(symbol:CommodityEnum): | |
| commodity = Commodity() | |
| return commodity.get_commodity_price(symbol=symbol) | |