pix / app.py
sarim's picture
update urls
7d68ccf
Raw History Blame Contribute Delete
21.5 kB
from fastapi import FastAPI,HTTPException
from bs4 import BeautifulSoup
import time
import requests
from fastapi.responses import HTMLResponse
import firebase_etf
from starlette.responses import FileResponse
from index_data import IndexEnum,INDEX_MAP
from typing import Dict, List
from pydantic import BaseModel
from models import CommodityEnum, PsxMarketResponse,PsxStock,IndexPrice,IndexPriceFormate,DataIndex,Metal
from threading import Thread
from datetime import datetime,timedelta
import re
from psx_scraper import PsxScraper
from metals_price import MetalsPrice, Commodity
import os
import pytz
from currency_exchange import CurrencyExchange
from ticker_detail import PSXScraper,build_ticker
from index_detail import PSXIndicesScraper,convert_indices_to_tickers
import httpx
from cache_ticker import CacheTicker
from tickers import PsxIndexAndTickers
CACHE = {
"dividends": None,
"gainers": None,
"last_updated": None,
"circuit_breaker": None,
"gold_price": None,
"silver_price": None,
"palladium_price": None,
"copper_price": None,
"platinum_price": None,
'ticker_details': None
}
BASE_URL = "https://www.psx.com.pk"
api_key_metal = os.getenv("metail_api")
app = FastAPI(
title="PSX web scraper",
docs_url="/",
description="Scrape data from PSX website"
)
def parse_dividend(dividend_str):
"""Convert dividend like '20%(i) (D)' into structured object"""
if not dividend_str or dividend_str.strip() in ["-", "No dividend"]:
return "-"
match = re.search(r"(\d+(?:\.\d+)?)%(?:\(([ivx]+)\))?\s*\((\w)\)", dividend_str)
if match:
percentage = match.group(1)
installment = match.group(2)
code = match.group(3)
type_map = {
"D": "cash",
"B": "bonus",
"R": "right",
"F": "final",
"I": "interim"
}
roman_map = {
"i": "1st",
"ii": "2nd",
"iii": "3rd",
"iv": "4th",
"v": "5th"
}
div_type = type_map.get(code, "unknown")
installment_str = f" ({roman_map.get(installment, installment)})" if installment else ""
return f"{percentage}% {div_type.title()} Dividend{installment_str}"
return "-"
def parse_board_meeting(bm_str):
"""Convert board meeting like '13/05/2026(EOGM)' into readable string"""
if not bm_str or bm_str.strip() == "-":
return "-"
match = re.search(r"(\d{2}/\d{2}/\d{4})(?:\((\w+)\))?", bm_str)
if match:
date = match.group(1)
meeting_type = match.group(2)
type_map = {
"EOGM": "Extra General",
"AGM": "Annual General",
"EGM": "Emergency General",
}
if meeting_type:
return f"{date} — {type_map.get(meeting_type, meeting_type)}"
return date
return "-"
def parse_year_ended(ye_str):
"""Convert '31/03/2026(IIIQ)' into readable string"""
if not ye_str or ye_str.strip() == "-":
return "-"
match = re.search(r"(\d{2}/\d{2}/\d{4})\((\w+)\)", ye_str)
if match:
date = match.group(1)
code = match.group(2)
period_map = {
"IQ": "1st Quarter",
"IIQ": "2nd Quarter",
"IIIQ": "3rd Quarter",
"IVQ": "4th Quarter",
"HYR": "Half Year",
"YR": "Full Year",
}
period = period_map.get(code, code)
return f"{period} ended {date}"
return "-"
def parse_profit_loss(pl_str):
"""Convert profit/loss like '33,738.884 (UCS)' into structured object"""
if not pl_str or pl_str.strip() == "-":
return "-"
match = re.search(r"^\(?([\d,]+\.?\d*)\)?(?:\s*\((CS|UCS)\))?", pl_str.strip())
if match:
value_str = match.group(1).replace(",", "")
consolidated = match.group(2)
is_loss = pl_str.strip().startswith("(")
value = float(value_str)
if is_loss:
value = -value
type_map = {
"CS": "consolidated",
"UCS": "unconsolidated",
None: "standalone"
}
label = type_map.get(consolidated, 'standalone')
formatted_value = f"{value:,.3f}"
return f"{formatted_value} {label}"
# return {
# "value": value,
# "type": type_map.get(consolidated, "standalone"),
# "formatted": f"{'('+str(value_str)+')' if is_loss else value_str} {type_map.get(consolidated, '')}"
# }
return "-"
def simple_dividend_extraction():
"""
Simple function to extract dividend data
"""
r = requests.get('https://www.psx.com.pk/psx/announcement/financial-announcements')
print("request done")
soup = BeautifulSoup(r.text, 'html.parser')
table = soup.find('table')
_table = soup.select('list')
results = []
if table:
print("in table")
#rows = table.find('tbody').findChildren('tr')
rows = table.find_all('tr')[1:]
for row in rows:
if not hasattr(row, 'find_all'):
print("no data")
continue
cells = row.find_all('td')
if len(cells) >= 6:
company_name = cells[0].get_text(strip=True)
dividend_amount = cells[3].get_text(strip=True) or "No dividend"
dividend_date = cells[8].get_text(strip=True) or "No date"
board_meeting = cells[7].get_text(strip=True) or "No meeting"
eps = cells[6].get_text(strip=True) or "No eps"
profit_loss_before_tax = cells[4].get_text(strip=True) or "No profit/loss"
profit_loss_after_tax = cells[5].get_text(strip=True) or "No profit/loss"
year_ended = cells[2].get_text(strip=True) or "No profit/loss"
results.append({
'Company': company_name,
'Dividend': parse_dividend(dividend_amount),
'Date': dividend_date,
'BoardMeeting':parse_board_meeting(board_meeting),
"Eps":eps,
'profitLossBeforeTax': parse_profit_loss(profit_loss_before_tax),
'profitLossAfterTax': parse_profit_loss(profit_loss_after_tax),
"yearEnded": parse_year_ended(year_ended)
})
return results
def get_active_gainers() -> PsxMarketResponse:
r = requests.get('https://www.psx.com.pk/market-summary/')
print("request done")
"""Extract all market data from the HTML tables"""
# Parse the HTML
soup = BeautifulSoup(r.text, 'html.parser')
main_div = soup.find('div', {'id': 'marketmainboard'})
if not main_div:
print("Main div not found")
return {}
all_data = {}
# Find all tables directly (not nested)
tables = main_div.find_all('table')
for i, table in enumerate(tables):
# Get the sector/category name from the table header
header = table.find('th')
if header:
h4_tag = header.find('h4')
if h4_tag:
sector_name = h4_tag.text.strip()
else:
sector_name = header.text.strip()
else:
sector_name = f"Sector_{i+1}"
# Skip if it's not a real sector (too short or just column headers)
if len(sector_name) < 5 or sector_name in ['SCRIP', 'LDCP', 'SYMBOL']:
continue
# Find the tbody - using find() not find_all()
tbody = table.find('tbody')
if not tbody:
print(f" No tbody found, trying direct rows...")
# Try to get rows directly from table
rows = table.find_all('tr')
else:
rows = tbody.find_all('tr')
sector_data = []
row_count = 0
# Process rows
for row_idx, row in enumerate(rows):
# Skip header rows
row_text = row.get_text().lower()
if any(x in row_text for x in ['scrip', 'ldcp', 'open', 'high', 'low', 'current', 'change', 'volume']):
continue
columns = row.find_all('td')
# Check if this is a data row (should have 8 columns)
if len(columns) >= 8:
row_count += 1
try:
# Extract script name and code
script_name = ""
script_code = ""
first_col = columns[0]
# Try to find dataportal class for script code
dataportal_cell = first_col.find(class_='dataportal')
if dataportal_cell:
if dataportal_cell.has_attr('data-srip'):
script_code = dataportal_cell['data-srip']
script_name = dataportal_cell.get_text(strip=True)
else:
script_name = first_col.get_text(strip=True)
# Clean numeric values
def clean_numeric(value):
# Remove commas and non-numeric chars except decimal and minus
cleaned = ''.join(c for c in str(value) if c.isdigit() or c in '.-')
return cleaned if cleaned else "0"
# Extract change value (might have span inside)
change_cell = columns[6]
change_text = change_cell.get_text(strip=True)
# Determine trend
trend = 'increase'
if 'red-text-td' in row.get('class', []):
trend = 'decrease'
elif 'green-text-td' in row.get('class', []):
trend = 'increase'
stock = PsxStock(
script_code=script_code,
script_name=script_name,
ldcp=clean_numeric(columns[1].get_text(strip=True)),
open=clean_numeric(columns[2].get_text(strip=True)),
high=clean_numeric(columns[3].get_text(strip=True)),
low=clean_numeric(columns[4].get_text(strip=True)),
current=clean_numeric(columns[5].get_text(strip=True)),
change=clean_numeric(columns[6].get_text(strip=True)),
volume=clean_numeric(columns[7].get_text(strip=True)),
trend=trend
)
sector_data.append(stock)
except Exception as e:
print(f" Error processing row {row_idx}: {e}")
print(f" Row columns: {[col.get_text(strip=True)[:20] for col in columns]}")
continue
if sector_data:
all_data[sector_name] = sector_data
else:
print(f" No data extracted from {sector_name}")
return PsxMarketResponse(sectors=all_data)
def get_announcements_scrap():
"""
Scrape announcements from HTML content or URL
Parameters:
- html_content: Raw HTML string (if provided)
- url: URL to fetch HTML from (if html_content not provided)
"""
r = requests.get('https://dps.psx.com.pk/announcements/companies')
print("request done")
"""Extract all market data from the HTML tables"""
# Parse the HTML
soup = BeautifulSoup(r.text, 'html.parser')
# Find the announcements table
table = soup.find('table', {'id': 'announcementsTable'})
if not table:
print("ERROR: Table with id 'announcementsTable' not found!")
return []
print(f"✓ Found announcements table")
# Extract announcements from table rows
announcements = []
rows = table.find('tbody').find_all('tr')
print(f"Found {len(rows)} announcement rows")
for i, row in enumerate(rows, 1):
try:
cells = row.find_all('td')
if len(cells) < 6:
print(f" Row {i}: Only {len(cells)} cells, skipping...")
continue
# Extract date
date = cells[0].get_text(strip=True)
# Extract time
time = cells[1].get_text(strip=True)
# Extract symbol (from link if available)
symbol_cell = cells[2]
symbol_link = symbol_cell.find('a')
symbol = symbol_link.get_text(strip=True) if symbol_link else symbol_cell.get_text(strip=True)
symbol_url = symbol_link.get('href', '') if symbol_link else ''
# Extract company name (from link if available)
name_cell = cells[3]
name_link = name_cell.find('a')
name = name_link.get_text(strip=True) if name_link else name_cell.get_text(strip=True)
name_url = name_link.get('href', '') if name_link else ''
# Extract title
title = cells[4].get_text(strip=True)
# Extract links from the last cell
links_cell = cells[5]
# Find view link (data-images attribute)
view_link = ''
view_anchor = links_cell.find('a', href=lambda x: x and 'javascript:' in str(x))
if view_anchor:
view_images = view_anchor.get('data-images', '')
if view_images:
view_link = view_images
# Find PDF link
pdf_link = ''
pdf_anchor = links_cell.find('a', href=lambda x: x and '.pdf' in str(x))
if pdf_anchor:
pdf_link = pdf_anchor.get('href', '')
# Extract document ID from PDF link
document_id = ''
if pdf_link:
match = re.search(r'/(\d+)\.pdf$', pdf_link)
if match:
document_id = match.group(1)
elif view_link:
# Try to extract from view link if no PDF link
match = re.search(r'(\d+)-\d+\.', view_link)
if match:
document_id = match.group(1)
# Create announcement dictionary
announcement = {
'Row_Number': i,
'Date': date,
'Time': time,
'Symbol': symbol,
'Symbol_URL': symbol_url,
'Company_Name': name,
'Company_URL': name_url,
'Title': title,
'View_Images': view_link,
'PDF_Link': pdf_link,
'Document_ID': document_id,
'Scraped_At': datetime.now().strftime('%Y-%m-%d %H:%M:%S')
}
announcements.append(announcement)
# Print progress for first few rows
if i <= 3:
print(f" Processed row {i}: {symbol} - {title[:50]}...")
except Exception as e:
print(f" Error processing row {i}: {e}")
continue
return announcements
def get_metal_data(metal:str):
BASE_URL = "https://api.gold-api.com/history"
# Initialize the class
metals_api = MetalsPrice(api_key=f"{api_key_metal}", base_url=BASE_URL)
# Last 730 days (2 years)
end = int(datetime.utcnow().timestamp())
start = int((datetime.utcnow() - timedelta(days=730)).timestamp())
data = metals_api.get_price_history(metal, start, end)
return data
def get_circuit_breakers():
r = requests.get("https://dps.psx.com.pk/circuit-breakers")
scraper = PsxScraper(html_content=r.text)
return {
"upper": scraper.fetch_circuit_breaker_table("upperCap"),
"lower": scraper.fetch_circuit_breaker_table("lowerCap"),
}
def get_all_tickers():
cache = CacheTicker()
result = cache.get_kse_30_ticker_detail()
print(result)
return result
def background_scrap():
while True:
try:
print("Updating cached data...")
pakistan_tz = pytz.timezone('Asia/Karachi')
current_time = datetime.now(pakistan_tz)
start_time = current_time.replace(hour=9, minute=0, second=0, microsecond=0)
end_time = current_time.replace(hour=17, minute=0, second=0, microsecond=0)
if start_time <= current_time <= end_time:
CACHE["dividends"] = simple_dividend_extraction()
CACHE["gainers"] = get_active_gainers()
CACHE["last_updated"] = time.time()
CACHE["circuit_breaker"] = get_circuit_breakers()
CACHE["gold_price"] = get_metal_data("XAU")
CACHE["silver_price"] = get_metal_data(metal="XAG")
CACHE['platinum_price'] = get_metal_data(metal="XPT")
CACHE['palladium_price'] = get_metal_data(metal="XPD")
CACHE['copper_price'] = get_metal_data(metal="HG")
CACHE['ticker_details'] = get_all_tickers()
print(f"Next update after 3 hours delay...")
time.sleep(10800)
else:
next_run = current_time.replace(hour=9, minute=0, second=0, microsecond=0)
if current_time > next_run:
next_run += timedelta(days=1)
sleep_seconds = (next_run - current_time).total_seconds()
print(f"Outside working hours. Next update at {next_run.strftime('%Y-%m-%d %H:%M:%S %Z')}")
time.sleep(sleep_seconds)
except Exception as e:
print("Background scrape error:", e)
time.sleep(3600)
@app.on_event("startup")
def start_background_task():
thread = Thread(target=background_scrap,daemon=True)
thread.start()
@app.get("/hello")
def greet_json():
return {"Hello": "World!"}
@app.get("/dividend_history")
def get_dividend():
return CACHE["dividends"]
@app.get("/PrivacyPolicy")
def get_privacy_policy():
return FileResponse('text.html')
@app.get("/portfolio")
def get_portfolio():
return FileResponse('portfolio.html')
@app.get("/index/{index_name}")
def get_index_companies(index_name: IndexEnum):
companies = INDEX_MAP.get(index_name)
if companies is None:
raise HTTPException(status_code=404, detail="Index not found")
return [company.to_dict() for company in companies]
@app.get("/gainer_loosers")
def get_gainers_loosers():
return CACHE["gainers"]
@app.get("/get_symbol_detail{symbol}")
def get_symbol_detail(symbol:str):
r = requests.get(f'https://dps.psx.com.pk/company/{symbol}')
scraper = PsxScraper(html_content=r.text)
company_data = scraper.scrape_all_data()
return company_data
@app.get("/circuit-breakers/all")
def get_all_circuit_breakers():
return CACHE["circuit_breaker"]
@app.get("/{metal}_price")
def get_historical_price(metal: Metal):
return CACHE[f"{metal.value}_price"]
@app.get("/get_all_currency")
def get_all_currency():
currencyExchange = CurrencyExchange()
response = currencyExchange.get_all_currency()
return response
@app.get("/get_index_price/{index_name}")
def get_index_price(index_name: str):
r = requests.get(f"https://dps.psx.com.pk/timeseries/int/{index_name}")
response = IndexPrice.model_validate(r.json())
formatted_data = [
DataIndex(
date=item[0],
price=item[1],
volume=item[2]
) for item in response.data
]
formatted_response = IndexPriceFormate(
status=response.status,
message=response.message,
data=formatted_data
)
return formatted_response
@app.get("/get_all_etfs")
def get_all_etf():
etf = firebase_etf
return etf.getAllEtf()
@app.get("/ticker/{symbol}")
def get_ticker_detail(symbol:str):
scraper = PSXScraper(symbol)
result = scraper.scrape()
ticker = build_ticker(result, symbol)
return ticker
@app.get("/indices")
def get_all_indices():
scraper = PSXIndicesScraper()
indices_raw = scraper.scrape()
ticker_list = convert_indices_to_tickers(indices_raw)
# If you want JSON serializable output
result = [ticker.model_dump() for ticker in ticker_list]
return result
@app.get("/get_all_ticker")
def get_all_ticker():
return CACHE['ticker_details']
@app.get("/get_all_symbols")
def get_all_symbols():
symbols = PsxIndexAndTickers()
return symbols.getAllSymbols()
@app.get("/get_all_index")
def get_all_index(indexName):
symbols = PsxIndexAndTickers()
return symbols.getIndexTicker(index=indexName)
@app.get("/get_all_commodity")
def get_all_commodity():
commodity = Commodity()
return commodity.get_commodity_list()
@app.get("/get_single_commodity/{symbol}")
def get_single_commodity(symbol:CommodityEnum):
commodity = Commodity()
return commodity.get_commodity_price(symbol=symbol)