more testing
Browse files
app.py
CHANGED
|
@@ -9,7 +9,8 @@ from typing import Dict, List
|
|
| 9 |
from pydantic import BaseModel
|
| 10 |
from models import PsxMarketResponse,PsxStock
|
| 11 |
from threading import Thread
|
| 12 |
-
from
|
|
|
|
| 13 |
|
| 14 |
|
| 15 |
CACHE = {
|
|
@@ -226,112 +227,107 @@ def get_announcements_scrap():
|
|
| 226 |
|
| 227 |
# Parse the HTML
|
| 228 |
soup = BeautifulSoup(r.text, 'html.parser')
|
|
|
|
| 229 |
|
| 230 |
|
| 231 |
# Find the announcements table
|
| 232 |
table = soup.find('table', {'id': 'announcementsTable'})
|
| 233 |
|
| 234 |
if not table:
|
| 235 |
-
print("
|
| 236 |
return []
|
| 237 |
|
| 238 |
-
|
| 239 |
-
headers = []
|
| 240 |
-
thead = table.find('thead')
|
| 241 |
-
if thead:
|
| 242 |
-
th_elements = thead.find_all('th')
|
| 243 |
-
headers = [th.get_text(strip=True) for th in th_elements]
|
| 244 |
-
# Add extra headers for links
|
| 245 |
-
headers.extend(['View_Link', 'PDF_Link', 'Document_ID'])
|
| 246 |
|
| 247 |
-
# Extract rows
|
| 248 |
announcements = []
|
| 249 |
-
|
| 250 |
|
| 251 |
-
|
| 252 |
-
print("No table body found")
|
| 253 |
-
return []
|
| 254 |
-
|
| 255 |
-
rows = tbody.find_all('tr')
|
| 256 |
|
| 257 |
-
for row in rows:
|
| 258 |
-
cells = row.find_all('td')
|
| 259 |
-
|
| 260 |
-
if len(cells) < 5:
|
| 261 |
-
continue
|
| 262 |
-
|
| 263 |
-
# Extract basic information
|
| 264 |
-
date_str = cells[0].get_text(strip=True)
|
| 265 |
-
time_str = cells[1].get_text(strip=True)
|
| 266 |
-
|
| 267 |
-
# Extract symbol and name with links
|
| 268 |
-
symbol_cell = cells[2]
|
| 269 |
-
symbol_link = symbol_cell.find('a')
|
| 270 |
-
symbol = symbol_link.get_text(strip=True) if symbol_link else symbol_cell.get_text(strip=True)
|
| 271 |
-
symbol_url = symbol_link.get('href') if symbol_link else ''
|
| 272 |
-
|
| 273 |
-
name_cell = cells[3]
|
| 274 |
-
name_link = name_cell.find('a')
|
| 275 |
-
name = name_link.get_text(strip=True) if name_link else name_cell.get_text(strip=True)
|
| 276 |
-
name_url = name_link.get('href') if name_link else ''
|
| 277 |
-
|
| 278 |
-
title = cells[4].get_text(strip=True)
|
| 279 |
-
|
| 280 |
-
# Extract action links (view and PDF)
|
| 281 |
-
action_cell = cells[5]
|
| 282 |
-
|
| 283 |
-
# Find view link
|
| 284 |
-
view_link = ''
|
| 285 |
-
view_anchor = action_cell.find('a', href=lambda x: x and 'javascript:' in x)
|
| 286 |
-
if view_anchor:
|
| 287 |
-
view_images = view_anchor.get('data-images', '')
|
| 288 |
-
if view_images:
|
| 289 |
-
view_link = f"View images: {view_images}"
|
| 290 |
-
|
| 291 |
-
# Find PDF link
|
| 292 |
-
pdf_link = ''
|
| 293 |
-
pdf_anchor = action_cell.find('a', href=lambda x: x and '.pdf' in x)
|
| 294 |
-
if pdf_anchor:
|
| 295 |
-
pdf_link = pdf_anchor.get('href', '')
|
| 296 |
-
# Make absolute URL if relative
|
| 297 |
-
if pdf_link.startswith('/'):
|
| 298 |
-
pdf_link = f"https://your-base-url.com{pdf_link}"
|
| 299 |
-
|
| 300 |
-
# Extract document ID from PDF link
|
| 301 |
-
document_id = ''
|
| 302 |
-
if pdf_link:
|
| 303 |
-
# Extract numeric ID from URL
|
| 304 |
-
import re
|
| 305 |
-
match = re.search(r'/(\d+)\.pdf$', pdf_link)
|
| 306 |
-
if match:
|
| 307 |
-
document_id = match.group(1)
|
| 308 |
-
|
| 309 |
-
# Combine date and time
|
| 310 |
try:
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
'
|
| 326 |
-
|
| 327 |
-
'
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 335 |
|
| 336 |
return announcements
|
| 337 |
|
|
|
|
| 9 |
from pydantic import BaseModel
|
| 10 |
from models import PsxMarketResponse,PsxStock
|
| 11 |
from threading import Thread
|
| 12 |
+
from datetime import datetime
|
| 13 |
+
import re
|
| 14 |
|
| 15 |
|
| 16 |
CACHE = {
|
|
|
|
| 227 |
|
| 228 |
# Parse the HTML
|
| 229 |
soup = BeautifulSoup(r.text, 'html.parser')
|
| 230 |
+
print(r.text)
|
| 231 |
|
| 232 |
|
| 233 |
# Find the announcements table
|
| 234 |
table = soup.find('table', {'id': 'announcementsTable'})
|
| 235 |
|
| 236 |
if not table:
|
| 237 |
+
print("ERROR: Table with id 'announcementsTable' not found!")
|
| 238 |
return []
|
| 239 |
|
| 240 |
+
print(f"✓ Found announcements table")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 241 |
|
| 242 |
+
# Extract announcements from table rows
|
| 243 |
announcements = []
|
| 244 |
+
rows = table.find('tbody').find_all('tr')
|
| 245 |
|
| 246 |
+
print(f"Found {len(rows)} announcement rows")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 247 |
|
| 248 |
+
for i, row in enumerate(rows, 1):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 249 |
try:
|
| 250 |
+
cells = row.find_all('td')
|
| 251 |
+
|
| 252 |
+
if len(cells) < 6:
|
| 253 |
+
print(f" Row {i}: Only {len(cells)} cells, skipping...")
|
| 254 |
+
continue
|
| 255 |
+
|
| 256 |
+
# Extract date
|
| 257 |
+
date = cells[0].get_text(strip=True)
|
| 258 |
+
|
| 259 |
+
# Extract time
|
| 260 |
+
time = cells[1].get_text(strip=True)
|
| 261 |
+
|
| 262 |
+
# Extract symbol (from link if available)
|
| 263 |
+
symbol_cell = cells[2]
|
| 264 |
+
symbol_link = symbol_cell.find('a')
|
| 265 |
+
symbol = symbol_link.get_text(strip=True) if symbol_link else symbol_cell.get_text(strip=True)
|
| 266 |
+
symbol_url = symbol_link.get('href', '') if symbol_link else ''
|
| 267 |
+
|
| 268 |
+
# Extract company name (from link if available)
|
| 269 |
+
name_cell = cells[3]
|
| 270 |
+
name_link = name_cell.find('a')
|
| 271 |
+
name = name_link.get_text(strip=True) if name_link else name_cell.get_text(strip=True)
|
| 272 |
+
name_url = name_link.get('href', '') if name_link else ''
|
| 273 |
+
|
| 274 |
+
# Extract title
|
| 275 |
+
title = cells[4].get_text(strip=True)
|
| 276 |
+
|
| 277 |
+
# Extract links from the last cell
|
| 278 |
+
links_cell = cells[5]
|
| 279 |
+
|
| 280 |
+
# Find view link (data-images attribute)
|
| 281 |
+
view_link = ''
|
| 282 |
+
view_anchor = links_cell.find('a', href=lambda x: x and 'javascript:' in str(x))
|
| 283 |
+
if view_anchor:
|
| 284 |
+
view_images = view_anchor.get('data-images', '')
|
| 285 |
+
if view_images:
|
| 286 |
+
view_link = view_images
|
| 287 |
+
|
| 288 |
+
# Find PDF link
|
| 289 |
+
pdf_link = ''
|
| 290 |
+
pdf_anchor = links_cell.find('a', href=lambda x: x and '.pdf' in str(x))
|
| 291 |
+
if pdf_anchor:
|
| 292 |
+
pdf_link = pdf_anchor.get('href', '')
|
| 293 |
+
|
| 294 |
+
# Extract document ID from PDF link
|
| 295 |
+
document_id = ''
|
| 296 |
+
if pdf_link:
|
| 297 |
+
match = re.search(r'/(\d+)\.pdf$', pdf_link)
|
| 298 |
+
if match:
|
| 299 |
+
document_id = match.group(1)
|
| 300 |
+
elif view_link:
|
| 301 |
+
# Try to extract from view link if no PDF link
|
| 302 |
+
match = re.search(r'(\d+)-\d+\.', view_link)
|
| 303 |
+
if match:
|
| 304 |
+
document_id = match.group(1)
|
| 305 |
+
|
| 306 |
+
# Create announcement dictionary
|
| 307 |
+
announcement = {
|
| 308 |
+
'Row_Number': i,
|
| 309 |
+
'Date': date,
|
| 310 |
+
'Time': time,
|
| 311 |
+
'Symbol': symbol,
|
| 312 |
+
'Symbol_URL': symbol_url,
|
| 313 |
+
'Company_Name': name,
|
| 314 |
+
'Company_URL': name_url,
|
| 315 |
+
'Title': title,
|
| 316 |
+
'View_Images': view_link,
|
| 317 |
+
'PDF_Link': pdf_link,
|
| 318 |
+
'Document_ID': document_id,
|
| 319 |
+
'Scraped_At': datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
| 320 |
+
}
|
| 321 |
+
|
| 322 |
+
announcements.append(announcement)
|
| 323 |
+
|
| 324 |
+
# Print progress for first few rows
|
| 325 |
+
if i <= 3:
|
| 326 |
+
print(f" Processed row {i}: {symbol} - {title[:50]}...")
|
| 327 |
+
|
| 328 |
+
except Exception as e:
|
| 329 |
+
print(f" Error processing row {i}: {e}")
|
| 330 |
+
continue
|
| 331 |
|
| 332 |
return announcements
|
| 333 |
|