sarim commited on
Commit
a34fd5e
·
1 Parent(s): 4597da0

more testing

Browse files
Files changed (1) hide show
  1. app.py +90 -94
app.py CHANGED
@@ -9,7 +9,8 @@ from typing import Dict, List
9
  from pydantic import BaseModel
10
  from models import PsxMarketResponse,PsxStock
11
  from threading import Thread
12
- from urllib.parse import urljoin
 
13
 
14
 
15
  CACHE = {
@@ -226,112 +227,107 @@ def get_announcements_scrap():
226
 
227
  # Parse the HTML
228
  soup = BeautifulSoup(r.text, 'html.parser')
 
229
 
230
 
231
  # Find the announcements table
232
  table = soup.find('table', {'id': 'announcementsTable'})
233
 
234
  if not table:
235
- print("No announcements table found")
236
  return []
237
 
238
- # Extract headers
239
- headers = []
240
- thead = table.find('thead')
241
- if thead:
242
- th_elements = thead.find_all('th')
243
- headers = [th.get_text(strip=True) for th in th_elements]
244
- # Add extra headers for links
245
- headers.extend(['View_Link', 'PDF_Link', 'Document_ID'])
246
 
247
- # Extract rows
248
  announcements = []
249
- tbody = table.find('tbody')
250
 
251
- if not tbody:
252
- print("No table body found")
253
- return []
254
-
255
- rows = tbody.find_all('tr')
256
 
257
- for row in rows:
258
- cells = row.find_all('td')
259
-
260
- if len(cells) < 5:
261
- continue
262
-
263
- # Extract basic information
264
- date_str = cells[0].get_text(strip=True)
265
- time_str = cells[1].get_text(strip=True)
266
-
267
- # Extract symbol and name with links
268
- symbol_cell = cells[2]
269
- symbol_link = symbol_cell.find('a')
270
- symbol = symbol_link.get_text(strip=True) if symbol_link else symbol_cell.get_text(strip=True)
271
- symbol_url = symbol_link.get('href') if symbol_link else ''
272
-
273
- name_cell = cells[3]
274
- name_link = name_cell.find('a')
275
- name = name_link.get_text(strip=True) if name_link else name_cell.get_text(strip=True)
276
- name_url = name_link.get('href') if name_link else ''
277
-
278
- title = cells[4].get_text(strip=True)
279
-
280
- # Extract action links (view and PDF)
281
- action_cell = cells[5]
282
-
283
- # Find view link
284
- view_link = ''
285
- view_anchor = action_cell.find('a', href=lambda x: x and 'javascript:' in x)
286
- if view_anchor:
287
- view_images = view_anchor.get('data-images', '')
288
- if view_images:
289
- view_link = f"View images: {view_images}"
290
-
291
- # Find PDF link
292
- pdf_link = ''
293
- pdf_anchor = action_cell.find('a', href=lambda x: x and '.pdf' in x)
294
- if pdf_anchor:
295
- pdf_link = pdf_anchor.get('href', '')
296
- # Make absolute URL if relative
297
- if pdf_link.startswith('/'):
298
- pdf_link = f"https://your-base-url.com{pdf_link}"
299
-
300
- # Extract document ID from PDF link
301
- document_id = ''
302
- if pdf_link:
303
- # Extract numeric ID from URL
304
- import re
305
- match = re.search(r'/(\d+)\.pdf$', pdf_link)
306
- if match:
307
- document_id = match.group(1)
308
-
309
- # Combine date and time
310
  try:
311
- # Parse date and time
312
- full_date_str = f"{date_str} {time_str}"
313
- # You might need to adjust the format based on actual date format
314
- datetime_obj = datetime.strptime(full_date_str, '%b %d, %Y %I:%M %p')
315
- formatted_date = datetime_obj.strftime('%Y-%m-%d %H:%M:%S')
316
- except ValueError:
317
- formatted_date = f"{date_str} {time_str}"
318
-
319
- announcement = {
320
- 'Date': formatted_date,
321
- 'Raw_Date': date_str,
322
- 'Raw_Time': time_str,
323
- 'Symbol': symbol,
324
- 'Symbol_URL': symbol_url,
325
- 'Company_Name': name,
326
- 'Company_URL': name_url,
327
- 'Title': title,
328
- 'View_Link': view_link,
329
- 'PDF_Link': pdf_link,
330
- 'Document_ID': document_id,
331
- 'Scraped_At': datetime.now().strftime('%Y-%m-%d %H:%M:%S')
332
- }
333
-
334
- announcements.append(announcement)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
335
 
336
  return announcements
337
 
 
9
  from pydantic import BaseModel
10
  from models import PsxMarketResponse,PsxStock
11
  from threading import Thread
12
+ from datetime import datetime
13
+ import re
14
 
15
 
16
  CACHE = {
 
227
 
228
  # Parse the HTML
229
  soup = BeautifulSoup(r.text, 'html.parser')
230
+ print(r.text)
231
 
232
 
233
  # Find the announcements table
234
  table = soup.find('table', {'id': 'announcementsTable'})
235
 
236
  if not table:
237
+ print("ERROR: Table with id 'announcementsTable' not found!")
238
  return []
239
 
240
+ print(f"✓ Found announcements table")
 
 
 
 
 
 
 
241
 
242
+ # Extract announcements from table rows
243
  announcements = []
244
+ rows = table.find('tbody').find_all('tr')
245
 
246
+ print(f"Found {len(rows)} announcement rows")
 
 
 
 
247
 
248
+ for i, row in enumerate(rows, 1):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
249
  try:
250
+ cells = row.find_all('td')
251
+
252
+ if len(cells) < 6:
253
+ print(f" Row {i}: Only {len(cells)} cells, skipping...")
254
+ continue
255
+
256
+ # Extract date
257
+ date = cells[0].get_text(strip=True)
258
+
259
+ # Extract time
260
+ time = cells[1].get_text(strip=True)
261
+
262
+ # Extract symbol (from link if available)
263
+ symbol_cell = cells[2]
264
+ symbol_link = symbol_cell.find('a')
265
+ symbol = symbol_link.get_text(strip=True) if symbol_link else symbol_cell.get_text(strip=True)
266
+ symbol_url = symbol_link.get('href', '') if symbol_link else ''
267
+
268
+ # Extract company name (from link if available)
269
+ name_cell = cells[3]
270
+ name_link = name_cell.find('a')
271
+ name = name_link.get_text(strip=True) if name_link else name_cell.get_text(strip=True)
272
+ name_url = name_link.get('href', '') if name_link else ''
273
+
274
+ # Extract title
275
+ title = cells[4].get_text(strip=True)
276
+
277
+ # Extract links from the last cell
278
+ links_cell = cells[5]
279
+
280
+ # Find view link (data-images attribute)
281
+ view_link = ''
282
+ view_anchor = links_cell.find('a', href=lambda x: x and 'javascript:' in str(x))
283
+ if view_anchor:
284
+ view_images = view_anchor.get('data-images', '')
285
+ if view_images:
286
+ view_link = view_images
287
+
288
+ # Find PDF link
289
+ pdf_link = ''
290
+ pdf_anchor = links_cell.find('a', href=lambda x: x and '.pdf' in str(x))
291
+ if pdf_anchor:
292
+ pdf_link = pdf_anchor.get('href', '')
293
+
294
+ # Extract document ID from PDF link
295
+ document_id = ''
296
+ if pdf_link:
297
+ match = re.search(r'/(\d+)\.pdf$', pdf_link)
298
+ if match:
299
+ document_id = match.group(1)
300
+ elif view_link:
301
+ # Try to extract from view link if no PDF link
302
+ match = re.search(r'(\d+)-\d+\.', view_link)
303
+ if match:
304
+ document_id = match.group(1)
305
+
306
+ # Create announcement dictionary
307
+ announcement = {
308
+ 'Row_Number': i,
309
+ 'Date': date,
310
+ 'Time': time,
311
+ 'Symbol': symbol,
312
+ 'Symbol_URL': symbol_url,
313
+ 'Company_Name': name,
314
+ 'Company_URL': name_url,
315
+ 'Title': title,
316
+ 'View_Images': view_link,
317
+ 'PDF_Link': pdf_link,
318
+ 'Document_ID': document_id,
319
+ 'Scraped_At': datetime.now().strftime('%Y-%m-%d %H:%M:%S')
320
+ }
321
+
322
+ announcements.append(announcement)
323
+
324
+ # Print progress for first few rows
325
+ if i <= 3:
326
+ print(f" Processed row {i}: {symbol} - {title[:50]}...")
327
+
328
+ except Exception as e:
329
+ print(f" Error processing row {i}: {e}")
330
+ continue
331
 
332
  return announcements
333