CBV01 commited on
Commit
1267ddc
·
1 Parent(s): a9d0ae5

fix: pyre strict queue IDE errors and missing json import

Browse files
Files changed (1) hide show
  1. backend/main.py +10 -9
backend/main.py CHANGED
@@ -12,6 +12,7 @@ from datetime import datetime, timedelta, timezone
12
  import collections
13
  import asyncio
14
  import random
 
15
  from typing import Optional, List, Dict, Any
16
  import re
17
  import ast
@@ -436,7 +437,7 @@ async def _scrape_lyzem(query: str, max_pages: int = 10, queue: Optional[asyncio
436
  }
437
  results.append(item)
438
  if queue:
439
- await queue.put({"type": "result", "layer": 2, "data": item})
440
 
441
  found_on_page += 1
442
 
@@ -514,7 +515,7 @@ async def _scrape_telegram_search(base_query: str, rows: list, queue: Optional[a
514
  }
515
  unique_groups[chat_id_str] = item
516
  if queue:
517
- await queue.put({"type": "result", "layer": 1, "data": item})
518
  await asyncio.sleep(0.4)
519
  except Exception as q_err:
520
  print(f"SCRAPER[telegram] query='{q}' acc={acc_phone} err: {q_err}")
@@ -585,7 +586,7 @@ async def _scrape_spider(seed_groups: List[Dict[str, Any]], rows: list, max_seed
585
  }
586
  results.append(item)
587
  if queue:
588
- await queue.put({"type": "result", "layer": 3, "data": item})
589
  found_in_seed = int(found_in_seed) + 1 # type: ignore
590
 
591
  # Look for t.me links or @mentions in text
@@ -613,7 +614,7 @@ async def _scrape_spider(seed_groups: List[Dict[str, Any]], rows: list, max_seed
613
  }
614
  results.append(item2)
615
  if queue:
616
- await queue.put({"type": "result", "layer": 3, "data": item2})
617
  found_in_seed = int(found_in_seed) + 1 # type: ignore
618
 
619
  if int(found_in_seed) > 0: # type: ignore
@@ -656,7 +657,7 @@ async def scrape_keyword_stream(
656
 
657
  async def scraper_task():
658
  try:
659
- await queue.put({"type": "progress", "msg": f"Layer 1 & 2: Scanning Telegram API & Lyzem Engine for '{base_query}'..."})
660
 
661
  t_task = asyncio.create_task(_scrape_telegram_search(base_query, rows, queue))
662
  l_task = asyncio.create_task(_scrape_lyzem(base_query, max_pages=10, queue=queue))
@@ -668,7 +669,7 @@ async def scrape_keyword_stream(
668
  k = f"@{item.get('username', '')}"
669
  if k not in ug and item.get('username'): ug[k] = item
670
 
671
- await queue.put({"type": "progress", "msg": f"Layer 3: Spidering {len(ug)} prominent groups..."})
672
  s_res = await _scrape_spider(list(ug.values()), rows, max_seeds=15, queue=queue)
673
 
674
  # DB insert
@@ -680,12 +681,12 @@ async def scrape_keyword_stream(
680
  if hasattr(conn2, "commit"): conn2.commit()
681
  if hasattr(conn2, "close"): conn2.close()
682
 
683
- await queue.put({"type": "done", "msg": "Scrape completed"})
684
  except asyncio.CancelledError:
685
  print("SCRAPER_STREAM canceled by client disconnect")
686
  raise
687
  except Exception as e:
688
- await queue.put({"type": "error", "msg": str(e)})
689
 
690
  # Start explicitly outside event_generator
691
  bg_task = asyncio.create_task(scraper_task())
@@ -693,7 +694,7 @@ async def scrape_keyword_stream(
693
  async def event_generator():
694
  try:
695
  while True:
696
- msg = await queue.get()
697
  yield f"data: {json.dumps(msg)}\n\n"
698
  if msg["type"] in ["done", "error"]:
699
  break
 
12
  import collections
13
  import asyncio
14
  import random
15
+ import json
16
  from typing import Optional, List, Dict, Any
17
  import re
18
  import ast
 
437
  }
438
  results.append(item)
439
  if queue:
440
+ await queue.put({"type": "result", "layer": 2, "data": item}) # pyre-ignore
441
 
442
  found_on_page += 1
443
 
 
515
  }
516
  unique_groups[chat_id_str] = item
517
  if queue:
518
+ await queue.put({"type": "result", "layer": 1, "data": item}) # pyre-ignore
519
  await asyncio.sleep(0.4)
520
  except Exception as q_err:
521
  print(f"SCRAPER[telegram] query='{q}' acc={acc_phone} err: {q_err}")
 
586
  }
587
  results.append(item)
588
  if queue:
589
+ await queue.put({"type": "result", "layer": 3, "data": item}) # pyre-ignore
590
  found_in_seed = int(found_in_seed) + 1 # type: ignore
591
 
592
  # Look for t.me links or @mentions in text
 
614
  }
615
  results.append(item2)
616
  if queue:
617
+ await queue.put({"type": "result", "layer": 3, "data": item2}) # pyre-ignore
618
  found_in_seed = int(found_in_seed) + 1 # type: ignore
619
 
620
  if int(found_in_seed) > 0: # type: ignore
 
657
 
658
  async def scraper_task():
659
  try:
660
+ await queue.put({"type": "progress", "msg": f"Layer 1 & 2: Scanning Telegram API & Lyzem Engine for '{base_query}'..."}) # pyre-ignore
661
 
662
  t_task = asyncio.create_task(_scrape_telegram_search(base_query, rows, queue))
663
  l_task = asyncio.create_task(_scrape_lyzem(base_query, max_pages=10, queue=queue))
 
669
  k = f"@{item.get('username', '')}"
670
  if k not in ug and item.get('username'): ug[k] = item
671
 
672
+ await queue.put({"type": "progress", "msg": f"Layer 3: Spidering {len(ug)} prominent groups..."}) # pyre-ignore
673
  s_res = await _scrape_spider(list(ug.values()), rows, max_seeds=15, queue=queue)
674
 
675
  # DB insert
 
681
  if hasattr(conn2, "commit"): conn2.commit()
682
  if hasattr(conn2, "close"): conn2.close()
683
 
684
+ await queue.put({"type": "done", "msg": "Scrape completed"}) # pyre-ignore
685
  except asyncio.CancelledError:
686
  print("SCRAPER_STREAM canceled by client disconnect")
687
  raise
688
  except Exception as e:
689
+ await queue.put({"type": "error", "msg": str(e)}) # pyre-ignore
690
 
691
  # Start explicitly outside event_generator
692
  bg_task = asyncio.create_task(scraper_task())
 
694
  async def event_generator():
695
  try:
696
  while True:
697
+ msg = await queue.get() # pyre-ignore
698
  yield f"data: {json.dumps(msg)}\n\n"
699
  if msg["type"] in ["done", "error"]:
700
  break