Spaces:
Paused
Paused
CBV01 commited on
Commit ·
1267ddc
1
Parent(s): a9d0ae5
fix: pyre strict queue IDE errors and missing json import
Browse files- backend/main.py +10 -9
backend/main.py
CHANGED
|
@@ -12,6 +12,7 @@ from datetime import datetime, timedelta, timezone
|
|
| 12 |
import collections
|
| 13 |
import asyncio
|
| 14 |
import random
|
|
|
|
| 15 |
from typing import Optional, List, Dict, Any
|
| 16 |
import re
|
| 17 |
import ast
|
|
@@ -436,7 +437,7 @@ async def _scrape_lyzem(query: str, max_pages: int = 10, queue: Optional[asyncio
|
|
| 436 |
}
|
| 437 |
results.append(item)
|
| 438 |
if queue:
|
| 439 |
-
await queue.put({"type": "result", "layer": 2, "data": item})
|
| 440 |
|
| 441 |
found_on_page += 1
|
| 442 |
|
|
@@ -514,7 +515,7 @@ async def _scrape_telegram_search(base_query: str, rows: list, queue: Optional[a
|
|
| 514 |
}
|
| 515 |
unique_groups[chat_id_str] = item
|
| 516 |
if queue:
|
| 517 |
-
await queue.put({"type": "result", "layer": 1, "data": item})
|
| 518 |
await asyncio.sleep(0.4)
|
| 519 |
except Exception as q_err:
|
| 520 |
print(f"SCRAPER[telegram] query='{q}' acc={acc_phone} err: {q_err}")
|
|
@@ -585,7 +586,7 @@ async def _scrape_spider(seed_groups: List[Dict[str, Any]], rows: list, max_seed
|
|
| 585 |
}
|
| 586 |
results.append(item)
|
| 587 |
if queue:
|
| 588 |
-
await queue.put({"type": "result", "layer": 3, "data": item})
|
| 589 |
found_in_seed = int(found_in_seed) + 1 # type: ignore
|
| 590 |
|
| 591 |
# Look for t.me links or @mentions in text
|
|
@@ -613,7 +614,7 @@ async def _scrape_spider(seed_groups: List[Dict[str, Any]], rows: list, max_seed
|
|
| 613 |
}
|
| 614 |
results.append(item2)
|
| 615 |
if queue:
|
| 616 |
-
await queue.put({"type": "result", "layer": 3, "data": item2})
|
| 617 |
found_in_seed = int(found_in_seed) + 1 # type: ignore
|
| 618 |
|
| 619 |
if int(found_in_seed) > 0: # type: ignore
|
|
@@ -656,7 +657,7 @@ async def scrape_keyword_stream(
|
|
| 656 |
|
| 657 |
async def scraper_task():
|
| 658 |
try:
|
| 659 |
-
await queue.put({"type": "progress", "msg": f"Layer 1 & 2: Scanning Telegram API & Lyzem Engine for '{base_query}'..."})
|
| 660 |
|
| 661 |
t_task = asyncio.create_task(_scrape_telegram_search(base_query, rows, queue))
|
| 662 |
l_task = asyncio.create_task(_scrape_lyzem(base_query, max_pages=10, queue=queue))
|
|
@@ -668,7 +669,7 @@ async def scrape_keyword_stream(
|
|
| 668 |
k = f"@{item.get('username', '')}"
|
| 669 |
if k not in ug and item.get('username'): ug[k] = item
|
| 670 |
|
| 671 |
-
await queue.put({"type": "progress", "msg": f"Layer 3: Spidering {len(ug)} prominent groups..."})
|
| 672 |
s_res = await _scrape_spider(list(ug.values()), rows, max_seeds=15, queue=queue)
|
| 673 |
|
| 674 |
# DB insert
|
|
@@ -680,12 +681,12 @@ async def scrape_keyword_stream(
|
|
| 680 |
if hasattr(conn2, "commit"): conn2.commit()
|
| 681 |
if hasattr(conn2, "close"): conn2.close()
|
| 682 |
|
| 683 |
-
await queue.put({"type": "done", "msg": "Scrape completed"})
|
| 684 |
except asyncio.CancelledError:
|
| 685 |
print("SCRAPER_STREAM canceled by client disconnect")
|
| 686 |
raise
|
| 687 |
except Exception as e:
|
| 688 |
-
await queue.put({"type": "error", "msg": str(e)})
|
| 689 |
|
| 690 |
# Start explicitly outside event_generator
|
| 691 |
bg_task = asyncio.create_task(scraper_task())
|
|
@@ -693,7 +694,7 @@ async def scrape_keyword_stream(
|
|
| 693 |
async def event_generator():
|
| 694 |
try:
|
| 695 |
while True:
|
| 696 |
-
msg = await queue.get()
|
| 697 |
yield f"data: {json.dumps(msg)}\n\n"
|
| 698 |
if msg["type"] in ["done", "error"]:
|
| 699 |
break
|
|
|
|
| 12 |
import collections
|
| 13 |
import asyncio
|
| 14 |
import random
|
| 15 |
+
import json
|
| 16 |
from typing import Optional, List, Dict, Any
|
| 17 |
import re
|
| 18 |
import ast
|
|
|
|
| 437 |
}
|
| 438 |
results.append(item)
|
| 439 |
if queue:
|
| 440 |
+
await queue.put({"type": "result", "layer": 2, "data": item}) # pyre-ignore
|
| 441 |
|
| 442 |
found_on_page += 1
|
| 443 |
|
|
|
|
| 515 |
}
|
| 516 |
unique_groups[chat_id_str] = item
|
| 517 |
if queue:
|
| 518 |
+
await queue.put({"type": "result", "layer": 1, "data": item}) # pyre-ignore
|
| 519 |
await asyncio.sleep(0.4)
|
| 520 |
except Exception as q_err:
|
| 521 |
print(f"SCRAPER[telegram] query='{q}' acc={acc_phone} err: {q_err}")
|
|
|
|
| 586 |
}
|
| 587 |
results.append(item)
|
| 588 |
if queue:
|
| 589 |
+
await queue.put({"type": "result", "layer": 3, "data": item}) # pyre-ignore
|
| 590 |
found_in_seed = int(found_in_seed) + 1 # type: ignore
|
| 591 |
|
| 592 |
# Look for t.me links or @mentions in text
|
|
|
|
| 614 |
}
|
| 615 |
results.append(item2)
|
| 616 |
if queue:
|
| 617 |
+
await queue.put({"type": "result", "layer": 3, "data": item2}) # pyre-ignore
|
| 618 |
found_in_seed = int(found_in_seed) + 1 # type: ignore
|
| 619 |
|
| 620 |
if int(found_in_seed) > 0: # type: ignore
|
|
|
|
| 657 |
|
| 658 |
async def scraper_task():
|
| 659 |
try:
|
| 660 |
+
await queue.put({"type": "progress", "msg": f"Layer 1 & 2: Scanning Telegram API & Lyzem Engine for '{base_query}'..."}) # pyre-ignore
|
| 661 |
|
| 662 |
t_task = asyncio.create_task(_scrape_telegram_search(base_query, rows, queue))
|
| 663 |
l_task = asyncio.create_task(_scrape_lyzem(base_query, max_pages=10, queue=queue))
|
|
|
|
| 669 |
k = f"@{item.get('username', '')}"
|
| 670 |
if k not in ug and item.get('username'): ug[k] = item
|
| 671 |
|
| 672 |
+
await queue.put({"type": "progress", "msg": f"Layer 3: Spidering {len(ug)} prominent groups..."}) # pyre-ignore
|
| 673 |
s_res = await _scrape_spider(list(ug.values()), rows, max_seeds=15, queue=queue)
|
| 674 |
|
| 675 |
# DB insert
|
|
|
|
| 681 |
if hasattr(conn2, "commit"): conn2.commit()
|
| 682 |
if hasattr(conn2, "close"): conn2.close()
|
| 683 |
|
| 684 |
+
await queue.put({"type": "done", "msg": "Scrape completed"}) # pyre-ignore
|
| 685 |
except asyncio.CancelledError:
|
| 686 |
print("SCRAPER_STREAM canceled by client disconnect")
|
| 687 |
raise
|
| 688 |
except Exception as e:
|
| 689 |
+
await queue.put({"type": "error", "msg": str(e)}) # pyre-ignore
|
| 690 |
|
| 691 |
# Start explicitly outside event_generator
|
| 692 |
bg_task = asyncio.create_task(scraper_task())
|
|
|
|
| 694 |
async def event_generator():
|
| 695 |
try:
|
| 696 |
while True:
|
| 697 |
+
msg = await queue.get() # pyre-ignore
|
| 698 |
yield f"data: {json.dumps(msg)}\n\n"
|
| 699 |
if msg["type"] in ["done", "error"]:
|
| 700 |
break
|