Update logic for resets
Browse files- tools/scraper.py +4 -2
- tools/worker.py +2 -2
tools/scraper.py
CHANGED
|
@@ -86,10 +86,12 @@ def mod_scraper_server(input, output, session, www_dir):
|
|
| 86 |
if host and "." in host:
|
| 87 |
with reactive.isolate():
|
| 88 |
current_filter = input.link_filter()
|
|
|
|
| 89 |
|
| 90 |
-
#
|
| 91 |
-
if not current_filter:
|
| 92 |
ui.update_text("link_filter", value=host, session=session)
|
|
|
|
| 93 |
# except Exception as e:
|
| 94 |
# print(f"DEBUG: sync_filter error: {e}", flush=True)
|
| 95 |
|
|
|
|
| 86 |
if host and "." in host:
|
| 87 |
with reactive.isolate():
|
| 88 |
current_filter = input.link_filter()
|
| 89 |
+
last_auto = last_auto_filter.get()
|
| 90 |
|
| 91 |
+
# Update if currently empty OR if it matches what we last auto-set
|
| 92 |
+
if not current_filter or current_filter == last_auto:
|
| 93 |
ui.update_text("link_filter", value=host, session=session)
|
| 94 |
+
last_auto_filter.set(host)
|
| 95 |
# except Exception as e:
|
| 96 |
# print(f"DEBUG: sync_filter error: {e}", flush=True)
|
| 97 |
|
tools/worker.py
CHANGED
|
@@ -186,7 +186,7 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
|
|
| 186 |
page_count = con.execute("SELECT COUNT(*) FROM pages").fetchone()[0]
|
| 187 |
queue_count = con.execute("SELECT COUNT(*) FROM queue").fetchone()[0]
|
| 188 |
|
| 189 |
-
if
|
| 190 |
con.execute("INSERT INTO queue (url, depth) VALUES (?, ?)", (start_url, 0))
|
| 191 |
|
| 192 |
print(f"Starting scraper for: {start_url}", flush=True)
|
|
@@ -235,7 +235,7 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
|
|
| 235 |
continue
|
| 236 |
|
| 237 |
# Check if already scraped (double check)
|
| 238 |
-
if con.execute("SELECT 1 FROM pages WHERE url = ?", (current_url,)).fetchone():
|
| 239 |
continue
|
| 240 |
|
| 241 |
resp = make_req(current_url, throttle_rate, cloudscraper_session=cloudscraper_session)
|
|
|
|
| 186 |
page_count = con.execute("SELECT COUNT(*) FROM pages").fetchone()[0]
|
| 187 |
queue_count = con.execute("SELECT COUNT(*) FROM queue").fetchone()[0]
|
| 188 |
|
| 189 |
+
if queue_count == 0:
|
| 190 |
con.execute("INSERT INTO queue (url, depth) VALUES (?, ?)", (start_url, 0))
|
| 191 |
|
| 192 |
print(f"Starting scraper for: {start_url}", flush=True)
|
|
|
|
| 235 |
continue
|
| 236 |
|
| 237 |
# Check if already scraped (double check)
|
| 238 |
+
if current_url != start_url and con.execute("SELECT 1 FROM pages WHERE url = ?", (current_url,)).fetchone():
|
| 239 |
continue
|
| 240 |
|
| 241 |
resp = make_req(current_url, throttle_rate, cloudscraper_session=cloudscraper_session)
|