thohan88 commited on
Commit
a863a04
·
1 Parent(s): 1301ea9

Update logic for resets

Browse files
Files changed (2) hide show
  1. tools/scraper.py +4 -2
  2. tools/worker.py +2 -2
tools/scraper.py CHANGED
@@ -86,10 +86,12 @@ def mod_scraper_server(input, output, session, www_dir):
86
  if host and "." in host:
87
  with reactive.isolate():
88
  current_filter = input.link_filter()
 
89
 
90
- # Simple logic: only update if currently empty
91
- if not current_filter:
92
  ui.update_text("link_filter", value=host, session=session)
 
93
  # except Exception as e:
94
  # print(f"DEBUG: sync_filter error: {e}", flush=True)
95
 
 
86
  if host and "." in host:
87
  with reactive.isolate():
88
  current_filter = input.link_filter()
89
+ last_auto = last_auto_filter.get()
90
 
91
+ # Update if currently empty OR if it matches what we last auto-set
92
+ if not current_filter or current_filter == last_auto:
93
  ui.update_text("link_filter", value=host, session=session)
94
+ last_auto_filter.set(host)
95
  # except Exception as e:
96
  # print(f"DEBUG: sync_filter error: {e}", flush=True)
97
 
tools/worker.py CHANGED
@@ -186,7 +186,7 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
186
  page_count = con.execute("SELECT COUNT(*) FROM pages").fetchone()[0]
187
  queue_count = con.execute("SELECT COUNT(*) FROM queue").fetchone()[0]
188
 
189
- if page_count == 0 and queue_count == 0:
190
  con.execute("INSERT INTO queue (url, depth) VALUES (?, ?)", (start_url, 0))
191
 
192
  print(f"Starting scraper for: {start_url}", flush=True)
@@ -235,7 +235,7 @@ def scrape(start_url, db_path, throttle_rate=2, max_reqs=100, max_depth=None, ke
235
  continue
236
 
237
  # Check if already scraped (double check)
238
- if con.execute("SELECT 1 FROM pages WHERE url = ?", (current_url,)).fetchone():
239
  continue
240
 
241
  resp = make_req(current_url, throttle_rate, cloudscraper_session=cloudscraper_session)
 
186
  page_count = con.execute("SELECT COUNT(*) FROM pages").fetchone()[0]
187
  queue_count = con.execute("SELECT COUNT(*) FROM queue").fetchone()[0]
188
 
189
+ if queue_count == 0:
190
  con.execute("INSERT INTO queue (url, depth) VALUES (?, ?)", (start_url, 0))
191
 
192
  print(f"Starting scraper for: {start_url}", flush=True)
 
235
  continue
236
 
237
  # Check if already scraped (double check)
238
+ if current_url != start_url and con.execute("SELECT 1 FROM pages WHERE url = ?", (current_url,)).fetchone():
239
  continue
240
 
241
  resp = make_req(current_url, throttle_rate, cloudscraper_session=cloudscraper_session)