Upload scraper_template.py with huggingface_hub

scraper_template.py

@@ -1,4 +1,3 @@
-
 # PMC Sniper v13.0 — injected by HF Worker Space
 SESSION_ID       = __SESSION_ID__
 RESUME_FROM_PAGE = __RESUME_FROM_PAGE__
@@ -7,10 +6,11 @@ TOTAL_SESSIONS   = __TOTAL_SESSIONS__
 
 import subprocess, sys
 def _install():
-    subprocess.run(['apt-get','install','-y','-q','xvfb','xauth','x11-utils',
-        'libnss3','libatk1.0-0','libatk-bridge2.0-0','libcups2','libxcomposite1',
-        'libxdamage1','libxfixes3','libxrandr2','libgbm1','libpango-1.0-0',
-        'libcairo2','libasound2','libx11-xcb1','libxcb-dri3-0'],
+    subprocess.run(['apt-get','install','-y','-q','xvfb','xauth',
+        'x11-utils','libnss3','libatk1.0-0','libatk-bridge2.0-0',
+        'libcups2','libxcomposite1','libxdamage1','libxfixes3',
+        'libxrandr2','libgbm1','libpango-1.0-0','libcairo2',
+        'libasound2','libx11-xcb1','libxcb-dri3-0'],
         check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
     subprocess.run([sys.executable,'-m','pip','install','-q',
         'playwright','pyvirtualdisplay','beautifulsoup4'], check=True)
@@ -27,8 +27,7 @@ from typing import Optional, Set
 from collections import deque
 try:
     import nest_asyncio; nest_asyncio.apply()
-except ImportError:
-    pass
+except ImportError: pass
 from playwright.async_api import async_playwright
 from pyvirtualdisplay import Display
 from bs4 import BeautifulSoup
@@ -129,8 +128,10 @@ def write_state(last_page, reason, stats):
     }
     OUTPUT_DIR.mkdir(parents=True,exist_ok=True)
     (OUTPUT_DIR/'run_state.json').write_text(json.dumps(state,indent=2))
-    print(f'\n##RUN_STATE##{json.dumps(state)}##RUN_STATE##\n')
-    print(f'{"="*55}\n{RUN_ID} | page {last_page} | {remaining} remaining | {reason}\n{"="*55}')
+    import sys as _sys
+    _sys.stdout.write(f'\n##RUN_STATE##{json.dumps(state)}##RUN_STATE##\n')
+    _sys.stdout.flush()
+    print(f'DONE {RUN_ID} | page {last_page} | {remaining} left | {reason}')
     return state
 
 def _parse_cookies(raw):
@@ -155,15 +156,15 @@ def get_cookies():
             except: continue
     return []
 
-STEALTH='''
-Object.defineProperty(navigator,'webdriver',{get:()=>undefined});
-window.chrome={runtime:{},loadTimes:function(){},csi:function(){},app:{}};
-Object.defineProperty(navigator,'languages',{get:()=>['en-US','en']});
-Object.defineProperty(screen,'width',{get:()=>1920});
-Object.defineProperty(screen,'height',{get:()=>1080});
-Object.defineProperty(navigator,'hardwareConcurrency',{get:()=>4});
-Object.defineProperty(navigator,'platform',{get:()=>'Win32'});
-'''
+STEALTH = (
+    'Object.defineProperty(navigator,"webdriver",{get:()=>undefined});'
+    'window.chrome={runtime:{},loadTimes:function(){},csi:function(){},app:{}};'
+    'Object.defineProperty(navigator,"languages",{get:()=>["en-US","en"]});'
+    'Object.defineProperty(screen,"width",{get:()=>1920});'
+    'Object.defineProperty(screen,"height",{get:()=>1080});'
+    'Object.defineProperty(navigator,"hardwareConcurrency",{get:()=>4});'
+    'Object.defineProperty(navigator,"platform",{get:()=>"Win32"});'
+)
 
 async def block_ads(route):
     try:
@@ -201,7 +202,8 @@ async def dismiss_popup(page):
     try:
         p=await page.query_selector('div.popup_wrap')
         if not p or not await p.is_visible(): return
-        for sel in ['div.popup_wrap a.btn:not(.cancel)','div.popup_wrap button:not(.cancel)']:
+        for sel in ['div.popup_wrap a.btn:not(.cancel)',
+                    'div.popup_wrap button:not(.cancel)']:
             for el in await page.query_selector_all(sel):
                 try:
                     txt=(await el.inner_text()).strip().lower()
@@ -229,7 +231,7 @@ async def clear_cf(ctx, page):
                 (OUTPUT_DIR/'fresh_cookies.json').write_text(json.dumps(fresh,indent=2))
             except: pass
             return True
-        for sel in ["#challenge-stage input[type='checkbox']",'.cf-turnstile input']:
+        for sel in ["#challenge-stage input[type='checkbox']",' .cf-turnstile input']:
             try:
                 el=await page.query_selector(sel)
                 if el: await el.click(); await page.wait_for_timeout(1000)
@@ -294,7 +296,6 @@ DL_SELS=[
 async def download_build(page, entry, out, log, diag):
     safe=re.sub(r'[^\w\-]','_',entry.name)[:40]
     folder=out/f'{entry.id[:40]}_{safe}'
-    meta=None
     if not diag[0]:
         diag[0]=True
         try:
@@ -304,8 +305,7 @@ async def download_build(page, entry, out, log, diag):
             await page.screenshot(path=str(out/'diag.png'),full_page=True)
             already=True
         except: already=False
-    else:
-        already=False
+    else: already=False
     try:
         if not already:
             await page.goto(entry.project_url,wait_until='domcontentloaded',timeout=25000)
@@ -403,8 +403,7 @@ async def run_worker(wid, page, tracker, queue, out, stop_ev, col_done, stats, d
         queue.task_done()
         if n>0 and n%50==0:
             p=random.randint(30,60); log(f'Think {p}s'); await asyncio.sleep(p)
-        else:
-            await asyncio.sleep(random.uniform(5.0,12.0))
+        else: await asyncio.sleep(random.uniform(5.0,12.0))
     log(f'Done ok={stats["success"]}'); await page.close()
 
 def parse_links(html):
@@ -485,15 +484,13 @@ async def main():
            'last_page':START_PAGE,'stop_reason':'RUNNING','t0':time.time()}
     if tracker.total>0: print(f'Resuming — {tracker.total} in log.csv')
     rate=RateTracker(max_hr=MAX_RATE_PER_HR); diag=[False]
-    print(f'{"="*55}\nPMC v13.0 | {RUN_ID} | s{SESSION_ID}/{TOTAL_SESSIONS-1}\n'
-          f'Slice {SLICE_START}–{SLICE_END} | Start page {START_PAGE}\n{"="*55}')
+    print(f'PMC v13.0 | {RUN_ID} | s{SESSION_ID}/{TOTAL_SESSIONS-1}')
     async with async_playwright() as pw:
         br,ctx=await setup_browser(pw)
         cfp=await ctx.new_page()
         if not await clear_cf(ctx,cfp):
             write_state(START_PAGE-1,'CF_FAILED',stats); await br.close(); return
         await cfp.close()
-        print(f'Starting {DL_WORKERS} workers...')
         wps=[]
         for i in range(DL_WORKERS):
             wp=await new_tab(ctx)
@@ -531,4 +528,4 @@ def run():
     try: asyncio.get_event_loop().run_until_complete(main())
     finally: d.stop()
 
-run()
+run()