q6 commited on
Commit
2ceb744
·
1 Parent(s): e460c43

Add deep scan page browsing

Browse files
Files changed (4) hide show
  1. backend/app.py +252 -17
  2. frontend/app.js +46 -10
  3. frontend/index.html +8 -2
  4. frontend/style.css +44 -1
backend/app.py CHANGED
@@ -206,6 +206,11 @@ async def init_db():
206
  {
207
  "sql": "CREATE TABLE IF NOT EXISTS pi_scans (post_id TEXT PRIMARY KEY, url TEXT, exif_type INTEGER)"
208
  },
 
 
 
 
 
209
  {"sql": "CREATE TABLE IF NOT EXISTS pi_meta (key TEXT PRIMARY KEY, value TEXT)"},
210
  {
211
  "sql": "CREATE TABLE IF NOT EXISTS pi_cache (key TEXT PRIMARY KEY, value TEXT, updated_at INTEGER)"
@@ -227,6 +232,8 @@ async def init_db():
227
  "ALTER TABLE pi_searches DROP COLUMN created_at",
228
  "ALTER TABLE pi_searches ADD COLUMN found_exif INTEGER DEFAULT 0",
229
  "ALTER TABLE pi_searches ADD COLUMN total_searched INTEGER DEFAULT 0",
 
 
230
  ):
231
  try:
232
  await turso_execute([{"sql": sql}])
@@ -355,16 +362,17 @@ def get_search_api_url(raw, keywords):
355
  return f"https://www.pixiv.net/ajax/search/artworks/{encoded}?{params}"
356
 
357
 
358
- async def save_search(search_id, post_ids):
359
  if not post_ids:
360
  return
361
  if task_canceled(search_id):
362
  raise TaskCanceled()
363
  stmt = {
364
- "sql": "INSERT OR REPLACE INTO pi_searches (id, post_ids) VALUES (?, ?)",
365
  "args": [
366
  {"type": "text", "value": search_id},
367
  {"type": "text", "value": json.dumps(post_ids)},
 
368
  ],
369
  }
370
  await turso_execute([stmt])
@@ -680,6 +688,32 @@ async def get_unscanned_post_ids(search_id):
680
  return [cell_value(row, 0, "") for row in first_rows(resp) if cell_value(row, 0, "")]
681
 
682
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
683
  async def get_incomplete_search_ids():
684
  resp = await turso_execute(
685
  [
@@ -689,13 +723,30 @@ async def get_incomplete_search_ids():
689
  "FROM pi_searches p JOIN json_each(p.post_ids) j "
690
  "LEFT JOIN pi_scans s ON s.post_id = CAST(j.value AS TEXT) "
691
  "GROUP BY p.id"
692
- ") SELECT id FROM counts WHERE total > scanned ORDER BY id",
 
 
 
 
693
  }
694
  ]
695
  )
696
  return [cell_value(row, 0, "") for row in first_rows(resp) if cell_value(row, 0, "")]
697
 
698
 
 
 
 
 
 
 
 
 
 
 
 
 
 
699
  async def get_refresh_search_ids():
700
  resp = await turso_execute(
701
  [
@@ -800,6 +851,28 @@ def decrypt_log(value):
800
  return json.loads(data.decode())
801
 
802
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
803
  async def get_result_page(search_id, page, allowed):
804
  where_sql, filter_args = result_filter_sql(allowed)
805
  page = max(page, 1)
@@ -858,11 +931,13 @@ async def get_result_page(search_id, page, allowed):
858
  int(cell_value(row, 0, "0")): int(cell_value(row, 1, "0"))
859
  for row in type_rows
860
  }
 
861
  items = []
862
  for row in item_rows:
863
  pid = cell_value(row, 0, "")
864
  url = cell_value(row, 2, "")
865
  exif_type = cell_value(row, 3)
 
866
  items.append(
867
  {
868
  "post_id": pid,
@@ -870,6 +945,8 @@ async def get_result_page(search_id, page, allowed):
870
  "url": url,
871
  "exif_type": int(exif_type) if exif_type else None,
872
  "scanned": True,
 
 
873
  **image_links(pid, url),
874
  }
875
  )
@@ -889,7 +966,7 @@ def user_search_id(user_id):
889
  return f"{base26_time()}_{user_id}"
890
 
891
 
892
- async def pixiv_search_live(url, pages, mode, phpsessid, search_id):
893
  keywords = get_search_keywords(url)
894
  api_url = get_search_api_url(url, keywords)
895
  first_url = f"{api_url}&p=1"
@@ -936,7 +1013,7 @@ async def pixiv_search_live(url, pages, mode, phpsessid, search_id):
936
  if done % 25 == 0:
937
  if task_canceled(search_id):
938
  raise TaskCanceled()
939
- await save_search(search_id, post_ids)
940
  finally:
941
  for task in tasks:
942
  if not task.done():
@@ -944,7 +1021,7 @@ async def pixiv_search_live(url, pages, mode, phpsessid, search_id):
944
  await asyncio.gather(*tasks, return_exceptions=True)
945
  if task_canceled(search_id):
946
  raise TaskCanceled()
947
- await save_search(search_id, post_ids)
948
  return post_ids, keywords, first_url
949
 
950
 
@@ -1412,6 +1489,52 @@ async def scan_post(post_id, session, post_sem, exif_sem, img_sem):
1412
  return post_id, None, None
1413
 
1414
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1415
  async def get_exif_range(url, session, sem):
1416
  hdrs = {
1417
  "Referer": "https://www.pixiv.net/",
@@ -1506,10 +1629,62 @@ async def run_scan(post_ids, phpsessid, task_id=None, save_live=False, done_offs
1506
  return results
1507
 
1508
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1509
  async def save_scan_results(results):
1510
  stmts = []
1511
  for post_id, url, exif_type in results:
1512
  short_url = url.replace(IMG_BASE, "", 1) if url else ""
 
 
 
 
 
 
1513
  stmts.append(
1514
  {
1515
  "sql": "INSERT OR REPLACE INTO pi_scans (post_id, url, exif_type) VALUES (?, ?, ?)",
@@ -1527,6 +1702,32 @@ async def save_scan_results(results):
1527
  await turso_batch(stmts[i : i + 200])
1528
 
1529
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1530
  async def get_reclass_targets(allowed):
1531
  where_sql, args = scan_filter_sql(allowed)
1532
  resp = await turso_execute(
@@ -1778,7 +1979,17 @@ async def bg_search_task(search_id, url, pages, mode, phpsessid):
1778
  await finish_task(search_id)
1779
 
1780
 
1781
- async def bg_user_task(search_id, user_id, phpsessid):
 
 
 
 
 
 
 
 
 
 
1782
  if not start_task(search_id, {
1783
  "type": "user_search",
1784
  "phase": "searching",
@@ -1797,7 +2008,7 @@ async def bg_user_task(search_id, user_id, phpsessid):
1797
  ACTIVE_TASKS[search_id]["done"] = 1
1798
  if not post_ids:
1799
  return
1800
- await save_search(search_id, post_ids)
1801
  to_scan = await get_unscanned_post_ids(search_id)
1802
  if to_scan:
1803
  ACTIVE_TASKS[search_id].update({
@@ -1805,6 +2016,8 @@ async def bg_user_task(search_id, user_id, phpsessid):
1805
  **await scan_progress_counts(search_id, len(to_scan)),
1806
  })
1807
  await run_scan(to_scan, phpsessid, task_id=search_id, save_live=True)
 
 
1808
  await update_search_stats(search_id)
1809
  await patch_explorer_cache(search_id, front=True)
1810
  except TaskCanceled:
@@ -1815,13 +2028,16 @@ async def bg_user_task(search_id, user_id, phpsessid):
1815
  await finish_task(search_id)
1816
 
1817
 
1818
- async def bg_user_batch_task(jobs, phpsessid):
1819
  await asyncio.gather(
1820
- *(bg_user_task(search_id, user_id, phpsessid) for search_id, user_id in jobs)
 
 
 
1821
  )
1822
 
1823
 
1824
- async def bg_scan_task(search_id, post_ids, phpsessid):
1825
  if not start_task(search_id, {
1826
  "type": "scan",
1827
  "phase": "scanning",
@@ -1832,6 +2048,8 @@ async def bg_scan_task(search_id, post_ids, phpsessid):
1832
  completed = False
1833
  try:
1834
  await run_scan(post_ids, phpsessid, task_id=search_id, save_live=True)
 
 
1835
  await update_search_stats(search_id)
1836
  await patch_explorer_cache(search_id)
1837
  completed = True
@@ -1846,7 +2064,9 @@ async def bg_scan_task(search_id, post_ids, phpsessid):
1846
  await finish_task(search_id)
1847
 
1848
 
1849
- async def bg_search_and_scan_task(search_id, url, pages, mode, phpsessid):
 
 
1850
  if not start_task(search_id, {
1851
  "type": "search+scan",
1852
  "phase": "searching",
@@ -1857,7 +2077,9 @@ async def bg_search_and_scan_task(search_id, url, pages, mode, phpsessid):
1857
  return
1858
  completed = False
1859
  try:
1860
- await pixiv_search_live(url, pages, mode, phpsessid, search_id)
 
 
1861
  to_scan = await get_unscanned_post_ids(search_id)
1862
  if to_scan:
1863
  ACTIVE_TASKS[search_id].update({
@@ -1865,6 +2087,8 @@ async def bg_search_and_scan_task(search_id, url, pages, mode, phpsessid):
1865
  **await scan_progress_counts(search_id, len(to_scan)),
1866
  })
1867
  await run_scan(to_scan, phpsessid, task_id=search_id, save_live=True)
 
 
1868
  await update_search_stats(search_id)
1869
  await patch_explorer_cache(search_id, front=True)
1870
  completed = True
@@ -1956,11 +2180,13 @@ class SearchRequest(BaseModel):
1956
  pages: int = 30
1957
  mode: str = "ai"
1958
  action: str = "search"
 
1959
 
1960
 
1961
  class UserSearchRequest(BaseModel):
1962
  user_ids: list
1963
  action: str = "search"
 
1964
 
1965
 
1966
  class PostSearchRequest(BaseModel):
@@ -2031,8 +2257,11 @@ async def resume_incomplete_scans():
2031
  if search_id in ACTIVE_TASKS:
2032
  continue
2033
  post_ids = await get_unscanned_post_ids(search_id)
2034
- if post_ids:
2035
- asyncio.create_task(bg_scan_task(search_id, post_ids, PHPSESSID))
 
 
 
2036
  started += 1
2037
  else:
2038
  await update_search_stats(search_id)
@@ -2127,7 +2356,13 @@ async def submit_search(req: SearchRequest, bg: BackgroundTasks):
2127
  search_id = base26_time()
2128
  phpsessid = PHPSESSID
2129
  bg.add_task(
2130
- bg_search_and_scan_task, search_id, req.url, req.pages, req.mode, phpsessid
 
 
 
 
 
 
2131
  )
2132
  return {"id": search_id, "status": "started"}
2133
 
@@ -2137,7 +2372,7 @@ async def submit_users(req: UserSearchRequest, bg: BackgroundTasks):
2137
  phpsessid = PHPSESSID
2138
  user_ids = list(dict.fromkeys(int(u) for u in req.user_ids))
2139
  jobs = [(user_search_id(uid), uid) for uid in user_ids]
2140
- bg.add_task(bg_user_batch_task, jobs, phpsessid)
2141
  return {"ids": [search_id for search_id, _ in jobs], "status": "started"}
2142
 
2143
 
 
206
  {
207
  "sql": "CREATE TABLE IF NOT EXISTS pi_scans (post_id TEXT PRIMARY KEY, url TEXT, exif_type INTEGER)"
208
  },
209
+ {
210
+ "sql": "CREATE TABLE IF NOT EXISTS pi_scan_pages ("
211
+ "post_id TEXT, page INTEGER, url TEXT, exif_type INTEGER, "
212
+ "PRIMARY KEY (post_id, page))"
213
+ },
214
  {"sql": "CREATE TABLE IF NOT EXISTS pi_meta (key TEXT PRIMARY KEY, value TEXT)"},
215
  {
216
  "sql": "CREATE TABLE IF NOT EXISTS pi_cache (key TEXT PRIMARY KEY, value TEXT, updated_at INTEGER)"
 
232
  "ALTER TABLE pi_searches DROP COLUMN created_at",
233
  "ALTER TABLE pi_searches ADD COLUMN found_exif INTEGER DEFAULT 0",
234
  "ALTER TABLE pi_searches ADD COLUMN total_searched INTEGER DEFAULT 0",
235
+ "ALTER TABLE pi_searches ADD COLUMN deep_scan INTEGER DEFAULT 0",
236
+ "ALTER TABLE pi_scans ADD COLUMN deep_scanned INTEGER DEFAULT 0",
237
  ):
238
  try:
239
  await turso_execute([{"sql": sql}])
 
362
  return f"https://www.pixiv.net/ajax/search/artworks/{encoded}?{params}"
363
 
364
 
365
+ async def save_search(search_id, post_ids, deep_scan=False):
366
  if not post_ids:
367
  return
368
  if task_canceled(search_id):
369
  raise TaskCanceled()
370
  stmt = {
371
+ "sql": "INSERT OR REPLACE INTO pi_searches (id, post_ids, deep_scan) VALUES (?, ?, ?)",
372
  "args": [
373
  {"type": "text", "value": search_id},
374
  {"type": "text", "value": json.dumps(post_ids)},
375
+ {"type": "integer", "value": "1" if deep_scan else "0"},
376
  ],
377
  }
378
  await turso_execute([stmt])
 
688
  return [cell_value(row, 0, "") for row in first_rows(resp) if cell_value(row, 0, "")]
689
 
690
 
691
+ async def get_deep_scan_targets(search_id):
692
+ resp = await turso_execute(
693
+ [
694
+ {
695
+ "sql": "WITH ids AS ("
696
+ "SELECT CAST(j.value AS TEXT) post_id, CAST(j.key AS INTEGER) pos "
697
+ "FROM pi_searches p, json_each(p.post_ids) j WHERE p.id = ?"
698
+ ") SELECT ids.post_id, s.url, s.exif_type FROM ids "
699
+ "JOIN pi_scans s ON s.post_id = ids.post_id "
700
+ "WHERE s.exif_type IS NOT NULL AND COALESCE(s.deep_scanned, 0) = 0 "
701
+ "ORDER BY ids.pos",
702
+ "args": [{"type": "text", "value": search_id}],
703
+ }
704
+ ]
705
+ )
706
+ return [
707
+ {
708
+ "post_id": cell_value(row, 0, ""),
709
+ "url": cell_value(row, 1, ""),
710
+ "exif_type": int(cell_value(row, 2, "0")),
711
+ }
712
+ for row in first_rows(resp)
713
+ if cell_value(row, 0, "")
714
+ ]
715
+
716
+
717
  async def get_incomplete_search_ids():
718
  resp = await turso_execute(
719
  [
 
723
  "FROM pi_searches p JOIN json_each(p.post_ids) j "
724
  "LEFT JOIN pi_scans s ON s.post_id = CAST(j.value AS TEXT) "
725
  "GROUP BY p.id"
726
+ ") SELECT p.id FROM pi_searches p JOIN counts ON counts.id = p.id "
727
+ "WHERE total > scanned OR (COALESCE(p.deep_scan, 0) = 1 AND EXISTS ("
728
+ "SELECT 1 FROM json_each(p.post_ids) j JOIN pi_scans s "
729
+ "ON s.post_id = CAST(j.value AS TEXT) WHERE s.exif_type IS NOT NULL "
730
+ "AND COALESCE(s.deep_scanned, 0) = 0)) ORDER BY p.id",
731
  }
732
  ]
733
  )
734
  return [cell_value(row, 0, "") for row in first_rows(resp) if cell_value(row, 0, "")]
735
 
736
 
737
+ async def search_uses_deep_scan(search_id):
738
+ resp = await turso_execute(
739
+ [
740
+ {
741
+ "sql": "SELECT deep_scan FROM pi_searches WHERE id = ?",
742
+ "args": [{"type": "text", "value": search_id}],
743
+ }
744
+ ]
745
+ )
746
+ rows = first_rows(resp)
747
+ return bool(rows and int(cell_value(rows[0], 0, "0")))
748
+
749
+
750
  async def get_refresh_search_ids():
751
  resp = await turso_execute(
752
  [
 
851
  return json.loads(data.decode())
852
 
853
 
854
+ async def get_result_pages(post_ids):
855
+ if not post_ids:
856
+ return {}
857
+ pages = {}
858
+ for i in range(0, len(post_ids), 500):
859
+ chunk = post_ids[i : i + 500]
860
+ marks = ",".join("?" for _ in chunk)
861
+ resp = await turso_execute(
862
+ [
863
+ {
864
+ "sql": f"SELECT post_id, page FROM pi_scan_pages WHERE post_id IN ({marks}) ORDER BY post_id, page",
865
+ "args": [{"type": "text", "value": str(pid)} for pid in chunk],
866
+ }
867
+ ]
868
+ )
869
+ for row in first_rows(resp):
870
+ pages.setdefault(cell_value(row, 0, ""), []).append(
871
+ int(cell_value(row, 1, "0"))
872
+ )
873
+ return pages
874
+
875
+
876
  async def get_result_page(search_id, page, allowed):
877
  where_sql, filter_args = result_filter_sql(allowed)
878
  page = max(page, 1)
 
931
  int(cell_value(row, 0, "0")): int(cell_value(row, 1, "0"))
932
  for row in type_rows
933
  }
934
+ result_pages = await get_result_pages([cell_value(row, 0, "") for row in item_rows])
935
  items = []
936
  for row in item_rows:
937
  pid = cell_value(row, 0, "")
938
  url = cell_value(row, 2, "")
939
  exif_type = cell_value(row, 3)
940
+ pages = result_pages.get(pid) or [page_num_from_url(url)]
941
  items.append(
942
  {
943
  "post_id": pid,
 
945
  "url": url,
946
  "exif_type": int(exif_type) if exif_type else None,
947
  "scanned": True,
948
+ "pages": pages,
949
+ "page_count": len(pages),
950
  **image_links(pid, url),
951
  }
952
  )
 
966
  return f"{base26_time()}_{user_id}"
967
 
968
 
969
+ async def pixiv_search_live(url, pages, mode, phpsessid, search_id, deep_scan=False):
970
  keywords = get_search_keywords(url)
971
  api_url = get_search_api_url(url, keywords)
972
  first_url = f"{api_url}&p=1"
 
1013
  if done % 25 == 0:
1014
  if task_canceled(search_id):
1015
  raise TaskCanceled()
1016
+ await save_search(search_id, post_ids, deep_scan)
1017
  finally:
1018
  for task in tasks:
1019
  if not task.done():
 
1021
  await asyncio.gather(*tasks, return_exceptions=True)
1022
  if task_canceled(search_id):
1023
  raise TaskCanceled()
1024
+ await save_search(search_id, post_ids, deep_scan)
1025
  return post_ids, keywords, first_url
1026
 
1027
 
 
1489
  return post_id, None, None
1490
 
1491
 
1492
+ async def scan_page_result(post_id, url, session, exif_sem, img_sem):
1493
+ metadata = await get_exif_range(url, session, exif_sem)
1494
+ exif_type = determine_exif_type(metadata)
1495
+ fallback = EXIF_TYPE_TO_CODE.get(exif_type) if exif_type in FALLBACK_EXIF_TYPES else None
1496
+ if exif_type and not fallback:
1497
+ return post_id, url, EXIF_TYPE_TO_CODE.get(exif_type)
1498
+ img_data = await fetch_image(session, url, img_sem)
1499
+ if not img_data:
1500
+ return (post_id, url, fallback) if fallback else None
1501
+ exif_type = determine_exif_type(parse_image_metadata(img_data))
1502
+ if exif_type in FALLBACK_EXIF_TYPES:
1503
+ fallback = fallback or EXIF_TYPE_TO_CODE.get(exif_type)
1504
+ elif exif_type:
1505
+ return post_id, url, EXIF_TYPE_TO_CODE.get(exif_type)
1506
+ metadata, detected = parse_stealth_metadata(img_data)
1507
+ exif_type = determine_exif_type(metadata)
1508
+ if exif_type in FALLBACK_EXIF_TYPES:
1509
+ fallback = fallback or EXIF_TYPE_TO_CODE.get(exif_type)
1510
+ elif exif_type:
1511
+ return post_id, url, EXIF_TYPE_TO_CODE.get(exif_type)
1512
+ if detected:
1513
+ return post_id, url, EXIF_TYPE_TO_CODE.get("stealth")
1514
+ return (post_id, url, fallback) if fallback else None
1515
+
1516
+
1517
+ async def deep_scan_post(item, session, post_sem, exif_sem, img_sem):
1518
+ async with post_sem:
1519
+ try:
1520
+ post_id = item["post_id"]
1521
+ pages = await get_post_pages(post_id, session)
1522
+ first_page = page_num_from_url(item["url"])
1523
+ results = [(post_id, first_page, item["url"], item["exif_type"])]
1524
+ for page in range(len(pages)):
1525
+ if page == first_page:
1526
+ continue
1527
+ url = pages[page]["original"]
1528
+ if image_ext(url) not in METADATA_EXTS:
1529
+ continue
1530
+ result = await scan_page_result(post_id, url, session, exif_sem, img_sem)
1531
+ if result:
1532
+ results.append((post_id, page, result[1], result[2]))
1533
+ return post_id, results
1534
+ except Exception:
1535
+ return None
1536
+
1537
+
1538
  async def get_exif_range(url, session, sem):
1539
  hdrs = {
1540
  "Referer": "https://www.pixiv.net/",
 
1629
  return results
1630
 
1631
 
1632
+ async def run_deep_scan(targets, phpsessid, task_id=None):
1633
+ post_sem = asyncio.Semaphore(POST_SCAN_LIMIT)
1634
+ exif_sem = asyncio.Semaphore(EXIF_RANGE_LIMIT)
1635
+ img_sem = asyncio.Semaphore(FULL_IMAGE_LIMIT)
1636
+ cookies = {"PHPSESSID": phpsessid}
1637
+ completed = 0
1638
+ pending = []
1639
+ async with aiohttp.ClientSession(cookies=cookies, headers=PIXIV_HEADERS) as session:
1640
+ tasks = {
1641
+ asyncio.create_task(deep_scan_post(item, session, post_sem, exif_sem, img_sem))
1642
+ for item in targets
1643
+ }
1644
+ created = set(tasks)
1645
+ if task_id:
1646
+ TASK_SCAN_TASKS.setdefault(task_id, set()).update(tasks)
1647
+ try:
1648
+ while tasks:
1649
+ if task_canceled(task_id):
1650
+ raise TaskCanceled()
1651
+ done, tasks = await asyncio.wait(
1652
+ tasks, timeout=1, return_when=asyncio.FIRST_COMPLETED
1653
+ )
1654
+ if task_canceled(task_id):
1655
+ raise TaskCanceled()
1656
+ for task in done:
1657
+ result = task.result()
1658
+ completed += 1
1659
+ if result:
1660
+ pending.append(result)
1661
+ if task_id and task_id in ACTIVE_TASKS:
1662
+ ACTIVE_TASKS[task_id]["done"] = completed
1663
+ if len(pending) >= 20:
1664
+ await save_deep_scan_results(pending)
1665
+ pending = []
1666
+ if pending:
1667
+ await save_deep_scan_results(pending)
1668
+ finally:
1669
+ for task in created:
1670
+ task.cancel()
1671
+ await asyncio.gather(*created, return_exceptions=True)
1672
+ if task_id and task_id in TASK_SCAN_TASKS:
1673
+ TASK_SCAN_TASKS[task_id].difference_update(created)
1674
+ if not TASK_SCAN_TASKS[task_id]:
1675
+ TASK_SCAN_TASKS.pop(task_id, None)
1676
+
1677
+
1678
  async def save_scan_results(results):
1679
  stmts = []
1680
  for post_id, url, exif_type in results:
1681
  short_url = url.replace(IMG_BASE, "", 1) if url else ""
1682
+ stmts.append(
1683
+ {
1684
+ "sql": "DELETE FROM pi_scan_pages WHERE post_id = ?",
1685
+ "args": [{"type": "text", "value": str(post_id)}],
1686
+ }
1687
+ )
1688
  stmts.append(
1689
  {
1690
  "sql": "INSERT OR REPLACE INTO pi_scans (post_id, url, exif_type) VALUES (?, ?, ?)",
 
1702
  await turso_batch(stmts[i : i + 200])
1703
 
1704
 
1705
+ async def save_deep_scan_results(results):
1706
+ stmts = []
1707
+ for post_id, pages in results:
1708
+ for _, page, url, exif_type in pages:
1709
+ short_url = url.replace(IMG_BASE, "", 1) if url else ""
1710
+ stmts.append(
1711
+ {
1712
+ "sql": "INSERT OR REPLACE INTO pi_scan_pages (post_id, page, url, exif_type) VALUES (?, ?, ?, ?)",
1713
+ "args": [
1714
+ {"type": "text", "value": str(post_id)},
1715
+ {"type": "integer", "value": str(page)},
1716
+ {"type": "text", "value": short_url},
1717
+ {"type": "integer", "value": str(exif_type)},
1718
+ ],
1719
+ }
1720
+ )
1721
+ stmts.append(
1722
+ {
1723
+ "sql": "UPDATE pi_scans SET deep_scanned = 1 WHERE post_id = ?",
1724
+ "args": [{"type": "text", "value": str(post_id)}],
1725
+ }
1726
+ )
1727
+ for i in range(0, len(stmts), 200):
1728
+ await turso_batch(stmts[i : i + 200])
1729
+
1730
+
1731
  async def get_reclass_targets(allowed):
1732
  where_sql, args = scan_filter_sql(allowed)
1733
  resp = await turso_execute(
 
1979
  await finish_task(search_id)
1980
 
1981
 
1982
+ async def deep_scan_search(search_id, phpsessid):
1983
+ targets = await get_deep_scan_targets(search_id)
1984
+ if not targets:
1985
+ return
1986
+ ACTIVE_TASKS[search_id].update(
1987
+ {"phase": "deep scanning", "total": len(targets), "done": 0, "raw_total": 0}
1988
+ )
1989
+ await run_deep_scan(targets, phpsessid, task_id=search_id)
1990
+
1991
+
1992
+ async def bg_user_task(search_id, user_id, phpsessid, deep_scan=False):
1993
  if not start_task(search_id, {
1994
  "type": "user_search",
1995
  "phase": "searching",
 
2008
  ACTIVE_TASKS[search_id]["done"] = 1
2009
  if not post_ids:
2010
  return
2011
+ await save_search(search_id, post_ids, deep_scan)
2012
  to_scan = await get_unscanned_post_ids(search_id)
2013
  if to_scan:
2014
  ACTIVE_TASKS[search_id].update({
 
2016
  **await scan_progress_counts(search_id, len(to_scan)),
2017
  })
2018
  await run_scan(to_scan, phpsessid, task_id=search_id, save_live=True)
2019
+ if deep_scan:
2020
+ await deep_scan_search(search_id, phpsessid)
2021
  await update_search_stats(search_id)
2022
  await patch_explorer_cache(search_id, front=True)
2023
  except TaskCanceled:
 
2028
  await finish_task(search_id)
2029
 
2030
 
2031
+ async def bg_user_batch_task(jobs, phpsessid, deep_scan=False):
2032
  await asyncio.gather(
2033
+ *(
2034
+ bg_user_task(search_id, user_id, phpsessid, deep_scan)
2035
+ for search_id, user_id in jobs
2036
+ )
2037
  )
2038
 
2039
 
2040
+ async def bg_scan_task(search_id, post_ids, phpsessid, deep_scan=False):
2041
  if not start_task(search_id, {
2042
  "type": "scan",
2043
  "phase": "scanning",
 
2048
  completed = False
2049
  try:
2050
  await run_scan(post_ids, phpsessid, task_id=search_id, save_live=True)
2051
+ if deep_scan:
2052
+ await deep_scan_search(search_id, phpsessid)
2053
  await update_search_stats(search_id)
2054
  await patch_explorer_cache(search_id)
2055
  completed = True
 
2064
  await finish_task(search_id)
2065
 
2066
 
2067
+ async def bg_search_and_scan_task(
2068
+ search_id, url, pages, mode, phpsessid, deep_scan=False
2069
+ ):
2070
  if not start_task(search_id, {
2071
  "type": "search+scan",
2072
  "phase": "searching",
 
2077
  return
2078
  completed = False
2079
  try:
2080
+ await pixiv_search_live(
2081
+ url, pages, mode, phpsessid, search_id, deep_scan
2082
+ )
2083
  to_scan = await get_unscanned_post_ids(search_id)
2084
  if to_scan:
2085
  ACTIVE_TASKS[search_id].update({
 
2087
  **await scan_progress_counts(search_id, len(to_scan)),
2088
  })
2089
  await run_scan(to_scan, phpsessid, task_id=search_id, save_live=True)
2090
+ if deep_scan:
2091
+ await deep_scan_search(search_id, phpsessid)
2092
  await update_search_stats(search_id)
2093
  await patch_explorer_cache(search_id, front=True)
2094
  completed = True
 
2180
  pages: int = 30
2181
  mode: str = "ai"
2182
  action: str = "search"
2183
+ deep_scan: bool = False
2184
 
2185
 
2186
  class UserSearchRequest(BaseModel):
2187
  user_ids: list
2188
  action: str = "search"
2189
+ deep_scan: bool = False
2190
 
2191
 
2192
  class PostSearchRequest(BaseModel):
 
2257
  if search_id in ACTIVE_TASKS:
2258
  continue
2259
  post_ids = await get_unscanned_post_ids(search_id)
2260
+ deep_scan = await search_uses_deep_scan(search_id)
2261
+ if post_ids or deep_scan:
2262
+ asyncio.create_task(
2263
+ bg_scan_task(search_id, post_ids, PHPSESSID, deep_scan)
2264
+ )
2265
  started += 1
2266
  else:
2267
  await update_search_stats(search_id)
 
2356
  search_id = base26_time()
2357
  phpsessid = PHPSESSID
2358
  bg.add_task(
2359
+ bg_search_and_scan_task,
2360
+ search_id,
2361
+ req.url,
2362
+ req.pages,
2363
+ req.mode,
2364
+ phpsessid,
2365
+ req.deep_scan,
2366
  )
2367
  return {"id": search_id, "status": "started"}
2368
 
 
2372
  phpsessid = PHPSESSID
2373
  user_ids = list(dict.fromkeys(int(u) for u in req.user_ids))
2374
  jobs = [(user_search_id(uid), uid) for uid in user_ids]
2375
+ bg.add_task(bg_user_batch_task, jobs, phpsessid, req.deep_scan)
2376
  return {"ids": [search_id for search_id, _ in jobs], "status": "started"}
2377
 
2378
 
frontend/app.js CHANGED
@@ -11,6 +11,9 @@ const PAGE_SIZE = 60
11
  const SEARCH_PAGE_SIZE = 10
12
  let viewerScale = 1
13
  let viewerDrag = null
 
 
 
14
  let explorerEvents = null
15
  let explorerPage = 1
16
  let explorerDebounce = null
@@ -37,6 +40,7 @@ $("#btn-submit").addEventListener("click", async () => {
37
  if (!url) return
38
  const pages = parseInt($("#input-pages").value) || 30
39
  const mode = $("#input-mode").value
 
40
  const status = $("#submit-status")
41
  status.textContent = "Submitting..."
42
  status.className = ""
@@ -57,13 +61,13 @@ $("#btn-submit").addEventListener("click", async () => {
57
  resp = await fetch("/api/submit_users", {
58
  method: "POST",
59
  headers: { "Content-Type": "application/json" },
60
- body: JSON.stringify({ user_ids: userIds })
61
  })
62
  } else {
63
  resp = await fetch("/api/submit", {
64
  method: "POST",
65
  headers: { "Content-Type": "application/json" },
66
- body: JSON.stringify({ url, pages, mode })
67
  })
68
  }
69
  const data = await resp.json()
@@ -548,7 +552,7 @@ function pageSuffix(url) {
548
  }
549
 
550
  function resultCardKey(item) {
551
- return `${item.post_id}|${item.url || ""}`
552
  }
553
 
554
  function resultBadge(item) {
@@ -563,6 +567,7 @@ function resultCardHtml(item) {
563
  const pg = item.url ? pageSuffix(item.url) : ""
564
  const label = pid + pg
565
  const index = item.scan_index || 0
 
566
  const download = item.download_url ? `<a class="download-link" href="${esc(item.download_url)}">Download</a>` : ""
567
  const thumb = item.image_url
568
  ? `<button class="thumb" data-preview="${esc(item.preview_url || item.image_url)}" data-download="${esc(item.download_url || "")}"><img src="${esc(item.image_url)}" loading="lazy" alt="${esc(label)}"></button>`
@@ -574,6 +579,7 @@ function resultCardHtml(item) {
574
  ${download}
575
  </div>
576
  ${resultBadge(item)}
 
577
  <span class="result-index">${index}</span>
578
  </div>`
579
  }
@@ -588,7 +594,7 @@ function createResultCard(item) {
588
  const thumb = card.querySelector(".thumb")
589
  if (thumb) {
590
  const img = thumb.querySelector("img")
591
- thumb.addEventListener("click", () => openViewer(thumb.dataset.preview, thumb.dataset.download))
592
  img.addEventListener("load", () => thumb.classList.add("thumb-loaded"))
593
  img.addEventListener("error", () => thumb.classList.add("thumb-error"))
594
  if (img.complete) thumb.classList.add(img.naturalWidth ? "thumb-loaded" : "thumb-error")
@@ -622,21 +628,44 @@ function renderPostSearchResults(items) {
622
  $("#post-search-grid").replaceChildren(...cards)
623
  }
624
 
625
- function openViewer(src, download) {
626
- if (!src) return
 
 
 
627
  viewerScale = 1
628
- $("#viewer-img").src = src
629
- $("#viewer-download").href = download || src
630
  $("#viewer").classList.remove("hidden")
631
  $("#viewer").setAttribute("aria-hidden", "false")
 
 
 
 
 
 
 
 
 
 
 
 
 
632
  applyViewerZoom()
633
  }
634
 
 
 
 
 
 
 
 
635
  function closeViewer() {
636
  $("#viewer").classList.add("hidden")
637
  $("#viewer").setAttribute("aria-hidden", "true")
638
  $("#viewer-img").src = ""
639
  $("#viewer-download").href = ""
 
 
640
  viewerDrag = null
641
  $("#viewer-stage").classList.remove("dragging")
642
  }
@@ -657,6 +686,8 @@ $("#viewer-zoom-out").addEventListener("click", () => {
657
  })
658
 
659
  $("#viewer-close").addEventListener("click", closeViewer)
 
 
660
  $("#viewer").addEventListener("click", e => { if (e.target.id === "viewer") closeViewer() })
661
  $("#viewer-stage").addEventListener("wheel", e => {
662
  e.preventDefault()
@@ -664,7 +695,7 @@ $("#viewer-stage").addEventListener("wheel", e => {
664
  applyViewerZoom()
665
  })
666
  $("#viewer-stage").addEventListener("pointerdown", e => {
667
- if (e.button !== 0) return
668
  const stage = $("#viewer-stage")
669
  viewerDrag = {
670
  x: e.clientX,
@@ -690,7 +721,12 @@ $("#viewer-stage").addEventListener("pointercancel", () => {
690
  viewerDrag = null
691
  $("#viewer-stage").classList.remove("dragging")
692
  })
693
- window.addEventListener("keydown", e => { if (e.key === "Escape") closeViewer() })
 
 
 
 
 
694
 
695
  function renderPager(id, data) {
696
  const pager = $("#pager")
 
11
  const SEARCH_PAGE_SIZE = 10
12
  let viewerScale = 1
13
  let viewerDrag = null
14
+ let viewerPostId = ""
15
+ let viewerPages = []
16
+ let viewerPageIndex = 0
17
  let explorerEvents = null
18
  let explorerPage = 1
19
  let explorerDebounce = null
 
40
  if (!url) return
41
  const pages = parseInt($("#input-pages").value) || 30
42
  const mode = $("#input-mode").value
43
+ const deepScan = $("#input-deep-scan").checked
44
  const status = $("#submit-status")
45
  status.textContent = "Submitting..."
46
  status.className = ""
 
61
  resp = await fetch("/api/submit_users", {
62
  method: "POST",
63
  headers: { "Content-Type": "application/json" },
64
+ body: JSON.stringify({ user_ids: userIds, deep_scan: deepScan })
65
  })
66
  } else {
67
  resp = await fetch("/api/submit", {
68
  method: "POST",
69
  headers: { "Content-Type": "application/json" },
70
+ body: JSON.stringify({ url, pages, mode, deep_scan: deepScan })
71
  })
72
  }
73
  const data = await resp.json()
 
552
  }
553
 
554
  function resultCardKey(item) {
555
+ return `${item.post_id}|${item.url || ""}|${(item.pages || []).join(",")}`
556
  }
557
 
558
  function resultBadge(item) {
 
567
  const pg = item.url ? pageSuffix(item.url) : ""
568
  const label = pid + pg
569
  const index = item.scan_index || 0
570
+ const pageCount = item.page_count || 1
571
  const download = item.download_url ? `<a class="download-link" href="${esc(item.download_url)}">Download</a>` : ""
572
  const thumb = item.image_url
573
  ? `<button class="thumb" data-preview="${esc(item.preview_url || item.image_url)}" data-download="${esc(item.download_url || "")}"><img src="${esc(item.image_url)}" loading="lazy" alt="${esc(label)}"></button>`
 
579
  ${download}
580
  </div>
581
  ${resultBadge(item)}
582
+ ${pageCount > 1 ? `<span class="result-pages">${pageCount}</span>` : ""}
583
  <span class="result-index">${index}</span>
584
  </div>`
585
  }
 
594
  const thumb = card.querySelector(".thumb")
595
  if (thumb) {
596
  const img = thumb.querySelector("img")
597
+ thumb.addEventListener("click", () => openViewer(item))
598
  img.addEventListener("load", () => thumb.classList.add("thumb-loaded"))
599
  img.addEventListener("error", () => thumb.classList.add("thumb-error"))
600
  if (img.complete) thumb.classList.add(img.naturalWidth ? "thumb-loaded" : "thumb-error")
 
628
  $("#post-search-grid").replaceChildren(...cards)
629
  }
630
 
631
+ function openViewer(item) {
632
+ if (!item.preview_url && !item.image_url) return
633
+ viewerPostId = item.post_id
634
+ viewerPages = item.pages?.length ? item.pages : [item.page || 0]
635
+ viewerPageIndex = Math.max(viewerPages.indexOf(item.page || 0), 0)
636
  viewerScale = 1
 
 
637
  $("#viewer").classList.remove("hidden")
638
  $("#viewer").setAttribute("aria-hidden", "false")
639
+ showViewerPage()
640
+ }
641
+
642
+ function showViewerPage() {
643
+ const page = viewerPages[viewerPageIndex]
644
+ const pid = encodeURIComponent(viewerPostId)
645
+ $("#viewer-img").src = `/api/i/${pid}/v?p=${page}`
646
+ $("#viewer-download").href = `/api/i/${pid}/o?p=${page}`
647
+ $("#viewer-prev").disabled = viewerPageIndex <= 0
648
+ $("#viewer-next").disabled = viewerPageIndex >= viewerPages.length - 1
649
+ $("#viewer-prev").classList.toggle("hidden", viewerPages.length <= 1)
650
+ $("#viewer-next").classList.toggle("hidden", viewerPages.length <= 1)
651
+ viewerScale = 1
652
  applyViewerZoom()
653
  }
654
 
655
+ function moveViewerPage(step) {
656
+ const next = Math.min(Math.max(viewerPageIndex + step, 0), viewerPages.length - 1)
657
+ if (next === viewerPageIndex) return
658
+ viewerPageIndex = next
659
+ showViewerPage()
660
+ }
661
+
662
  function closeViewer() {
663
  $("#viewer").classList.add("hidden")
664
  $("#viewer").setAttribute("aria-hidden", "true")
665
  $("#viewer-img").src = ""
666
  $("#viewer-download").href = ""
667
+ viewerPostId = ""
668
+ viewerPages = []
669
  viewerDrag = null
670
  $("#viewer-stage").classList.remove("dragging")
671
  }
 
686
  })
687
 
688
  $("#viewer-close").addEventListener("click", closeViewer)
689
+ $("#viewer-prev").addEventListener("click", () => moveViewerPage(-1))
690
+ $("#viewer-next").addEventListener("click", () => moveViewerPage(1))
691
  $("#viewer").addEventListener("click", e => { if (e.target.id === "viewer") closeViewer() })
692
  $("#viewer-stage").addEventListener("wheel", e => {
693
  e.preventDefault()
 
695
  applyViewerZoom()
696
  })
697
  $("#viewer-stage").addEventListener("pointerdown", e => {
698
+ if (e.button !== 0 || e.target.closest(".viewer-nav")) return
699
  const stage = $("#viewer-stage")
700
  viewerDrag = {
701
  x: e.clientX,
 
721
  viewerDrag = null
722
  $("#viewer-stage").classList.remove("dragging")
723
  })
724
+ window.addEventListener("keydown", e => {
725
+ if ($("#viewer").classList.contains("hidden")) return
726
+ if (e.key === "Escape") closeViewer()
727
+ if (e.key === "ArrowLeft") { e.preventDefault(); moveViewerPage(-1) }
728
+ if (e.key === "ArrowRight") { e.preventDefault(); moveViewerPage(1) }
729
+ })
730
 
731
  function renderPager(id, data) {
732
  const pager = $("#pager")
frontend/index.html CHANGED
@@ -5,7 +5,7 @@
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
  <link rel="icon" href="data:;base64,=">
7
  <title>Pixif2</title>
8
- <link rel="stylesheet" href="style.css?v=20260704-1">
9
  </head>
10
  <body>
11
  <div class="topbar">
@@ -41,6 +41,10 @@
41
  <option value="real">Real only</option>
42
  </select>
43
  </div>
 
 
 
 
44
  <button id="btn-submit" class="btn-primary">Submit</button>
45
  </div>
46
  <div id="submit-status"></div>
@@ -106,9 +110,11 @@
106
  <button id="viewer-close" class="btn-secondary">Close</button>
107
  </div>
108
  <div id="viewer-stage">
 
109
  <img id="viewer-img" alt="">
 
110
  </div>
111
  </div>
112
- <script src="app.js?v=20260715-1"></script>
113
  </body>
114
  </html>
 
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
  <link rel="icon" href="data:;base64,=">
7
  <title>Pixif2</title>
8
+ <link rel="stylesheet" href="style.css?v=20260716-1">
9
  </head>
10
  <body>
11
  <div class="topbar">
 
41
  <option value="real">Real only</option>
42
  </select>
43
  </div>
44
+ <label class="check-field">
45
+ <input type="checkbox" id="input-deep-scan">
46
+ <span>Deep scan</span>
47
+ </label>
48
  <button id="btn-submit" class="btn-primary">Submit</button>
49
  </div>
50
  <div id="submit-status"></div>
 
110
  <button id="viewer-close" class="btn-secondary">Close</button>
111
  </div>
112
  <div id="viewer-stage">
113
+ <button id="viewer-prev" class="viewer-nav" aria-label="Previous result page">&lsaquo;</button>
114
  <img id="viewer-img" alt="">
115
+ <button id="viewer-next" class="viewer-nav" aria-label="Next result page">&rsaquo;</button>
116
  </div>
117
  </div>
118
+ <script src="app.js?v=20260716-1"></script>
119
  </body>
120
  </html>
frontend/style.css CHANGED
@@ -91,10 +91,21 @@ h1 {
91
  }
92
  .submit-grid {
93
  display: grid;
94
- grid-template-columns: minmax(18rem, 1fr) 8rem 10rem auto;
95
  gap: .75rem;
96
  align-items: end;
97
  }
 
 
 
 
 
 
 
 
 
 
 
98
  .form-group { min-width: 0; }
99
  .form-group label {
100
  display: block;
@@ -487,6 +498,15 @@ input:focus, select:focus, textarea:focus { outline: none; border-color: var(--a
487
  font-size: .72rem;
488
  font-weight: 800;
489
  }
 
 
 
 
 
 
 
 
 
490
  #viewer {
491
  position: fixed;
492
  inset: 0;
@@ -506,6 +526,12 @@ input:focus, select:focus, textarea:focus { outline: none; border-color: var(--a
506
  background: #080808ee;
507
  border-bottom: 1px solid var(--line);
508
  }
 
 
 
 
 
 
509
  #viewer-zoom {
510
  min-width: 4rem;
511
  text-align: center;
@@ -513,6 +539,7 @@ input:focus, select:focus, textarea:focus { outline: none; border-color: var(--a
513
  font-size: .85rem;
514
  }
515
  #viewer-stage {
 
516
  flex: 1;
517
  display: flex;
518
  align-items: flex-start;
@@ -522,6 +549,22 @@ input:focus, select:focus, textarea:focus { outline: none; border-color: var(--a
522
  cursor: grab;
523
  touch-action: none;
524
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
525
  #viewer-stage.dragging { cursor: grabbing; }
526
  #viewer-img {
527
  max-width: none;
 
91
  }
92
  .submit-grid {
93
  display: grid;
94
+ grid-template-columns: minmax(14rem, 1fr) 6rem 8rem auto auto;
95
  gap: .75rem;
96
  align-items: end;
97
  }
98
+ .check-field {
99
+ height: 42px;
100
+ display: flex;
101
+ align-items: center;
102
+ gap: .45rem;
103
+ color: var(--muted);
104
+ font-size: .78rem;
105
+ font-weight: 800;
106
+ white-space: nowrap;
107
+ }
108
+ .check-field input { width: auto; height: auto; accent-color: var(--accent); }
109
  .form-group { min-width: 0; }
110
  .form-group label {
111
  display: block;
 
498
  font-size: .72rem;
499
  font-weight: 800;
500
  }
501
+ .result-pages {
502
+ position: absolute;
503
+ left: .55rem;
504
+ bottom: .45rem;
505
+ color: var(--accent);
506
+ font-family: Consolas, "SFMono-Regular", monospace;
507
+ font-size: .72rem;
508
+ font-weight: 800;
509
+ }
510
  #viewer {
511
  position: fixed;
512
  inset: 0;
 
526
  background: #080808ee;
527
  border-bottom: 1px solid var(--line);
528
  }
529
+ #viewer-download {
530
+ display: inline-flex;
531
+ align-items: center;
532
+ padding-top: .12rem;
533
+ line-height: 1;
534
+ }
535
  #viewer-zoom {
536
  min-width: 4rem;
537
  text-align: center;
 
539
  font-size: .85rem;
540
  }
541
  #viewer-stage {
542
+ position: relative;
543
  flex: 1;
544
  display: flex;
545
  align-items: flex-start;
 
549
  cursor: grab;
550
  touch-action: none;
551
  }
552
+ .viewer-nav {
553
+ position: fixed;
554
+ top: 50%;
555
+ z-index: 1;
556
+ width: 46px;
557
+ height: 64px;
558
+ background: #080808dd;
559
+ border: 1px solid var(--line);
560
+ color: var(--text);
561
+ font-size: 2rem;
562
+ cursor: pointer;
563
+ }
564
+ .viewer-nav:hover { color: var(--accent); border-color: var(--accent); }
565
+ .viewer-nav:disabled { opacity: .3; cursor: default; }
566
+ #viewer-prev { left: 1rem; }
567
+ #viewer-next { right: 1rem; }
568
  #viewer-stage.dragging { cursor: grabbing; }
569
  #viewer-img {
570
  max-width: none;