q6 commited on
Commit
ca3f25b
·
1 Parent(s): 055e7c6

Deep scan all pages and show multiple tags

Browse files
Files changed (4) hide show
  1. backend/app.py +125 -75
  2. frontend/app.js +35 -8
  3. frontend/index.html +1 -1
  4. frontend/style.css +1 -0
backend/app.py CHANGED
@@ -239,7 +239,7 @@ async def init_db():
239
  "ALTER TABLE pi_searches DROP COLUMN created_at",
240
  "ALTER TABLE pi_searches ADD COLUMN found_exif INTEGER DEFAULT 0",
241
  "ALTER TABLE pi_searches ADD COLUMN total_searched INTEGER DEFAULT 0",
242
- "ALTER TABLE pi_searches ADD COLUMN deep_scan INTEGER DEFAULT 0",
243
  "ALTER TABLE pi_scans ADD COLUMN deep_scanned INTEGER DEFAULT 0",
244
  ):
245
  try:
@@ -369,7 +369,7 @@ def get_search_api_url(raw, keywords):
369
  return f"https://www.pixiv.net/ajax/search/artworks/{encoded}?{params}"
370
 
371
 
372
- async def save_search(search_id, post_ids, deep_scan=False):
373
  if not post_ids:
374
  return
375
  if task_canceled(search_id):
@@ -702,20 +702,16 @@ async def get_deep_scan_targets(search_id):
702
  "sql": "WITH ids AS ("
703
  "SELECT CAST(j.value AS TEXT) post_id, CAST(j.key AS INTEGER) pos "
704
  "FROM pi_searches p, json_each(p.post_ids) j WHERE p.id = ?"
705
- ") SELECT ids.post_id, s.url, s.exif_type FROM ids "
706
  "JOIN pi_scans s ON s.post_id = ids.post_id "
707
- "WHERE s.exif_type IS NOT NULL AND COALESCE(s.deep_scanned, 0) = 0 "
708
  "ORDER BY ids.pos",
709
  "args": [{"type": "text", "value": search_id}],
710
  }
711
  ]
712
  )
713
  return [
714
- {
715
- "post_id": cell_value(row, 0, ""),
716
- "url": cell_value(row, 1, ""),
717
- "exif_type": int(cell_value(row, 2, "0")),
718
- }
719
  for row in first_rows(resp)
720
  if cell_value(row, 0, "")
721
  ]
@@ -733,8 +729,8 @@ async def get_incomplete_search_ids():
733
  ") SELECT p.id FROM pi_searches p JOIN counts ON counts.id = p.id "
734
  "WHERE total > scanned OR (COALESCE(p.deep_scan, 0) = 1 AND EXISTS ("
735
  "SELECT 1 FROM json_each(p.post_ids) j JOIN pi_scans s "
736
- "ON s.post_id = CAST(j.value AS TEXT) WHERE s.exif_type IS NOT NULL "
737
- "AND COALESCE(s.deep_scanned, 0) = 0)) ORDER BY p.id",
738
  }
739
  ]
740
  )
@@ -769,25 +765,21 @@ async def get_refresh_search_ids():
769
 
770
 
771
  async def get_result_counts(search_id):
772
- base_sql = (
773
- "WITH ids AS ("
774
- "SELECT CAST(j.value AS TEXT) post_id "
775
- "FROM pi_searches p, json_each(p.post_ids) j WHERE p.id = ?"
776
- ") "
777
- )
778
  id_arg = {"type": "text", "value": search_id}
779
  resp = await turso_execute(
780
  [
781
  {
782
- "sql": base_sql
783
- + "SELECT COUNT(*), COUNT(s.post_id), COUNT(s.exif_type) "
 
784
  "FROM ids LEFT JOIN pi_scans s ON s.post_id = ids.post_id",
785
  "args": [id_arg],
786
  },
787
  {
788
- "sql": base_sql
789
- + "SELECT COALESCE(s.exif_type, 0), COUNT(*) FROM ids JOIN pi_scans s ON s.post_id = ids.post_id "
790
- "GROUP BY COALESCE(s.exif_type, 0)",
 
791
  "args": [id_arg],
792
  },
793
  ]
@@ -807,16 +799,31 @@ async def get_result_counts(search_id):
807
  }
808
 
809
 
 
 
 
 
 
 
 
 
 
 
 
 
810
  def result_filter_sql(allowed):
811
  if allowed is None:
812
- return "s.exif_type IS NOT NULL", []
813
  parts = []
814
  args = []
815
  if 0 in allowed:
816
- parts.append("s.exif_type IS NULL")
817
  codes = [x for x in allowed if x]
818
  if codes:
819
- parts.append(f"s.exif_type IN ({','.join('?' for _ in codes)})")
 
 
 
820
  args = [{"type": "integer", "value": str(x)} for x in codes]
821
  if not parts:
822
  return "0", []
@@ -868,14 +875,18 @@ async def get_result_pages(post_ids):
868
  resp = await turso_execute(
869
  [
870
  {
871
- "sql": f"SELECT post_id, page FROM pi_scan_pages WHERE post_id IN ({marks}) ORDER BY post_id, page",
872
  "args": [{"type": "text", "value": str(pid)} for pid in chunk],
873
  }
874
  ]
875
  )
876
  for row in first_rows(resp):
877
  pages.setdefault(cell_value(row, 0, ""), []).append(
878
- int(cell_value(row, 1, "0"))
 
 
 
 
879
  )
880
  return pages
881
 
@@ -885,10 +896,8 @@ async def get_download_targets(search_id, through, allowed):
885
  resp = await turso_execute(
886
  [
887
  {
888
- "sql": "WITH ids AS ("
889
- "SELECT CAST(j.value AS TEXT) post_id, CAST(j.key AS INTEGER) + 1 scan_index "
890
- "FROM pi_searches p, json_each(p.post_ids) j WHERE p.id = ?"
891
- ") SELECT ids.post_id, ids.scan_index, COALESCE(p.url, s.url) "
892
  "FROM ids JOIN pi_scans s ON s.post_id = ids.post_id "
893
  "LEFT JOIN pi_scan_pages p ON p.post_id = ids.post_id "
894
  f"WHERE ids.scan_index <= ? AND {where_sql} "
@@ -955,29 +964,24 @@ async def get_result_page(search_id, page, allowed):
955
  where_sql, filter_args = result_filter_sql(allowed)
956
  page = max(page, 1)
957
  offset = (page - 1) * PAGE_SIZE
958
- base_sql = (
959
- "WITH ids AS ("
960
- "SELECT CAST(j.value AS TEXT) post_id, CAST(j.key AS INTEGER) + 1 scan_index "
961
- "FROM pi_searches p, json_each(p.post_ids) j WHERE p.id = ?"
962
- ") "
963
- )
964
  id_arg = {"type": "text", "value": search_id}
965
  resp = await turso_execute(
966
  [
967
  {
968
- "sql": base_sql
969
- + "SELECT COUNT(*), COUNT(s.post_id), COUNT(s.exif_type) "
 
970
  "FROM ids LEFT JOIN pi_scans s ON s.post_id = ids.post_id",
971
  "args": [id_arg],
972
  },
973
  {
974
- "sql": base_sql
975
  + "SELECT COUNT(*) FROM ids JOIN pi_scans s ON s.post_id = ids.post_id "
976
  f"WHERE {where_sql}",
977
  "args": [id_arg, *filter_args],
978
  },
979
  {
980
- "sql": base_sql
981
  + "SELECT ids.post_id, ids.scan_index, s.url, s.exif_type "
982
  "FROM ids JOIN pi_scans s ON s.post_id = ids.post_id "
983
  f"WHERE {where_sql} ORDER BY ids.scan_index LIMIT ? OFFSET ?",
@@ -989,9 +993,10 @@ async def get_result_page(search_id, page, allowed):
989
  ],
990
  },
991
  {
992
- "sql": base_sql
993
- + "SELECT COALESCE(s.exif_type, 0), COUNT(*) FROM ids JOIN pi_scans s ON s.post_id = ids.post_id "
994
- "GROUP BY COALESCE(s.exif_type, 0)",
 
995
  "args": [id_arg],
996
  },
997
  ]
@@ -1015,15 +1020,31 @@ async def get_result_page(search_id, page, allowed):
1015
  pid = cell_value(row, 0, "")
1016
  url = cell_value(row, 2, "")
1017
  exif_type = cell_value(row, 3)
1018
- pages = result_pages.get(pid) or [page_num_from_url(url)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1019
  items.append(
1020
  {
1021
  "post_id": pid,
1022
  "scan_index": int(cell_value(row, 1, "0")),
1023
  "url": url,
1024
  "exif_type": int(exif_type) if exif_type else None,
 
1025
  "scanned": True,
1026
  "pages": pages,
 
1027
  "page_count": len(pages),
1028
  **image_links(pid, url),
1029
  }
@@ -1044,7 +1065,7 @@ def user_search_id(user_id):
1044
  return f"{base26_time()}_{user_id}"
1045
 
1046
 
1047
- async def pixiv_search_live(url, pages, mode, phpsessid, search_id, deep_scan=False):
1048
  keywords = get_search_keywords(url)
1049
  api_url = get_search_api_url(url, keywords)
1050
  first_url = f"{api_url}&p=1"
@@ -1567,16 +1588,12 @@ async def scan_post(post_id, session, post_sem, exif_sem, img_sem):
1567
  return post_id, None, None
1568
 
1569
 
1570
- async def scan_page_result(post_id, url, session, exif_sem, img_sem):
1571
  page = page_num_from_url(url)
1572
- metadata = await get_exif_range(post_id, page, url, session, exif_sem)
1573
- exif_type = determine_exif_type(metadata)
1574
- fallback = EXIF_TYPE_TO_CODE.get(exif_type) if exif_type in FALLBACK_EXIF_TYPES else None
1575
- if exif_type and not fallback:
1576
- return post_id, url, EXIF_TYPE_TO_CODE.get(exif_type)
1577
  img_data = await fetch_image(session, url, img_sem, post_id, page)
1578
  if not img_data:
1579
- return (post_id, url, fallback) if fallback else None
 
1580
  exif_type = determine_exif_type(parse_image_metadata(img_data))
1581
  if exif_type in FALLBACK_EXIF_TYPES:
1582
  fallback = fallback or EXIF_TYPE_TO_CODE.get(exif_type)
@@ -1593,20 +1610,15 @@ async def scan_page_result(post_id, url, session, exif_sem, img_sem):
1593
  return (post_id, url, fallback) if fallback else None
1594
 
1595
 
1596
- async def deep_scan_post(item, session, post_sem, exif_sem, img_sem):
1597
  async with post_sem:
1598
  try:
1599
  post_id = item["post_id"]
1600
  pages = await get_post_pages(post_id, session)
1601
- first_page = page_num_from_url(item["url"])
1602
- results = [(post_id, first_page, item["url"], item["exif_type"])]
1603
- for page in range(len(pages)):
1604
- if page == first_page:
1605
- continue
1606
- url = pages[page]["original"]
1607
- if image_ext(url) not in METADATA_EXTS:
1608
- continue
1609
- result = await scan_page_result(post_id, url, session, exif_sem, img_sem)
1610
  if result:
1611
  results.append((post_id, page, result[1], result[2]))
1612
  return post_id, results
@@ -1711,14 +1723,13 @@ async def run_scan(post_ids, phpsessid, task_id=None, save_live=False, done_offs
1711
 
1712
  async def run_deep_scan(targets, phpsessid, task_id=None):
1713
  post_sem = asyncio.Semaphore(POST_SCAN_LIMIT)
1714
- exif_sem = asyncio.Semaphore(EXIF_RANGE_LIMIT)
1715
  img_sem = asyncio.Semaphore(FULL_IMAGE_LIMIT)
1716
  cookies = {"PHPSESSID": phpsessid}
1717
  completed = 0
1718
  pending = []
1719
  async with aiohttp.ClientSession(cookies=cookies, headers=PIXIV_HEADERS) as session:
1720
  tasks = {
1721
- asyncio.create_task(deep_scan_post(item, session, post_sem, exif_sem, img_sem))
1722
  for item in targets
1723
  }
1724
  created = set(tasks)
@@ -1785,6 +1796,12 @@ async def save_scan_results(results):
1785
  async def save_deep_scan_results(results):
1786
  stmts = []
1787
  for post_id, pages in results:
 
 
 
 
 
 
1788
  for _, page, url, exif_type in pages:
1789
  short_url = url.replace(IMG_BASE, "", 1) if url else ""
1790
  stmts.append(
@@ -1798,10 +1815,22 @@ async def save_deep_scan_results(results):
1798
  ],
1799
  }
1800
  )
 
 
 
 
 
 
1801
  stmts.append(
1802
  {
1803
- "sql": "UPDATE pi_scans SET deep_scanned = 1 WHERE post_id = ?",
1804
- "args": [{"type": "text", "value": str(post_id)}],
 
 
 
 
 
 
1805
  }
1806
  )
1807
  for i in range(0, len(stmts), 200):
@@ -2117,7 +2146,7 @@ async def deep_scan_search(search_id, phpsessid):
2117
  await run_deep_scan(targets, phpsessid, task_id=search_id)
2118
 
2119
 
2120
- async def bg_user_task(search_id, user_id, phpsessid, deep_scan=False):
2121
  if not start_task(search_id, {
2122
  "type": "user_search",
2123
  "phase": "searching",
@@ -2156,7 +2185,7 @@ async def bg_user_task(search_id, user_id, phpsessid, deep_scan=False):
2156
  await finish_task(search_id)
2157
 
2158
 
2159
- async def bg_user_batch_task(jobs, phpsessid, deep_scan=False):
2160
  await asyncio.gather(
2161
  *(
2162
  bg_user_task(search_id, user_id, phpsessid, deep_scan)
@@ -2165,7 +2194,7 @@ async def bg_user_batch_task(jobs, phpsessid, deep_scan=False):
2165
  )
2166
 
2167
 
2168
- async def bg_scan_task(search_id, post_ids, phpsessid, deep_scan=False):
2169
  if not start_task(search_id, {
2170
  "type": "scan",
2171
  "phase": "scanning",
@@ -2193,7 +2222,7 @@ async def bg_scan_task(search_id, post_ids, phpsessid, deep_scan=False):
2193
 
2194
 
2195
  async def bg_search_and_scan_task(
2196
- search_id, url, pages, mode, phpsessid, deep_scan=False
2197
  ):
2198
  if not start_task(search_id, {
2199
  "type": "search+scan",
@@ -2308,13 +2337,13 @@ class SearchRequest(BaseModel):
2308
  pages: int = 30
2309
  mode: str = "ai"
2310
  action: str = "search"
2311
- deep_scan: bool = False
2312
 
2313
 
2314
  class UserSearchRequest(BaseModel):
2315
  user_ids: list
2316
  action: str = "search"
2317
- deep_scan: bool = False
2318
 
2319
 
2320
  class PostSearchRequest(BaseModel):
@@ -2514,17 +2543,29 @@ async def post_search(req: PostSearchRequest):
2514
  return {"error": "no post ids"}
2515
  post_ids = post_ids[:POST_SEARCH_LIMIT]
2516
  scanned = await get_scanned_post_ids(post_ids)
 
2517
  items = []
2518
  for index, pid in enumerate(post_ids, 1):
2519
  row = scanned.get(pid, {})
2520
  url = row.get("url", "")
 
 
 
 
 
2521
  items.append(
2522
  {
2523
  "post_id": pid,
2524
  "scan_index": index,
2525
  "url": url,
2526
  "exif_type": row.get("exif_type"),
 
2527
  "scanned": pid in scanned,
 
 
 
 
 
2528
  **image_links(pid, url),
2529
  }
2530
  )
@@ -2539,8 +2580,17 @@ async def scan_search(req: ScanRequest, bg: BackgroundTasks):
2539
  counts = await get_search_counts(req.search_id)
2540
  if not counts or not counts["raw_total"]:
2541
  return {"error": "not found"}
 
 
 
 
 
 
 
 
2542
  to_scan = await get_unscanned_post_ids(req.search_id)
2543
- if not to_scan:
 
2544
  return {"status": "already_scanned", "count": counts["raw_total"]}
2545
  if not start_task(req.search_id, {
2546
  "type": "scan",
@@ -2549,7 +2599,7 @@ async def scan_search(req: ScanRequest, bg: BackgroundTasks):
2549
  }):
2550
  return {"status": "canceling"}
2551
  await publish_task_event(req.search_id)
2552
- bg.add_task(bg_scan_task, req.search_id, to_scan, phpsessid)
2553
  return {"status": "scanning", "total": counts["raw_total"], "to_scan": len(to_scan)}
2554
 
2555
 
 
239
  "ALTER TABLE pi_searches DROP COLUMN created_at",
240
  "ALTER TABLE pi_searches ADD COLUMN found_exif INTEGER DEFAULT 0",
241
  "ALTER TABLE pi_searches ADD COLUMN total_searched INTEGER DEFAULT 0",
242
+ "ALTER TABLE pi_searches ADD COLUMN deep_scan INTEGER DEFAULT 1",
243
  "ALTER TABLE pi_scans ADD COLUMN deep_scanned INTEGER DEFAULT 0",
244
  ):
245
  try:
 
369
  return f"https://www.pixiv.net/ajax/search/artworks/{encoded}?{params}"
370
 
371
 
372
+ async def save_search(search_id, post_ids, deep_scan=True):
373
  if not post_ids:
374
  return
375
  if task_canceled(search_id):
 
702
  "sql": "WITH ids AS ("
703
  "SELECT CAST(j.value AS TEXT) post_id, CAST(j.key AS INTEGER) pos "
704
  "FROM pi_searches p, json_each(p.post_ids) j WHERE p.id = ?"
705
+ ") SELECT ids.post_id FROM ids "
706
  "JOIN pi_scans s ON s.post_id = ids.post_id "
707
+ "WHERE COALESCE(s.deep_scanned, 0) = 0 "
708
  "ORDER BY ids.pos",
709
  "args": [{"type": "text", "value": search_id}],
710
  }
711
  ]
712
  )
713
  return [
714
+ {"post_id": cell_value(row, 0, "")}
 
 
 
 
715
  for row in first_rows(resp)
716
  if cell_value(row, 0, "")
717
  ]
 
729
  ") SELECT p.id FROM pi_searches p JOIN counts ON counts.id = p.id "
730
  "WHERE total > scanned OR (COALESCE(p.deep_scan, 0) = 1 AND EXISTS ("
731
  "SELECT 1 FROM json_each(p.post_ids) j JOIN pi_scans s "
732
+ "ON s.post_id = CAST(j.value AS TEXT) "
733
+ "WHERE COALESCE(s.deep_scanned, 0) = 0)) ORDER BY p.id",
734
  }
735
  ]
736
  )
 
765
 
766
 
767
  async def get_result_counts(search_id):
 
 
 
 
 
 
768
  id_arg = {"type": "text", "value": search_id}
769
  resp = await turso_execute(
770
  [
771
  {
772
+ "sql": RESULT_BASE_SQL
773
+ + "SELECT COUNT(*), COUNT(s.post_id), "
774
+ "(SELECT COUNT(DISTINCT post_id) FROM tags) "
775
  "FROM ids LEFT JOIN pi_scans s ON s.post_id = ids.post_id",
776
  "args": [id_arg],
777
  },
778
  {
779
+ "sql": RESULT_BASE_SQL
780
+ + "SELECT exif_type, COUNT(DISTINCT post_id) FROM tags GROUP BY exif_type "
781
+ "UNION ALL SELECT 0, COUNT(*) FROM ids "
782
+ "WHERE NOT EXISTS (SELECT 1 FROM tags WHERE tags.post_id = ids.post_id)",
783
  "args": [id_arg],
784
  },
785
  ]
 
799
  }
800
 
801
 
802
+ RESULT_BASE_SQL = (
803
+ "WITH ids AS ("
804
+ "SELECT CAST(j.value AS TEXT) post_id, CAST(j.key AS INTEGER) + 1 scan_index "
805
+ "FROM pi_searches p, json_each(p.post_ids) j WHERE p.id = ?"
806
+ "), tags AS ("
807
+ "SELECT p.post_id, p.exif_type FROM pi_scan_pages p JOIN ids ON ids.post_id = p.post_id "
808
+ "WHERE p.exif_type IS NOT NULL UNION "
809
+ "SELECT s.post_id, s.exif_type FROM pi_scans s JOIN ids ON ids.post_id = s.post_id "
810
+ "WHERE s.exif_type IS NOT NULL) "
811
+ )
812
+
813
+
814
  def result_filter_sql(allowed):
815
  if allowed is None:
816
+ return "EXISTS (SELECT 1 FROM tags t WHERE t.post_id = s.post_id)", []
817
  parts = []
818
  args = []
819
  if 0 in allowed:
820
+ parts.append("NOT EXISTS (SELECT 1 FROM tags t WHERE t.post_id = s.post_id)")
821
  codes = [x for x in allowed if x]
822
  if codes:
823
+ parts.append(
824
+ "EXISTS (SELECT 1 FROM tags t WHERE t.post_id = s.post_id "
825
+ f"AND t.exif_type IN ({','.join('?' for _ in codes)}))"
826
+ )
827
  args = [{"type": "integer", "value": str(x)} for x in codes]
828
  if not parts:
829
  return "0", []
 
875
  resp = await turso_execute(
876
  [
877
  {
878
+ "sql": f"SELECT post_id, page, url, exif_type FROM pi_scan_pages WHERE post_id IN ({marks}) ORDER BY post_id, page",
879
  "args": [{"type": "text", "value": str(pid)} for pid in chunk],
880
  }
881
  ]
882
  )
883
  for row in first_rows(resp):
884
  pages.setdefault(cell_value(row, 0, ""), []).append(
885
+ {
886
+ "page": int(cell_value(row, 1, "0")),
887
+ "url": cell_value(row, 2, ""),
888
+ "exif_type": int(cell_value(row, 3, "0")),
889
+ }
890
  )
891
  return pages
892
 
 
896
  resp = await turso_execute(
897
  [
898
  {
899
+ "sql": RESULT_BASE_SQL
900
+ + "SELECT ids.post_id, ids.scan_index, COALESCE(p.url, s.url) "
 
 
901
  "FROM ids JOIN pi_scans s ON s.post_id = ids.post_id "
902
  "LEFT JOIN pi_scan_pages p ON p.post_id = ids.post_id "
903
  f"WHERE ids.scan_index <= ? AND {where_sql} "
 
964
  where_sql, filter_args = result_filter_sql(allowed)
965
  page = max(page, 1)
966
  offset = (page - 1) * PAGE_SIZE
 
 
 
 
 
 
967
  id_arg = {"type": "text", "value": search_id}
968
  resp = await turso_execute(
969
  [
970
  {
971
+ "sql": RESULT_BASE_SQL
972
+ + "SELECT COUNT(*), COUNT(s.post_id), "
973
+ "(SELECT COUNT(DISTINCT post_id) FROM tags) "
974
  "FROM ids LEFT JOIN pi_scans s ON s.post_id = ids.post_id",
975
  "args": [id_arg],
976
  },
977
  {
978
+ "sql": RESULT_BASE_SQL
979
  + "SELECT COUNT(*) FROM ids JOIN pi_scans s ON s.post_id = ids.post_id "
980
  f"WHERE {where_sql}",
981
  "args": [id_arg, *filter_args],
982
  },
983
  {
984
+ "sql": RESULT_BASE_SQL
985
  + "SELECT ids.post_id, ids.scan_index, s.url, s.exif_type "
986
  "FROM ids JOIN pi_scans s ON s.post_id = ids.post_id "
987
  f"WHERE {where_sql} ORDER BY ids.scan_index LIMIT ? OFFSET ?",
 
993
  ],
994
  },
995
  {
996
+ "sql": RESULT_BASE_SQL
997
+ + "SELECT exif_type, COUNT(DISTINCT post_id) FROM tags GROUP BY exif_type "
998
+ "UNION ALL SELECT 0, COUNT(*) FROM ids "
999
+ "WHERE NOT EXISTS (SELECT 1 FROM tags WHERE tags.post_id = ids.post_id)",
1000
  "args": [id_arg],
1001
  },
1002
  ]
 
1020
  pid = cell_value(row, 0, "")
1021
  url = cell_value(row, 2, "")
1022
  exif_type = cell_value(row, 3)
1023
+ page_rows = result_pages.get(pid) or []
1024
+ exif_types = sorted({item["exif_type"] for item in page_rows})
1025
+ if not exif_types and exif_type:
1026
+ exif_types = [int(exif_type)]
1027
+ enabled = set(allowed if allowed is not None else exif_types)
1028
+ visible_rows = [item for item in page_rows if item["exif_type"] in enabled]
1029
+ selected = visible_rows[0] if visible_rows else None
1030
+ if selected:
1031
+ url = selected["url"]
1032
+ pages = [item["page"] for item in visible_rows] or [page_num_from_url(url)]
1033
+ page_types = {
1034
+ str(item["page"]): item["exif_type"] for item in page_rows
1035
+ }
1036
+ if not page_rows:
1037
+ page_types[str(pages[0])] = int(exif_type) if exif_type else 0
1038
  items.append(
1039
  {
1040
  "post_id": pid,
1041
  "scan_index": int(cell_value(row, 1, "0")),
1042
  "url": url,
1043
  "exif_type": int(exif_type) if exif_type else None,
1044
+ "exif_types": exif_types,
1045
  "scanned": True,
1046
  "pages": pages,
1047
+ "page_types": page_types,
1048
  "page_count": len(pages),
1049
  **image_links(pid, url),
1050
  }
 
1065
  return f"{base26_time()}_{user_id}"
1066
 
1067
 
1068
+ async def pixiv_search_live(url, pages, mode, phpsessid, search_id, deep_scan=True):
1069
  keywords = get_search_keywords(url)
1070
  api_url = get_search_api_url(url, keywords)
1071
  first_url = f"{api_url}&p=1"
 
1588
  return post_id, None, None
1589
 
1590
 
1591
+ async def scan_page_result(post_id, url, session, img_sem):
1592
  page = page_num_from_url(url)
 
 
 
 
 
1593
  img_data = await fetch_image(session, url, img_sem, post_id, page)
1594
  if not img_data:
1595
+ return None
1596
+ fallback = None
1597
  exif_type = determine_exif_type(parse_image_metadata(img_data))
1598
  if exif_type in FALLBACK_EXIF_TYPES:
1599
  fallback = fallback or EXIF_TYPE_TO_CODE.get(exif_type)
 
1610
  return (post_id, url, fallback) if fallback else None
1611
 
1612
 
1613
+ async def deep_scan_post(item, session, post_sem, img_sem):
1614
  async with post_sem:
1615
  try:
1616
  post_id = item["post_id"]
1617
  pages = await get_post_pages(post_id, session)
1618
+ results = []
1619
+ for page, urls in enumerate(pages):
1620
+ url = urls["original"]
1621
+ result = await scan_page_result(post_id, url, session, img_sem)
 
 
 
 
 
1622
  if result:
1623
  results.append((post_id, page, result[1], result[2]))
1624
  return post_id, results
 
1723
 
1724
  async def run_deep_scan(targets, phpsessid, task_id=None):
1725
  post_sem = asyncio.Semaphore(POST_SCAN_LIMIT)
 
1726
  img_sem = asyncio.Semaphore(FULL_IMAGE_LIMIT)
1727
  cookies = {"PHPSESSID": phpsessid}
1728
  completed = 0
1729
  pending = []
1730
  async with aiohttp.ClientSession(cookies=cookies, headers=PIXIV_HEADERS) as session:
1731
  tasks = {
1732
+ asyncio.create_task(deep_scan_post(item, session, post_sem, img_sem))
1733
  for item in targets
1734
  }
1735
  created = set(tasks)
 
1796
  async def save_deep_scan_results(results):
1797
  stmts = []
1798
  for post_id, pages in results:
1799
+ stmts.append(
1800
+ {
1801
+ "sql": "DELETE FROM pi_scan_pages WHERE post_id = ?",
1802
+ "args": [{"type": "text", "value": str(post_id)}],
1803
+ }
1804
+ )
1805
  for _, page, url, exif_type in pages:
1806
  short_url = url.replace(IMG_BASE, "", 1) if url else ""
1807
  stmts.append(
 
1815
  ],
1816
  }
1817
  )
1818
+ counts = {}
1819
+ for _, _, _, exif_type in pages:
1820
+ counts[exif_type] = counts.get(exif_type, 0) + 1
1821
+ best = max(counts, key=lambda code: (counts[code], -code)) if counts else None
1822
+ url = next((row[2] for row in pages if row[3] == best), "")
1823
+ short_url = url.replace(IMG_BASE, "", 1) if url else ""
1824
  stmts.append(
1825
  {
1826
+ "sql": "UPDATE pi_scans SET url = ?, exif_type = ?, deep_scanned = 1 WHERE post_id = ?",
1827
+ "args": [
1828
+ {"type": "text", "value": short_url},
1829
+ {"type": "integer", "value": str(best)}
1830
+ if best
1831
+ else {"type": "null"},
1832
+ {"type": "text", "value": str(post_id)},
1833
+ ],
1834
  }
1835
  )
1836
  for i in range(0, len(stmts), 200):
 
2146
  await run_deep_scan(targets, phpsessid, task_id=search_id)
2147
 
2148
 
2149
+ async def bg_user_task(search_id, user_id, phpsessid, deep_scan=True):
2150
  if not start_task(search_id, {
2151
  "type": "user_search",
2152
  "phase": "searching",
 
2185
  await finish_task(search_id)
2186
 
2187
 
2188
+ async def bg_user_batch_task(jobs, phpsessid, deep_scan=True):
2189
  await asyncio.gather(
2190
  *(
2191
  bg_user_task(search_id, user_id, phpsessid, deep_scan)
 
2194
  )
2195
 
2196
 
2197
+ async def bg_scan_task(search_id, post_ids, phpsessid, deep_scan=True):
2198
  if not start_task(search_id, {
2199
  "type": "scan",
2200
  "phase": "scanning",
 
2222
 
2223
 
2224
  async def bg_search_and_scan_task(
2225
+ search_id, url, pages, mode, phpsessid, deep_scan=True
2226
  ):
2227
  if not start_task(search_id, {
2228
  "type": "search+scan",
 
2337
  pages: int = 30
2338
  mode: str = "ai"
2339
  action: str = "search"
2340
+ deep_scan: bool = True
2341
 
2342
 
2343
  class UserSearchRequest(BaseModel):
2344
  user_ids: list
2345
  action: str = "search"
2346
+ deep_scan: bool = True
2347
 
2348
 
2349
  class PostSearchRequest(BaseModel):
 
2543
  return {"error": "no post ids"}
2544
  post_ids = post_ids[:POST_SEARCH_LIMIT]
2545
  scanned = await get_scanned_post_ids(post_ids)
2546
+ result_pages = await get_result_pages(post_ids)
2547
  items = []
2548
  for index, pid in enumerate(post_ids, 1):
2549
  row = scanned.get(pid, {})
2550
  url = row.get("url", "")
2551
+ page_rows = result_pages.get(pid) or []
2552
+ exif_types = sorted({item["exif_type"] for item in page_rows})
2553
+ if not exif_types and row.get("exif_type"):
2554
+ exif_types = [row["exif_type"]]
2555
+ pages = [item["page"] for item in page_rows] or [page_num_from_url(url)]
2556
  items.append(
2557
  {
2558
  "post_id": pid,
2559
  "scan_index": index,
2560
  "url": url,
2561
  "exif_type": row.get("exif_type"),
2562
+ "exif_types": exif_types,
2563
  "scanned": pid in scanned,
2564
+ "pages": pages,
2565
+ "page_types": {
2566
+ str(item["page"]): item["exif_type"] for item in page_rows
2567
+ },
2568
+ "page_count": len(pages),
2569
  **image_links(pid, url),
2570
  }
2571
  )
 
2580
  counts = await get_search_counts(req.search_id)
2581
  if not counts or not counts["raw_total"]:
2582
  return {"error": "not found"}
2583
+ await turso_execute(
2584
+ [
2585
+ {
2586
+ "sql": "UPDATE pi_searches SET deep_scan = 1 WHERE id = ?",
2587
+ "args": [{"type": "text", "value": req.search_id}],
2588
+ }
2589
+ ]
2590
+ )
2591
  to_scan = await get_unscanned_post_ids(req.search_id)
2592
+ deep_targets = await get_deep_scan_targets(req.search_id)
2593
+ if not to_scan and not deep_targets:
2594
  return {"status": "already_scanned", "count": counts["raw_total"]}
2595
  if not start_task(req.search_id, {
2596
  "type": "scan",
 
2599
  }):
2600
  return {"status": "canceling"}
2601
  await publish_task_event(req.search_id)
2602
+ bg.add_task(bg_scan_task, req.search_id, to_scan, phpsessid, True)
2603
  return {"status": "scanning", "total": counts["raw_total"], "to_scan": len(to_scan)}
2604
 
2605
 
frontend/app.js CHANGED
@@ -15,6 +15,8 @@ let viewerDrag = null
15
  let viewerPostId = ""
16
  let viewerPages = []
17
  let viewerPageIndex = 0
 
 
18
  let explorerEvents = null
19
  let explorerPage = 1
20
  let explorerDebounce = null
@@ -387,7 +389,10 @@ function renderExifFilters(id, submitted = false, counts = null) {
387
  const shownCounts = counts ?? exifCountsCache.get(id) ?? visibleExifCounts(id)
388
  if (shownCounts) exifCountsCache.set(id, shownCounts)
389
  const active = new Set(getExifTypes())
390
- const buttons = FILTER_CODES.map(code => {
 
 
 
391
  const count = shownCounts ? ` (${shownCounts[code] || 0})` : ""
392
  return `<button class="filter-btn${active.has(code) ? " active" : ""}" data-code="${code}">${esc(CATEGORY_NAMES[code])}${count}</button>`
393
  }).join("")
@@ -569,14 +574,16 @@ function pageSuffix(url) {
569
  }
570
 
571
  function resultCardKey(item) {
572
- return `${item.post_id}|${item.url || ""}|${(item.pages || []).join(",")}`
573
  }
574
 
575
  function resultBadge(item) {
576
  if (!item.scanned) return `<span class="result-badge not-scanned">not scanned</span>`
577
- if (!item.exif_type) return `<span class="result-badge no-exif">NIL</span>`
578
- const name = EXIF_NAMES[item.exif_type] || "?"
579
- return `<span class="result-badge exif-badge exif-${item.exif_type}">${name}</span>`
 
 
580
  }
581
 
582
  function resultCardHtml(item) {
@@ -620,6 +627,7 @@ function createResultCard(item) {
620
  }
621
 
622
  function renderResults(data) {
 
623
  const cards = data.items.map(item => {
624
  const key = resultCardKey(item)
625
  let card = resultCardCache.get(key)
@@ -633,6 +641,7 @@ function renderResults(data) {
633
  }
634
 
635
  function renderPostSearchResults(items) {
 
636
  const cards = items.map(item => {
637
  const key = resultCardKey(item)
638
  let card = postSearchCardCache.get(key)
@@ -647,15 +656,23 @@ function renderPostSearchResults(items) {
647
 
648
  function openViewer(item) {
649
  if (!item.preview_url && !item.image_url) return
650
- viewerPostId = item.post_id
651
- viewerPages = item.pages?.length ? item.pages : [item.page || 0]
652
- viewerPageIndex = Math.max(viewerPages.indexOf(item.page || 0), 0)
653
  viewerScale = 1
654
  $("#viewer").classList.remove("hidden")
655
  $("#viewer").setAttribute("aria-hidden", "false")
656
  showViewerPage()
657
  }
658
 
 
 
 
 
 
 
 
 
 
659
  function showViewerPage() {
660
  const page = viewerPages[viewerPageIndex]
661
  const pid = encodeURIComponent(viewerPostId)
@@ -676,6 +693,14 @@ function moveViewerPage(step) {
676
  showViewerPage()
677
  }
678
 
 
 
 
 
 
 
 
 
679
  function closeViewer() {
680
  $("#viewer").classList.add("hidden")
681
  $("#viewer").setAttribute("aria-hidden", "true")
@@ -743,6 +768,8 @@ window.addEventListener("keydown", e => {
743
  if (e.key === "Escape") closeViewer()
744
  if (e.key === "ArrowLeft") { e.preventDefault(); moveViewerPage(-1) }
745
  if (e.key === "ArrowRight") { e.preventDefault(); moveViewerPage(1) }
 
 
746
  })
747
 
748
  function renderPager(id, data) {
 
15
  let viewerPostId = ""
16
  let viewerPages = []
17
  let viewerPageIndex = 0
18
+ let viewerItems = []
19
+ let viewerItemIndex = 0
20
  let explorerEvents = null
21
  let explorerPage = 1
22
  let explorerDebounce = null
 
389
  const shownCounts = counts ?? exifCountsCache.get(id) ?? visibleExifCounts(id)
390
  if (shownCounts) exifCountsCache.set(id, shownCounts)
391
  const active = new Set(getExifTypes())
392
+ const codes = [...FILTER_CODES].sort((a, b) =>
393
+ (shownCounts?.[b] || 0) - (shownCounts?.[a] || 0) || FILTER_CODES.indexOf(a) - FILTER_CODES.indexOf(b)
394
+ )
395
+ const buttons = codes.map(code => {
396
  const count = shownCounts ? ` (${shownCounts[code] || 0})` : ""
397
  return `<button class="filter-btn${active.has(code) ? " active" : ""}" data-code="${code}">${esc(CATEGORY_NAMES[code])}${count}</button>`
398
  }).join("")
 
574
  }
575
 
576
  function resultCardKey(item) {
577
+ return `${item.post_id}|${item.url || ""}|${(item.pages || []).join(",")}|${(item.exif_types || []).join(",")}`
578
  }
579
 
580
  function resultBadge(item) {
581
  if (!item.scanned) return `<span class="result-badge not-scanned">not scanned</span>`
582
+ const types = item.exif_types?.length ? item.exif_types : item.exif_type ? [item.exif_type] : []
583
+ if (!types.length) return `<span class="result-badge no-exif">NIL</span>`
584
+ return `<span class="result-badges">${types.map(type =>
585
+ `<span class="result-badge exif-badge exif-${type}">${EXIF_NAMES[type] || "?"}</span>`
586
+ ).join("")}</span>`
587
  }
588
 
589
  function resultCardHtml(item) {
 
627
  }
628
 
629
  function renderResults(data) {
630
+ viewerItems = data.items
631
  const cards = data.items.map(item => {
632
  const key = resultCardKey(item)
633
  let card = resultCardCache.get(key)
 
641
  }
642
 
643
  function renderPostSearchResults(items) {
644
+ viewerItems = items
645
  const cards = items.map(item => {
646
  const key = resultCardKey(item)
647
  let card = postSearchCardCache.get(key)
 
656
 
657
  function openViewer(item) {
658
  if (!item.preview_url && !item.image_url) return
659
+ viewerItemIndex = Math.max(viewerItems.findIndex(row => row.post_id === item.post_id), 0)
660
+ setViewerItem(item)
 
661
  viewerScale = 1
662
  $("#viewer").classList.remove("hidden")
663
  $("#viewer").setAttribute("aria-hidden", "false")
664
  showViewerPage()
665
  }
666
 
667
+ function setViewerItem(item) {
668
+ viewerPostId = item.post_id
669
+ const pages = item.pages?.length ? item.pages : [item.page || 0]
670
+ const enabled = new Set(getExifTypes())
671
+ viewerPages = pages.filter(page => enabled.has(Number(item.page_types?.[page] ?? item.exif_type ?? 0)))
672
+ if (!viewerPages.length) viewerPages = pages
673
+ viewerPageIndex = Math.max(viewerPages.indexOf(item.page || 0), 0)
674
+ }
675
+
676
  function showViewerPage() {
677
  const page = viewerPages[viewerPageIndex]
678
  const pid = encodeURIComponent(viewerPostId)
 
693
  showViewerPage()
694
  }
695
 
696
+ function moveViewerPost(step) {
697
+ const next = Math.min(Math.max(viewerItemIndex + step, 0), viewerItems.length - 1)
698
+ if (next === viewerItemIndex) return
699
+ viewerItemIndex = next
700
+ setViewerItem(viewerItems[next])
701
+ showViewerPage()
702
+ }
703
+
704
  function closeViewer() {
705
  $("#viewer").classList.add("hidden")
706
  $("#viewer").setAttribute("aria-hidden", "true")
 
768
  if (e.key === "Escape") closeViewer()
769
  if (e.key === "ArrowLeft") { e.preventDefault(); moveViewerPage(-1) }
770
  if (e.key === "ArrowRight") { e.preventDefault(); moveViewerPage(1) }
771
+ if (e.key === "ArrowUp") { e.preventDefault(); moveViewerPost(-1) }
772
+ if (e.key === "ArrowDown") { e.preventDefault(); moveViewerPost(1) }
773
  })
774
 
775
  function renderPager(id, data) {
frontend/index.html CHANGED
@@ -42,7 +42,7 @@
42
  </select>
43
  </div>
44
  <label class="check-field">
45
- <input type="checkbox" id="input-deep-scan">
46
  <span>Deep scan</span>
47
  </label>
48
  <button id="btn-submit" class="btn-primary">Submit</button>
 
42
  </select>
43
  </div>
44
  <label class="check-field">
45
+ <input type="checkbox" id="input-deep-scan" checked>
46
  <span>Deep scan</span>
47
  </label>
48
  <button id="btn-submit" class="btn-primary">Submit</button>
frontend/style.css CHANGED
@@ -490,6 +490,7 @@ input:focus, select:focus, textarea:focus { outline: none; border-color: var(--a
490
  letter-spacing: .06rem;
491
  text-transform: uppercase;
492
  }
 
493
  .exif-1 { background: #18233a; color: #8bd; }
494
  .exif-2 { background: #13281d; color: var(--ok); }
495
  .exif-3 { background: #2d2515; color: var(--warn); }
 
490
  letter-spacing: .06rem;
491
  text-transform: uppercase;
492
  }
493
+ .result-badges { display: flex; gap: .25rem; flex-wrap: wrap; }
494
  .exif-1 { background: #18233a; color: #8bd; }
495
  .exif-2 { background: #13281d; color: var(--ok); }
496
  .exif-3 { background: #2d2515; color: var(--warn); }