Add deep scan page browsing
Browse files- backend/app.py +252 -17
- frontend/app.js +46 -10
- frontend/index.html +8 -2
- frontend/style.css +44 -1
backend/app.py
CHANGED
|
@@ -206,6 +206,11 @@ async def init_db():
|
|
| 206 |
{
|
| 207 |
"sql": "CREATE TABLE IF NOT EXISTS pi_scans (post_id TEXT PRIMARY KEY, url TEXT, exif_type INTEGER)"
|
| 208 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
{"sql": "CREATE TABLE IF NOT EXISTS pi_meta (key TEXT PRIMARY KEY, value TEXT)"},
|
| 210 |
{
|
| 211 |
"sql": "CREATE TABLE IF NOT EXISTS pi_cache (key TEXT PRIMARY KEY, value TEXT, updated_at INTEGER)"
|
|
@@ -227,6 +232,8 @@ async def init_db():
|
|
| 227 |
"ALTER TABLE pi_searches DROP COLUMN created_at",
|
| 228 |
"ALTER TABLE pi_searches ADD COLUMN found_exif INTEGER DEFAULT 0",
|
| 229 |
"ALTER TABLE pi_searches ADD COLUMN total_searched INTEGER DEFAULT 0",
|
|
|
|
|
|
|
| 230 |
):
|
| 231 |
try:
|
| 232 |
await turso_execute([{"sql": sql}])
|
|
@@ -355,16 +362,17 @@ def get_search_api_url(raw, keywords):
|
|
| 355 |
return f"https://www.pixiv.net/ajax/search/artworks/{encoded}?{params}"
|
| 356 |
|
| 357 |
|
| 358 |
-
async def save_search(search_id, post_ids):
|
| 359 |
if not post_ids:
|
| 360 |
return
|
| 361 |
if task_canceled(search_id):
|
| 362 |
raise TaskCanceled()
|
| 363 |
stmt = {
|
| 364 |
-
"sql": "INSERT OR REPLACE INTO pi_searches (id, post_ids) VALUES (?, ?)",
|
| 365 |
"args": [
|
| 366 |
{"type": "text", "value": search_id},
|
| 367 |
{"type": "text", "value": json.dumps(post_ids)},
|
|
|
|
| 368 |
],
|
| 369 |
}
|
| 370 |
await turso_execute([stmt])
|
|
@@ -680,6 +688,32 @@ async def get_unscanned_post_ids(search_id):
|
|
| 680 |
return [cell_value(row, 0, "") for row in first_rows(resp) if cell_value(row, 0, "")]
|
| 681 |
|
| 682 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 683 |
async def get_incomplete_search_ids():
|
| 684 |
resp = await turso_execute(
|
| 685 |
[
|
|
@@ -689,13 +723,30 @@ async def get_incomplete_search_ids():
|
|
| 689 |
"FROM pi_searches p JOIN json_each(p.post_ids) j "
|
| 690 |
"LEFT JOIN pi_scans s ON s.post_id = CAST(j.value AS TEXT) "
|
| 691 |
"GROUP BY p.id"
|
| 692 |
-
") SELECT id FROM
|
|
|
|
|
|
|
|
|
|
|
|
|
| 693 |
}
|
| 694 |
]
|
| 695 |
)
|
| 696 |
return [cell_value(row, 0, "") for row in first_rows(resp) if cell_value(row, 0, "")]
|
| 697 |
|
| 698 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 699 |
async def get_refresh_search_ids():
|
| 700 |
resp = await turso_execute(
|
| 701 |
[
|
|
@@ -800,6 +851,28 @@ def decrypt_log(value):
|
|
| 800 |
return json.loads(data.decode())
|
| 801 |
|
| 802 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 803 |
async def get_result_page(search_id, page, allowed):
|
| 804 |
where_sql, filter_args = result_filter_sql(allowed)
|
| 805 |
page = max(page, 1)
|
|
@@ -858,11 +931,13 @@ async def get_result_page(search_id, page, allowed):
|
|
| 858 |
int(cell_value(row, 0, "0")): int(cell_value(row, 1, "0"))
|
| 859 |
for row in type_rows
|
| 860 |
}
|
|
|
|
| 861 |
items = []
|
| 862 |
for row in item_rows:
|
| 863 |
pid = cell_value(row, 0, "")
|
| 864 |
url = cell_value(row, 2, "")
|
| 865 |
exif_type = cell_value(row, 3)
|
|
|
|
| 866 |
items.append(
|
| 867 |
{
|
| 868 |
"post_id": pid,
|
|
@@ -870,6 +945,8 @@ async def get_result_page(search_id, page, allowed):
|
|
| 870 |
"url": url,
|
| 871 |
"exif_type": int(exif_type) if exif_type else None,
|
| 872 |
"scanned": True,
|
|
|
|
|
|
|
| 873 |
**image_links(pid, url),
|
| 874 |
}
|
| 875 |
)
|
|
@@ -889,7 +966,7 @@ def user_search_id(user_id):
|
|
| 889 |
return f"{base26_time()}_{user_id}"
|
| 890 |
|
| 891 |
|
| 892 |
-
async def pixiv_search_live(url, pages, mode, phpsessid, search_id):
|
| 893 |
keywords = get_search_keywords(url)
|
| 894 |
api_url = get_search_api_url(url, keywords)
|
| 895 |
first_url = f"{api_url}&p=1"
|
|
@@ -936,7 +1013,7 @@ async def pixiv_search_live(url, pages, mode, phpsessid, search_id):
|
|
| 936 |
if done % 25 == 0:
|
| 937 |
if task_canceled(search_id):
|
| 938 |
raise TaskCanceled()
|
| 939 |
-
await save_search(search_id, post_ids)
|
| 940 |
finally:
|
| 941 |
for task in tasks:
|
| 942 |
if not task.done():
|
|
@@ -944,7 +1021,7 @@ async def pixiv_search_live(url, pages, mode, phpsessid, search_id):
|
|
| 944 |
await asyncio.gather(*tasks, return_exceptions=True)
|
| 945 |
if task_canceled(search_id):
|
| 946 |
raise TaskCanceled()
|
| 947 |
-
await save_search(search_id, post_ids)
|
| 948 |
return post_ids, keywords, first_url
|
| 949 |
|
| 950 |
|
|
@@ -1412,6 +1489,52 @@ async def scan_post(post_id, session, post_sem, exif_sem, img_sem):
|
|
| 1412 |
return post_id, None, None
|
| 1413 |
|
| 1414 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1415 |
async def get_exif_range(url, session, sem):
|
| 1416 |
hdrs = {
|
| 1417 |
"Referer": "https://www.pixiv.net/",
|
|
@@ -1506,10 +1629,62 @@ async def run_scan(post_ids, phpsessid, task_id=None, save_live=False, done_offs
|
|
| 1506 |
return results
|
| 1507 |
|
| 1508 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1509 |
async def save_scan_results(results):
|
| 1510 |
stmts = []
|
| 1511 |
for post_id, url, exif_type in results:
|
| 1512 |
short_url = url.replace(IMG_BASE, "", 1) if url else ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1513 |
stmts.append(
|
| 1514 |
{
|
| 1515 |
"sql": "INSERT OR REPLACE INTO pi_scans (post_id, url, exif_type) VALUES (?, ?, ?)",
|
|
@@ -1527,6 +1702,32 @@ async def save_scan_results(results):
|
|
| 1527 |
await turso_batch(stmts[i : i + 200])
|
| 1528 |
|
| 1529 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1530 |
async def get_reclass_targets(allowed):
|
| 1531 |
where_sql, args = scan_filter_sql(allowed)
|
| 1532 |
resp = await turso_execute(
|
|
@@ -1778,7 +1979,17 @@ async def bg_search_task(search_id, url, pages, mode, phpsessid):
|
|
| 1778 |
await finish_task(search_id)
|
| 1779 |
|
| 1780 |
|
| 1781 |
-
async def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1782 |
if not start_task(search_id, {
|
| 1783 |
"type": "user_search",
|
| 1784 |
"phase": "searching",
|
|
@@ -1797,7 +2008,7 @@ async def bg_user_task(search_id, user_id, phpsessid):
|
|
| 1797 |
ACTIVE_TASKS[search_id]["done"] = 1
|
| 1798 |
if not post_ids:
|
| 1799 |
return
|
| 1800 |
-
await save_search(search_id, post_ids)
|
| 1801 |
to_scan = await get_unscanned_post_ids(search_id)
|
| 1802 |
if to_scan:
|
| 1803 |
ACTIVE_TASKS[search_id].update({
|
|
@@ -1805,6 +2016,8 @@ async def bg_user_task(search_id, user_id, phpsessid):
|
|
| 1805 |
**await scan_progress_counts(search_id, len(to_scan)),
|
| 1806 |
})
|
| 1807 |
await run_scan(to_scan, phpsessid, task_id=search_id, save_live=True)
|
|
|
|
|
|
|
| 1808 |
await update_search_stats(search_id)
|
| 1809 |
await patch_explorer_cache(search_id, front=True)
|
| 1810 |
except TaskCanceled:
|
|
@@ -1815,13 +2028,16 @@ async def bg_user_task(search_id, user_id, phpsessid):
|
|
| 1815 |
await finish_task(search_id)
|
| 1816 |
|
| 1817 |
|
| 1818 |
-
async def bg_user_batch_task(jobs, phpsessid):
|
| 1819 |
await asyncio.gather(
|
| 1820 |
-
*(
|
|
|
|
|
|
|
|
|
|
| 1821 |
)
|
| 1822 |
|
| 1823 |
|
| 1824 |
-
async def bg_scan_task(search_id, post_ids, phpsessid):
|
| 1825 |
if not start_task(search_id, {
|
| 1826 |
"type": "scan",
|
| 1827 |
"phase": "scanning",
|
|
@@ -1832,6 +2048,8 @@ async def bg_scan_task(search_id, post_ids, phpsessid):
|
|
| 1832 |
completed = False
|
| 1833 |
try:
|
| 1834 |
await run_scan(post_ids, phpsessid, task_id=search_id, save_live=True)
|
|
|
|
|
|
|
| 1835 |
await update_search_stats(search_id)
|
| 1836 |
await patch_explorer_cache(search_id)
|
| 1837 |
completed = True
|
|
@@ -1846,7 +2064,9 @@ async def bg_scan_task(search_id, post_ids, phpsessid):
|
|
| 1846 |
await finish_task(search_id)
|
| 1847 |
|
| 1848 |
|
| 1849 |
-
async def bg_search_and_scan_task(
|
|
|
|
|
|
|
| 1850 |
if not start_task(search_id, {
|
| 1851 |
"type": "search+scan",
|
| 1852 |
"phase": "searching",
|
|
@@ -1857,7 +2077,9 @@ async def bg_search_and_scan_task(search_id, url, pages, mode, phpsessid):
|
|
| 1857 |
return
|
| 1858 |
completed = False
|
| 1859 |
try:
|
| 1860 |
-
await pixiv_search_live(
|
|
|
|
|
|
|
| 1861 |
to_scan = await get_unscanned_post_ids(search_id)
|
| 1862 |
if to_scan:
|
| 1863 |
ACTIVE_TASKS[search_id].update({
|
|
@@ -1865,6 +2087,8 @@ async def bg_search_and_scan_task(search_id, url, pages, mode, phpsessid):
|
|
| 1865 |
**await scan_progress_counts(search_id, len(to_scan)),
|
| 1866 |
})
|
| 1867 |
await run_scan(to_scan, phpsessid, task_id=search_id, save_live=True)
|
|
|
|
|
|
|
| 1868 |
await update_search_stats(search_id)
|
| 1869 |
await patch_explorer_cache(search_id, front=True)
|
| 1870 |
completed = True
|
|
@@ -1956,11 +2180,13 @@ class SearchRequest(BaseModel):
|
|
| 1956 |
pages: int = 30
|
| 1957 |
mode: str = "ai"
|
| 1958 |
action: str = "search"
|
|
|
|
| 1959 |
|
| 1960 |
|
| 1961 |
class UserSearchRequest(BaseModel):
|
| 1962 |
user_ids: list
|
| 1963 |
action: str = "search"
|
|
|
|
| 1964 |
|
| 1965 |
|
| 1966 |
class PostSearchRequest(BaseModel):
|
|
@@ -2031,8 +2257,11 @@ async def resume_incomplete_scans():
|
|
| 2031 |
if search_id in ACTIVE_TASKS:
|
| 2032 |
continue
|
| 2033 |
post_ids = await get_unscanned_post_ids(search_id)
|
| 2034 |
-
|
| 2035 |
-
|
|
|
|
|
|
|
|
|
|
| 2036 |
started += 1
|
| 2037 |
else:
|
| 2038 |
await update_search_stats(search_id)
|
|
@@ -2127,7 +2356,13 @@ async def submit_search(req: SearchRequest, bg: BackgroundTasks):
|
|
| 2127 |
search_id = base26_time()
|
| 2128 |
phpsessid = PHPSESSID
|
| 2129 |
bg.add_task(
|
| 2130 |
-
bg_search_and_scan_task,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2131 |
)
|
| 2132 |
return {"id": search_id, "status": "started"}
|
| 2133 |
|
|
@@ -2137,7 +2372,7 @@ async def submit_users(req: UserSearchRequest, bg: BackgroundTasks):
|
|
| 2137 |
phpsessid = PHPSESSID
|
| 2138 |
user_ids = list(dict.fromkeys(int(u) for u in req.user_ids))
|
| 2139 |
jobs = [(user_search_id(uid), uid) for uid in user_ids]
|
| 2140 |
-
bg.add_task(bg_user_batch_task, jobs, phpsessid)
|
| 2141 |
return {"ids": [search_id for search_id, _ in jobs], "status": "started"}
|
| 2142 |
|
| 2143 |
|
|
|
|
| 206 |
{
|
| 207 |
"sql": "CREATE TABLE IF NOT EXISTS pi_scans (post_id TEXT PRIMARY KEY, url TEXT, exif_type INTEGER)"
|
| 208 |
},
|
| 209 |
+
{
|
| 210 |
+
"sql": "CREATE TABLE IF NOT EXISTS pi_scan_pages ("
|
| 211 |
+
"post_id TEXT, page INTEGER, url TEXT, exif_type INTEGER, "
|
| 212 |
+
"PRIMARY KEY (post_id, page))"
|
| 213 |
+
},
|
| 214 |
{"sql": "CREATE TABLE IF NOT EXISTS pi_meta (key TEXT PRIMARY KEY, value TEXT)"},
|
| 215 |
{
|
| 216 |
"sql": "CREATE TABLE IF NOT EXISTS pi_cache (key TEXT PRIMARY KEY, value TEXT, updated_at INTEGER)"
|
|
|
|
| 232 |
"ALTER TABLE pi_searches DROP COLUMN created_at",
|
| 233 |
"ALTER TABLE pi_searches ADD COLUMN found_exif INTEGER DEFAULT 0",
|
| 234 |
"ALTER TABLE pi_searches ADD COLUMN total_searched INTEGER DEFAULT 0",
|
| 235 |
+
"ALTER TABLE pi_searches ADD COLUMN deep_scan INTEGER DEFAULT 0",
|
| 236 |
+
"ALTER TABLE pi_scans ADD COLUMN deep_scanned INTEGER DEFAULT 0",
|
| 237 |
):
|
| 238 |
try:
|
| 239 |
await turso_execute([{"sql": sql}])
|
|
|
|
| 362 |
return f"https://www.pixiv.net/ajax/search/artworks/{encoded}?{params}"
|
| 363 |
|
| 364 |
|
| 365 |
+
async def save_search(search_id, post_ids, deep_scan=False):
|
| 366 |
if not post_ids:
|
| 367 |
return
|
| 368 |
if task_canceled(search_id):
|
| 369 |
raise TaskCanceled()
|
| 370 |
stmt = {
|
| 371 |
+
"sql": "INSERT OR REPLACE INTO pi_searches (id, post_ids, deep_scan) VALUES (?, ?, ?)",
|
| 372 |
"args": [
|
| 373 |
{"type": "text", "value": search_id},
|
| 374 |
{"type": "text", "value": json.dumps(post_ids)},
|
| 375 |
+
{"type": "integer", "value": "1" if deep_scan else "0"},
|
| 376 |
],
|
| 377 |
}
|
| 378 |
await turso_execute([stmt])
|
|
|
|
| 688 |
return [cell_value(row, 0, "") for row in first_rows(resp) if cell_value(row, 0, "")]
|
| 689 |
|
| 690 |
|
| 691 |
+
async def get_deep_scan_targets(search_id):
|
| 692 |
+
resp = await turso_execute(
|
| 693 |
+
[
|
| 694 |
+
{
|
| 695 |
+
"sql": "WITH ids AS ("
|
| 696 |
+
"SELECT CAST(j.value AS TEXT) post_id, CAST(j.key AS INTEGER) pos "
|
| 697 |
+
"FROM pi_searches p, json_each(p.post_ids) j WHERE p.id = ?"
|
| 698 |
+
") SELECT ids.post_id, s.url, s.exif_type FROM ids "
|
| 699 |
+
"JOIN pi_scans s ON s.post_id = ids.post_id "
|
| 700 |
+
"WHERE s.exif_type IS NOT NULL AND COALESCE(s.deep_scanned, 0) = 0 "
|
| 701 |
+
"ORDER BY ids.pos",
|
| 702 |
+
"args": [{"type": "text", "value": search_id}],
|
| 703 |
+
}
|
| 704 |
+
]
|
| 705 |
+
)
|
| 706 |
+
return [
|
| 707 |
+
{
|
| 708 |
+
"post_id": cell_value(row, 0, ""),
|
| 709 |
+
"url": cell_value(row, 1, ""),
|
| 710 |
+
"exif_type": int(cell_value(row, 2, "0")),
|
| 711 |
+
}
|
| 712 |
+
for row in first_rows(resp)
|
| 713 |
+
if cell_value(row, 0, "")
|
| 714 |
+
]
|
| 715 |
+
|
| 716 |
+
|
| 717 |
async def get_incomplete_search_ids():
|
| 718 |
resp = await turso_execute(
|
| 719 |
[
|
|
|
|
| 723 |
"FROM pi_searches p JOIN json_each(p.post_ids) j "
|
| 724 |
"LEFT JOIN pi_scans s ON s.post_id = CAST(j.value AS TEXT) "
|
| 725 |
"GROUP BY p.id"
|
| 726 |
+
") SELECT p.id FROM pi_searches p JOIN counts ON counts.id = p.id "
|
| 727 |
+
"WHERE total > scanned OR (COALESCE(p.deep_scan, 0) = 1 AND EXISTS ("
|
| 728 |
+
"SELECT 1 FROM json_each(p.post_ids) j JOIN pi_scans s "
|
| 729 |
+
"ON s.post_id = CAST(j.value AS TEXT) WHERE s.exif_type IS NOT NULL "
|
| 730 |
+
"AND COALESCE(s.deep_scanned, 0) = 0)) ORDER BY p.id",
|
| 731 |
}
|
| 732 |
]
|
| 733 |
)
|
| 734 |
return [cell_value(row, 0, "") for row in first_rows(resp) if cell_value(row, 0, "")]
|
| 735 |
|
| 736 |
|
| 737 |
+
async def search_uses_deep_scan(search_id):
|
| 738 |
+
resp = await turso_execute(
|
| 739 |
+
[
|
| 740 |
+
{
|
| 741 |
+
"sql": "SELECT deep_scan FROM pi_searches WHERE id = ?",
|
| 742 |
+
"args": [{"type": "text", "value": search_id}],
|
| 743 |
+
}
|
| 744 |
+
]
|
| 745 |
+
)
|
| 746 |
+
rows = first_rows(resp)
|
| 747 |
+
return bool(rows and int(cell_value(rows[0], 0, "0")))
|
| 748 |
+
|
| 749 |
+
|
| 750 |
async def get_refresh_search_ids():
|
| 751 |
resp = await turso_execute(
|
| 752 |
[
|
|
|
|
| 851 |
return json.loads(data.decode())
|
| 852 |
|
| 853 |
|
| 854 |
+
async def get_result_pages(post_ids):
|
| 855 |
+
if not post_ids:
|
| 856 |
+
return {}
|
| 857 |
+
pages = {}
|
| 858 |
+
for i in range(0, len(post_ids), 500):
|
| 859 |
+
chunk = post_ids[i : i + 500]
|
| 860 |
+
marks = ",".join("?" for _ in chunk)
|
| 861 |
+
resp = await turso_execute(
|
| 862 |
+
[
|
| 863 |
+
{
|
| 864 |
+
"sql": f"SELECT post_id, page FROM pi_scan_pages WHERE post_id IN ({marks}) ORDER BY post_id, page",
|
| 865 |
+
"args": [{"type": "text", "value": str(pid)} for pid in chunk],
|
| 866 |
+
}
|
| 867 |
+
]
|
| 868 |
+
)
|
| 869 |
+
for row in first_rows(resp):
|
| 870 |
+
pages.setdefault(cell_value(row, 0, ""), []).append(
|
| 871 |
+
int(cell_value(row, 1, "0"))
|
| 872 |
+
)
|
| 873 |
+
return pages
|
| 874 |
+
|
| 875 |
+
|
| 876 |
async def get_result_page(search_id, page, allowed):
|
| 877 |
where_sql, filter_args = result_filter_sql(allowed)
|
| 878 |
page = max(page, 1)
|
|
|
|
| 931 |
int(cell_value(row, 0, "0")): int(cell_value(row, 1, "0"))
|
| 932 |
for row in type_rows
|
| 933 |
}
|
| 934 |
+
result_pages = await get_result_pages([cell_value(row, 0, "") for row in item_rows])
|
| 935 |
items = []
|
| 936 |
for row in item_rows:
|
| 937 |
pid = cell_value(row, 0, "")
|
| 938 |
url = cell_value(row, 2, "")
|
| 939 |
exif_type = cell_value(row, 3)
|
| 940 |
+
pages = result_pages.get(pid) or [page_num_from_url(url)]
|
| 941 |
items.append(
|
| 942 |
{
|
| 943 |
"post_id": pid,
|
|
|
|
| 945 |
"url": url,
|
| 946 |
"exif_type": int(exif_type) if exif_type else None,
|
| 947 |
"scanned": True,
|
| 948 |
+
"pages": pages,
|
| 949 |
+
"page_count": len(pages),
|
| 950 |
**image_links(pid, url),
|
| 951 |
}
|
| 952 |
)
|
|
|
|
| 966 |
return f"{base26_time()}_{user_id}"
|
| 967 |
|
| 968 |
|
| 969 |
+
async def pixiv_search_live(url, pages, mode, phpsessid, search_id, deep_scan=False):
|
| 970 |
keywords = get_search_keywords(url)
|
| 971 |
api_url = get_search_api_url(url, keywords)
|
| 972 |
first_url = f"{api_url}&p=1"
|
|
|
|
| 1013 |
if done % 25 == 0:
|
| 1014 |
if task_canceled(search_id):
|
| 1015 |
raise TaskCanceled()
|
| 1016 |
+
await save_search(search_id, post_ids, deep_scan)
|
| 1017 |
finally:
|
| 1018 |
for task in tasks:
|
| 1019 |
if not task.done():
|
|
|
|
| 1021 |
await asyncio.gather(*tasks, return_exceptions=True)
|
| 1022 |
if task_canceled(search_id):
|
| 1023 |
raise TaskCanceled()
|
| 1024 |
+
await save_search(search_id, post_ids, deep_scan)
|
| 1025 |
return post_ids, keywords, first_url
|
| 1026 |
|
| 1027 |
|
|
|
|
| 1489 |
return post_id, None, None
|
| 1490 |
|
| 1491 |
|
| 1492 |
+
async def scan_page_result(post_id, url, session, exif_sem, img_sem):
|
| 1493 |
+
metadata = await get_exif_range(url, session, exif_sem)
|
| 1494 |
+
exif_type = determine_exif_type(metadata)
|
| 1495 |
+
fallback = EXIF_TYPE_TO_CODE.get(exif_type) if exif_type in FALLBACK_EXIF_TYPES else None
|
| 1496 |
+
if exif_type and not fallback:
|
| 1497 |
+
return post_id, url, EXIF_TYPE_TO_CODE.get(exif_type)
|
| 1498 |
+
img_data = await fetch_image(session, url, img_sem)
|
| 1499 |
+
if not img_data:
|
| 1500 |
+
return (post_id, url, fallback) if fallback else None
|
| 1501 |
+
exif_type = determine_exif_type(parse_image_metadata(img_data))
|
| 1502 |
+
if exif_type in FALLBACK_EXIF_TYPES:
|
| 1503 |
+
fallback = fallback or EXIF_TYPE_TO_CODE.get(exif_type)
|
| 1504 |
+
elif exif_type:
|
| 1505 |
+
return post_id, url, EXIF_TYPE_TO_CODE.get(exif_type)
|
| 1506 |
+
metadata, detected = parse_stealth_metadata(img_data)
|
| 1507 |
+
exif_type = determine_exif_type(metadata)
|
| 1508 |
+
if exif_type in FALLBACK_EXIF_TYPES:
|
| 1509 |
+
fallback = fallback or EXIF_TYPE_TO_CODE.get(exif_type)
|
| 1510 |
+
elif exif_type:
|
| 1511 |
+
return post_id, url, EXIF_TYPE_TO_CODE.get(exif_type)
|
| 1512 |
+
if detected:
|
| 1513 |
+
return post_id, url, EXIF_TYPE_TO_CODE.get("stealth")
|
| 1514 |
+
return (post_id, url, fallback) if fallback else None
|
| 1515 |
+
|
| 1516 |
+
|
| 1517 |
+
async def deep_scan_post(item, session, post_sem, exif_sem, img_sem):
|
| 1518 |
+
async with post_sem:
|
| 1519 |
+
try:
|
| 1520 |
+
post_id = item["post_id"]
|
| 1521 |
+
pages = await get_post_pages(post_id, session)
|
| 1522 |
+
first_page = page_num_from_url(item["url"])
|
| 1523 |
+
results = [(post_id, first_page, item["url"], item["exif_type"])]
|
| 1524 |
+
for page in range(len(pages)):
|
| 1525 |
+
if page == first_page:
|
| 1526 |
+
continue
|
| 1527 |
+
url = pages[page]["original"]
|
| 1528 |
+
if image_ext(url) not in METADATA_EXTS:
|
| 1529 |
+
continue
|
| 1530 |
+
result = await scan_page_result(post_id, url, session, exif_sem, img_sem)
|
| 1531 |
+
if result:
|
| 1532 |
+
results.append((post_id, page, result[1], result[2]))
|
| 1533 |
+
return post_id, results
|
| 1534 |
+
except Exception:
|
| 1535 |
+
return None
|
| 1536 |
+
|
| 1537 |
+
|
| 1538 |
async def get_exif_range(url, session, sem):
|
| 1539 |
hdrs = {
|
| 1540 |
"Referer": "https://www.pixiv.net/",
|
|
|
|
| 1629 |
return results
|
| 1630 |
|
| 1631 |
|
| 1632 |
+
async def run_deep_scan(targets, phpsessid, task_id=None):
|
| 1633 |
+
post_sem = asyncio.Semaphore(POST_SCAN_LIMIT)
|
| 1634 |
+
exif_sem = asyncio.Semaphore(EXIF_RANGE_LIMIT)
|
| 1635 |
+
img_sem = asyncio.Semaphore(FULL_IMAGE_LIMIT)
|
| 1636 |
+
cookies = {"PHPSESSID": phpsessid}
|
| 1637 |
+
completed = 0
|
| 1638 |
+
pending = []
|
| 1639 |
+
async with aiohttp.ClientSession(cookies=cookies, headers=PIXIV_HEADERS) as session:
|
| 1640 |
+
tasks = {
|
| 1641 |
+
asyncio.create_task(deep_scan_post(item, session, post_sem, exif_sem, img_sem))
|
| 1642 |
+
for item in targets
|
| 1643 |
+
}
|
| 1644 |
+
created = set(tasks)
|
| 1645 |
+
if task_id:
|
| 1646 |
+
TASK_SCAN_TASKS.setdefault(task_id, set()).update(tasks)
|
| 1647 |
+
try:
|
| 1648 |
+
while tasks:
|
| 1649 |
+
if task_canceled(task_id):
|
| 1650 |
+
raise TaskCanceled()
|
| 1651 |
+
done, tasks = await asyncio.wait(
|
| 1652 |
+
tasks, timeout=1, return_when=asyncio.FIRST_COMPLETED
|
| 1653 |
+
)
|
| 1654 |
+
if task_canceled(task_id):
|
| 1655 |
+
raise TaskCanceled()
|
| 1656 |
+
for task in done:
|
| 1657 |
+
result = task.result()
|
| 1658 |
+
completed += 1
|
| 1659 |
+
if result:
|
| 1660 |
+
pending.append(result)
|
| 1661 |
+
if task_id and task_id in ACTIVE_TASKS:
|
| 1662 |
+
ACTIVE_TASKS[task_id]["done"] = completed
|
| 1663 |
+
if len(pending) >= 20:
|
| 1664 |
+
await save_deep_scan_results(pending)
|
| 1665 |
+
pending = []
|
| 1666 |
+
if pending:
|
| 1667 |
+
await save_deep_scan_results(pending)
|
| 1668 |
+
finally:
|
| 1669 |
+
for task in created:
|
| 1670 |
+
task.cancel()
|
| 1671 |
+
await asyncio.gather(*created, return_exceptions=True)
|
| 1672 |
+
if task_id and task_id in TASK_SCAN_TASKS:
|
| 1673 |
+
TASK_SCAN_TASKS[task_id].difference_update(created)
|
| 1674 |
+
if not TASK_SCAN_TASKS[task_id]:
|
| 1675 |
+
TASK_SCAN_TASKS.pop(task_id, None)
|
| 1676 |
+
|
| 1677 |
+
|
| 1678 |
async def save_scan_results(results):
|
| 1679 |
stmts = []
|
| 1680 |
for post_id, url, exif_type in results:
|
| 1681 |
short_url = url.replace(IMG_BASE, "", 1) if url else ""
|
| 1682 |
+
stmts.append(
|
| 1683 |
+
{
|
| 1684 |
+
"sql": "DELETE FROM pi_scan_pages WHERE post_id = ?",
|
| 1685 |
+
"args": [{"type": "text", "value": str(post_id)}],
|
| 1686 |
+
}
|
| 1687 |
+
)
|
| 1688 |
stmts.append(
|
| 1689 |
{
|
| 1690 |
"sql": "INSERT OR REPLACE INTO pi_scans (post_id, url, exif_type) VALUES (?, ?, ?)",
|
|
|
|
| 1702 |
await turso_batch(stmts[i : i + 200])
|
| 1703 |
|
| 1704 |
|
| 1705 |
+
async def save_deep_scan_results(results):
|
| 1706 |
+
stmts = []
|
| 1707 |
+
for post_id, pages in results:
|
| 1708 |
+
for _, page, url, exif_type in pages:
|
| 1709 |
+
short_url = url.replace(IMG_BASE, "", 1) if url else ""
|
| 1710 |
+
stmts.append(
|
| 1711 |
+
{
|
| 1712 |
+
"sql": "INSERT OR REPLACE INTO pi_scan_pages (post_id, page, url, exif_type) VALUES (?, ?, ?, ?)",
|
| 1713 |
+
"args": [
|
| 1714 |
+
{"type": "text", "value": str(post_id)},
|
| 1715 |
+
{"type": "integer", "value": str(page)},
|
| 1716 |
+
{"type": "text", "value": short_url},
|
| 1717 |
+
{"type": "integer", "value": str(exif_type)},
|
| 1718 |
+
],
|
| 1719 |
+
}
|
| 1720 |
+
)
|
| 1721 |
+
stmts.append(
|
| 1722 |
+
{
|
| 1723 |
+
"sql": "UPDATE pi_scans SET deep_scanned = 1 WHERE post_id = ?",
|
| 1724 |
+
"args": [{"type": "text", "value": str(post_id)}],
|
| 1725 |
+
}
|
| 1726 |
+
)
|
| 1727 |
+
for i in range(0, len(stmts), 200):
|
| 1728 |
+
await turso_batch(stmts[i : i + 200])
|
| 1729 |
+
|
| 1730 |
+
|
| 1731 |
async def get_reclass_targets(allowed):
|
| 1732 |
where_sql, args = scan_filter_sql(allowed)
|
| 1733 |
resp = await turso_execute(
|
|
|
|
| 1979 |
await finish_task(search_id)
|
| 1980 |
|
| 1981 |
|
| 1982 |
+
async def deep_scan_search(search_id, phpsessid):
|
| 1983 |
+
targets = await get_deep_scan_targets(search_id)
|
| 1984 |
+
if not targets:
|
| 1985 |
+
return
|
| 1986 |
+
ACTIVE_TASKS[search_id].update(
|
| 1987 |
+
{"phase": "deep scanning", "total": len(targets), "done": 0, "raw_total": 0}
|
| 1988 |
+
)
|
| 1989 |
+
await run_deep_scan(targets, phpsessid, task_id=search_id)
|
| 1990 |
+
|
| 1991 |
+
|
| 1992 |
+
async def bg_user_task(search_id, user_id, phpsessid, deep_scan=False):
|
| 1993 |
if not start_task(search_id, {
|
| 1994 |
"type": "user_search",
|
| 1995 |
"phase": "searching",
|
|
|
|
| 2008 |
ACTIVE_TASKS[search_id]["done"] = 1
|
| 2009 |
if not post_ids:
|
| 2010 |
return
|
| 2011 |
+
await save_search(search_id, post_ids, deep_scan)
|
| 2012 |
to_scan = await get_unscanned_post_ids(search_id)
|
| 2013 |
if to_scan:
|
| 2014 |
ACTIVE_TASKS[search_id].update({
|
|
|
|
| 2016 |
**await scan_progress_counts(search_id, len(to_scan)),
|
| 2017 |
})
|
| 2018 |
await run_scan(to_scan, phpsessid, task_id=search_id, save_live=True)
|
| 2019 |
+
if deep_scan:
|
| 2020 |
+
await deep_scan_search(search_id, phpsessid)
|
| 2021 |
await update_search_stats(search_id)
|
| 2022 |
await patch_explorer_cache(search_id, front=True)
|
| 2023 |
except TaskCanceled:
|
|
|
|
| 2028 |
await finish_task(search_id)
|
| 2029 |
|
| 2030 |
|
| 2031 |
+
async def bg_user_batch_task(jobs, phpsessid, deep_scan=False):
|
| 2032 |
await asyncio.gather(
|
| 2033 |
+
*(
|
| 2034 |
+
bg_user_task(search_id, user_id, phpsessid, deep_scan)
|
| 2035 |
+
for search_id, user_id in jobs
|
| 2036 |
+
)
|
| 2037 |
)
|
| 2038 |
|
| 2039 |
|
| 2040 |
+
async def bg_scan_task(search_id, post_ids, phpsessid, deep_scan=False):
|
| 2041 |
if not start_task(search_id, {
|
| 2042 |
"type": "scan",
|
| 2043 |
"phase": "scanning",
|
|
|
|
| 2048 |
completed = False
|
| 2049 |
try:
|
| 2050 |
await run_scan(post_ids, phpsessid, task_id=search_id, save_live=True)
|
| 2051 |
+
if deep_scan:
|
| 2052 |
+
await deep_scan_search(search_id, phpsessid)
|
| 2053 |
await update_search_stats(search_id)
|
| 2054 |
await patch_explorer_cache(search_id)
|
| 2055 |
completed = True
|
|
|
|
| 2064 |
await finish_task(search_id)
|
| 2065 |
|
| 2066 |
|
| 2067 |
+
async def bg_search_and_scan_task(
|
| 2068 |
+
search_id, url, pages, mode, phpsessid, deep_scan=False
|
| 2069 |
+
):
|
| 2070 |
if not start_task(search_id, {
|
| 2071 |
"type": "search+scan",
|
| 2072 |
"phase": "searching",
|
|
|
|
| 2077 |
return
|
| 2078 |
completed = False
|
| 2079 |
try:
|
| 2080 |
+
await pixiv_search_live(
|
| 2081 |
+
url, pages, mode, phpsessid, search_id, deep_scan
|
| 2082 |
+
)
|
| 2083 |
to_scan = await get_unscanned_post_ids(search_id)
|
| 2084 |
if to_scan:
|
| 2085 |
ACTIVE_TASKS[search_id].update({
|
|
|
|
| 2087 |
**await scan_progress_counts(search_id, len(to_scan)),
|
| 2088 |
})
|
| 2089 |
await run_scan(to_scan, phpsessid, task_id=search_id, save_live=True)
|
| 2090 |
+
if deep_scan:
|
| 2091 |
+
await deep_scan_search(search_id, phpsessid)
|
| 2092 |
await update_search_stats(search_id)
|
| 2093 |
await patch_explorer_cache(search_id, front=True)
|
| 2094 |
completed = True
|
|
|
|
| 2180 |
pages: int = 30
|
| 2181 |
mode: str = "ai"
|
| 2182 |
action: str = "search"
|
| 2183 |
+
deep_scan: bool = False
|
| 2184 |
|
| 2185 |
|
| 2186 |
class UserSearchRequest(BaseModel):
|
| 2187 |
user_ids: list
|
| 2188 |
action: str = "search"
|
| 2189 |
+
deep_scan: bool = False
|
| 2190 |
|
| 2191 |
|
| 2192 |
class PostSearchRequest(BaseModel):
|
|
|
|
| 2257 |
if search_id in ACTIVE_TASKS:
|
| 2258 |
continue
|
| 2259 |
post_ids = await get_unscanned_post_ids(search_id)
|
| 2260 |
+
deep_scan = await search_uses_deep_scan(search_id)
|
| 2261 |
+
if post_ids or deep_scan:
|
| 2262 |
+
asyncio.create_task(
|
| 2263 |
+
bg_scan_task(search_id, post_ids, PHPSESSID, deep_scan)
|
| 2264 |
+
)
|
| 2265 |
started += 1
|
| 2266 |
else:
|
| 2267 |
await update_search_stats(search_id)
|
|
|
|
| 2356 |
search_id = base26_time()
|
| 2357 |
phpsessid = PHPSESSID
|
| 2358 |
bg.add_task(
|
| 2359 |
+
bg_search_and_scan_task,
|
| 2360 |
+
search_id,
|
| 2361 |
+
req.url,
|
| 2362 |
+
req.pages,
|
| 2363 |
+
req.mode,
|
| 2364 |
+
phpsessid,
|
| 2365 |
+
req.deep_scan,
|
| 2366 |
)
|
| 2367 |
return {"id": search_id, "status": "started"}
|
| 2368 |
|
|
|
|
| 2372 |
phpsessid = PHPSESSID
|
| 2373 |
user_ids = list(dict.fromkeys(int(u) for u in req.user_ids))
|
| 2374 |
jobs = [(user_search_id(uid), uid) for uid in user_ids]
|
| 2375 |
+
bg.add_task(bg_user_batch_task, jobs, phpsessid, req.deep_scan)
|
| 2376 |
return {"ids": [search_id for search_id, _ in jobs], "status": "started"}
|
| 2377 |
|
| 2378 |
|
frontend/app.js
CHANGED
|
@@ -11,6 +11,9 @@ const PAGE_SIZE = 60
|
|
| 11 |
const SEARCH_PAGE_SIZE = 10
|
| 12 |
let viewerScale = 1
|
| 13 |
let viewerDrag = null
|
|
|
|
|
|
|
|
|
|
| 14 |
let explorerEvents = null
|
| 15 |
let explorerPage = 1
|
| 16 |
let explorerDebounce = null
|
|
@@ -37,6 +40,7 @@ $("#btn-submit").addEventListener("click", async () => {
|
|
| 37 |
if (!url) return
|
| 38 |
const pages = parseInt($("#input-pages").value) || 30
|
| 39 |
const mode = $("#input-mode").value
|
|
|
|
| 40 |
const status = $("#submit-status")
|
| 41 |
status.textContent = "Submitting..."
|
| 42 |
status.className = ""
|
|
@@ -57,13 +61,13 @@ $("#btn-submit").addEventListener("click", async () => {
|
|
| 57 |
resp = await fetch("/api/submit_users", {
|
| 58 |
method: "POST",
|
| 59 |
headers: { "Content-Type": "application/json" },
|
| 60 |
-
body: JSON.stringify({ user_ids: userIds })
|
| 61 |
})
|
| 62 |
} else {
|
| 63 |
resp = await fetch("/api/submit", {
|
| 64 |
method: "POST",
|
| 65 |
headers: { "Content-Type": "application/json" },
|
| 66 |
-
body: JSON.stringify({ url, pages, mode })
|
| 67 |
})
|
| 68 |
}
|
| 69 |
const data = await resp.json()
|
|
@@ -548,7 +552,7 @@ function pageSuffix(url) {
|
|
| 548 |
}
|
| 549 |
|
| 550 |
function resultCardKey(item) {
|
| 551 |
-
return `${item.post_id}|${item.url || ""}`
|
| 552 |
}
|
| 553 |
|
| 554 |
function resultBadge(item) {
|
|
@@ -563,6 +567,7 @@ function resultCardHtml(item) {
|
|
| 563 |
const pg = item.url ? pageSuffix(item.url) : ""
|
| 564 |
const label = pid + pg
|
| 565 |
const index = item.scan_index || 0
|
|
|
|
| 566 |
const download = item.download_url ? `<a class="download-link" href="${esc(item.download_url)}">Download</a>` : ""
|
| 567 |
const thumb = item.image_url
|
| 568 |
? `<button class="thumb" data-preview="${esc(item.preview_url || item.image_url)}" data-download="${esc(item.download_url || "")}"><img src="${esc(item.image_url)}" loading="lazy" alt="${esc(label)}"></button>`
|
|
@@ -574,6 +579,7 @@ function resultCardHtml(item) {
|
|
| 574 |
${download}
|
| 575 |
</div>
|
| 576 |
${resultBadge(item)}
|
|
|
|
| 577 |
<span class="result-index">${index}</span>
|
| 578 |
</div>`
|
| 579 |
}
|
|
@@ -588,7 +594,7 @@ function createResultCard(item) {
|
|
| 588 |
const thumb = card.querySelector(".thumb")
|
| 589 |
if (thumb) {
|
| 590 |
const img = thumb.querySelector("img")
|
| 591 |
-
thumb.addEventListener("click", () => openViewer(
|
| 592 |
img.addEventListener("load", () => thumb.classList.add("thumb-loaded"))
|
| 593 |
img.addEventListener("error", () => thumb.classList.add("thumb-error"))
|
| 594 |
if (img.complete) thumb.classList.add(img.naturalWidth ? "thumb-loaded" : "thumb-error")
|
|
@@ -622,21 +628,44 @@ function renderPostSearchResults(items) {
|
|
| 622 |
$("#post-search-grid").replaceChildren(...cards)
|
| 623 |
}
|
| 624 |
|
| 625 |
-
function openViewer(
|
| 626 |
-
if (!
|
|
|
|
|
|
|
|
|
|
| 627 |
viewerScale = 1
|
| 628 |
-
$("#viewer-img").src = src
|
| 629 |
-
$("#viewer-download").href = download || src
|
| 630 |
$("#viewer").classList.remove("hidden")
|
| 631 |
$("#viewer").setAttribute("aria-hidden", "false")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 632 |
applyViewerZoom()
|
| 633 |
}
|
| 634 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 635 |
function closeViewer() {
|
| 636 |
$("#viewer").classList.add("hidden")
|
| 637 |
$("#viewer").setAttribute("aria-hidden", "true")
|
| 638 |
$("#viewer-img").src = ""
|
| 639 |
$("#viewer-download").href = ""
|
|
|
|
|
|
|
| 640 |
viewerDrag = null
|
| 641 |
$("#viewer-stage").classList.remove("dragging")
|
| 642 |
}
|
|
@@ -657,6 +686,8 @@ $("#viewer-zoom-out").addEventListener("click", () => {
|
|
| 657 |
})
|
| 658 |
|
| 659 |
$("#viewer-close").addEventListener("click", closeViewer)
|
|
|
|
|
|
|
| 660 |
$("#viewer").addEventListener("click", e => { if (e.target.id === "viewer") closeViewer() })
|
| 661 |
$("#viewer-stage").addEventListener("wheel", e => {
|
| 662 |
e.preventDefault()
|
|
@@ -664,7 +695,7 @@ $("#viewer-stage").addEventListener("wheel", e => {
|
|
| 664 |
applyViewerZoom()
|
| 665 |
})
|
| 666 |
$("#viewer-stage").addEventListener("pointerdown", e => {
|
| 667 |
-
if (e.button !== 0) return
|
| 668 |
const stage = $("#viewer-stage")
|
| 669 |
viewerDrag = {
|
| 670 |
x: e.clientX,
|
|
@@ -690,7 +721,12 @@ $("#viewer-stage").addEventListener("pointercancel", () => {
|
|
| 690 |
viewerDrag = null
|
| 691 |
$("#viewer-stage").classList.remove("dragging")
|
| 692 |
})
|
| 693 |
-
window.addEventListener("keydown", e => {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 694 |
|
| 695 |
function renderPager(id, data) {
|
| 696 |
const pager = $("#pager")
|
|
|
|
| 11 |
const SEARCH_PAGE_SIZE = 10
|
| 12 |
let viewerScale = 1
|
| 13 |
let viewerDrag = null
|
| 14 |
+
let viewerPostId = ""
|
| 15 |
+
let viewerPages = []
|
| 16 |
+
let viewerPageIndex = 0
|
| 17 |
let explorerEvents = null
|
| 18 |
let explorerPage = 1
|
| 19 |
let explorerDebounce = null
|
|
|
|
| 40 |
if (!url) return
|
| 41 |
const pages = parseInt($("#input-pages").value) || 30
|
| 42 |
const mode = $("#input-mode").value
|
| 43 |
+
const deepScan = $("#input-deep-scan").checked
|
| 44 |
const status = $("#submit-status")
|
| 45 |
status.textContent = "Submitting..."
|
| 46 |
status.className = ""
|
|
|
|
| 61 |
resp = await fetch("/api/submit_users", {
|
| 62 |
method: "POST",
|
| 63 |
headers: { "Content-Type": "application/json" },
|
| 64 |
+
body: JSON.stringify({ user_ids: userIds, deep_scan: deepScan })
|
| 65 |
})
|
| 66 |
} else {
|
| 67 |
resp = await fetch("/api/submit", {
|
| 68 |
method: "POST",
|
| 69 |
headers: { "Content-Type": "application/json" },
|
| 70 |
+
body: JSON.stringify({ url, pages, mode, deep_scan: deepScan })
|
| 71 |
})
|
| 72 |
}
|
| 73 |
const data = await resp.json()
|
|
|
|
| 552 |
}
|
| 553 |
|
| 554 |
function resultCardKey(item) {
|
| 555 |
+
return `${item.post_id}|${item.url || ""}|${(item.pages || []).join(",")}`
|
| 556 |
}
|
| 557 |
|
| 558 |
function resultBadge(item) {
|
|
|
|
| 567 |
const pg = item.url ? pageSuffix(item.url) : ""
|
| 568 |
const label = pid + pg
|
| 569 |
const index = item.scan_index || 0
|
| 570 |
+
const pageCount = item.page_count || 1
|
| 571 |
const download = item.download_url ? `<a class="download-link" href="${esc(item.download_url)}">Download</a>` : ""
|
| 572 |
const thumb = item.image_url
|
| 573 |
? `<button class="thumb" data-preview="${esc(item.preview_url || item.image_url)}" data-download="${esc(item.download_url || "")}"><img src="${esc(item.image_url)}" loading="lazy" alt="${esc(label)}"></button>`
|
|
|
|
| 579 |
${download}
|
| 580 |
</div>
|
| 581 |
${resultBadge(item)}
|
| 582 |
+
${pageCount > 1 ? `<span class="result-pages">${pageCount}</span>` : ""}
|
| 583 |
<span class="result-index">${index}</span>
|
| 584 |
</div>`
|
| 585 |
}
|
|
|
|
| 594 |
const thumb = card.querySelector(".thumb")
|
| 595 |
if (thumb) {
|
| 596 |
const img = thumb.querySelector("img")
|
| 597 |
+
thumb.addEventListener("click", () => openViewer(item))
|
| 598 |
img.addEventListener("load", () => thumb.classList.add("thumb-loaded"))
|
| 599 |
img.addEventListener("error", () => thumb.classList.add("thumb-error"))
|
| 600 |
if (img.complete) thumb.classList.add(img.naturalWidth ? "thumb-loaded" : "thumb-error")
|
|
|
|
| 628 |
$("#post-search-grid").replaceChildren(...cards)
|
| 629 |
}
|
| 630 |
|
| 631 |
+
function openViewer(item) {
|
| 632 |
+
if (!item.preview_url && !item.image_url) return
|
| 633 |
+
viewerPostId = item.post_id
|
| 634 |
+
viewerPages = item.pages?.length ? item.pages : [item.page || 0]
|
| 635 |
+
viewerPageIndex = Math.max(viewerPages.indexOf(item.page || 0), 0)
|
| 636 |
viewerScale = 1
|
|
|
|
|
|
|
| 637 |
$("#viewer").classList.remove("hidden")
|
| 638 |
$("#viewer").setAttribute("aria-hidden", "false")
|
| 639 |
+
showViewerPage()
|
| 640 |
+
}
|
| 641 |
+
|
| 642 |
+
function showViewerPage() {
|
| 643 |
+
const page = viewerPages[viewerPageIndex]
|
| 644 |
+
const pid = encodeURIComponent(viewerPostId)
|
| 645 |
+
$("#viewer-img").src = `/api/i/${pid}/v?p=${page}`
|
| 646 |
+
$("#viewer-download").href = `/api/i/${pid}/o?p=${page}`
|
| 647 |
+
$("#viewer-prev").disabled = viewerPageIndex <= 0
|
| 648 |
+
$("#viewer-next").disabled = viewerPageIndex >= viewerPages.length - 1
|
| 649 |
+
$("#viewer-prev").classList.toggle("hidden", viewerPages.length <= 1)
|
| 650 |
+
$("#viewer-next").classList.toggle("hidden", viewerPages.length <= 1)
|
| 651 |
+
viewerScale = 1
|
| 652 |
applyViewerZoom()
|
| 653 |
}
|
| 654 |
|
| 655 |
+
function moveViewerPage(step) {
|
| 656 |
+
const next = Math.min(Math.max(viewerPageIndex + step, 0), viewerPages.length - 1)
|
| 657 |
+
if (next === viewerPageIndex) return
|
| 658 |
+
viewerPageIndex = next
|
| 659 |
+
showViewerPage()
|
| 660 |
+
}
|
| 661 |
+
|
| 662 |
function closeViewer() {
|
| 663 |
$("#viewer").classList.add("hidden")
|
| 664 |
$("#viewer").setAttribute("aria-hidden", "true")
|
| 665 |
$("#viewer-img").src = ""
|
| 666 |
$("#viewer-download").href = ""
|
| 667 |
+
viewerPostId = ""
|
| 668 |
+
viewerPages = []
|
| 669 |
viewerDrag = null
|
| 670 |
$("#viewer-stage").classList.remove("dragging")
|
| 671 |
}
|
|
|
|
| 686 |
})
|
| 687 |
|
| 688 |
$("#viewer-close").addEventListener("click", closeViewer)
|
| 689 |
+
$("#viewer-prev").addEventListener("click", () => moveViewerPage(-1))
|
| 690 |
+
$("#viewer-next").addEventListener("click", () => moveViewerPage(1))
|
| 691 |
$("#viewer").addEventListener("click", e => { if (e.target.id === "viewer") closeViewer() })
|
| 692 |
$("#viewer-stage").addEventListener("wheel", e => {
|
| 693 |
e.preventDefault()
|
|
|
|
| 695 |
applyViewerZoom()
|
| 696 |
})
|
| 697 |
$("#viewer-stage").addEventListener("pointerdown", e => {
|
| 698 |
+
if (e.button !== 0 || e.target.closest(".viewer-nav")) return
|
| 699 |
const stage = $("#viewer-stage")
|
| 700 |
viewerDrag = {
|
| 701 |
x: e.clientX,
|
|
|
|
| 721 |
viewerDrag = null
|
| 722 |
$("#viewer-stage").classList.remove("dragging")
|
| 723 |
})
|
| 724 |
+
window.addEventListener("keydown", e => {
|
| 725 |
+
if ($("#viewer").classList.contains("hidden")) return
|
| 726 |
+
if (e.key === "Escape") closeViewer()
|
| 727 |
+
if (e.key === "ArrowLeft") { e.preventDefault(); moveViewerPage(-1) }
|
| 728 |
+
if (e.key === "ArrowRight") { e.preventDefault(); moveViewerPage(1) }
|
| 729 |
+
})
|
| 730 |
|
| 731 |
function renderPager(id, data) {
|
| 732 |
const pager = $("#pager")
|
frontend/index.html
CHANGED
|
@@ -5,7 +5,7 @@
|
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
| 6 |
<link rel="icon" href="data:;base64,=">
|
| 7 |
<title>Pixif2</title>
|
| 8 |
-
<link rel="stylesheet" href="style.css?v=
|
| 9 |
</head>
|
| 10 |
<body>
|
| 11 |
<div class="topbar">
|
|
@@ -41,6 +41,10 @@
|
|
| 41 |
<option value="real">Real only</option>
|
| 42 |
</select>
|
| 43 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
<button id="btn-submit" class="btn-primary">Submit</button>
|
| 45 |
</div>
|
| 46 |
<div id="submit-status"></div>
|
|
@@ -106,9 +110,11 @@
|
|
| 106 |
<button id="viewer-close" class="btn-secondary">Close</button>
|
| 107 |
</div>
|
| 108 |
<div id="viewer-stage">
|
|
|
|
| 109 |
<img id="viewer-img" alt="">
|
|
|
|
| 110 |
</div>
|
| 111 |
</div>
|
| 112 |
-
<script src="app.js?v=
|
| 113 |
</body>
|
| 114 |
</html>
|
|
|
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
| 6 |
<link rel="icon" href="data:;base64,=">
|
| 7 |
<title>Pixif2</title>
|
| 8 |
+
<link rel="stylesheet" href="style.css?v=20260716-1">
|
| 9 |
</head>
|
| 10 |
<body>
|
| 11 |
<div class="topbar">
|
|
|
|
| 41 |
<option value="real">Real only</option>
|
| 42 |
</select>
|
| 43 |
</div>
|
| 44 |
+
<label class="check-field">
|
| 45 |
+
<input type="checkbox" id="input-deep-scan">
|
| 46 |
+
<span>Deep scan</span>
|
| 47 |
+
</label>
|
| 48 |
<button id="btn-submit" class="btn-primary">Submit</button>
|
| 49 |
</div>
|
| 50 |
<div id="submit-status"></div>
|
|
|
|
| 110 |
<button id="viewer-close" class="btn-secondary">Close</button>
|
| 111 |
</div>
|
| 112 |
<div id="viewer-stage">
|
| 113 |
+
<button id="viewer-prev" class="viewer-nav" aria-label="Previous result page">‹</button>
|
| 114 |
<img id="viewer-img" alt="">
|
| 115 |
+
<button id="viewer-next" class="viewer-nav" aria-label="Next result page">›</button>
|
| 116 |
</div>
|
| 117 |
</div>
|
| 118 |
+
<script src="app.js?v=20260716-1"></script>
|
| 119 |
</body>
|
| 120 |
</html>
|
frontend/style.css
CHANGED
|
@@ -91,10 +91,21 @@ h1 {
|
|
| 91 |
}
|
| 92 |
.submit-grid {
|
| 93 |
display: grid;
|
| 94 |
-
grid-template-columns: minmax(
|
| 95 |
gap: .75rem;
|
| 96 |
align-items: end;
|
| 97 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
.form-group { min-width: 0; }
|
| 99 |
.form-group label {
|
| 100 |
display: block;
|
|
@@ -487,6 +498,15 @@ input:focus, select:focus, textarea:focus { outline: none; border-color: var(--a
|
|
| 487 |
font-size: .72rem;
|
| 488 |
font-weight: 800;
|
| 489 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 490 |
#viewer {
|
| 491 |
position: fixed;
|
| 492 |
inset: 0;
|
|
@@ -506,6 +526,12 @@ input:focus, select:focus, textarea:focus { outline: none; border-color: var(--a
|
|
| 506 |
background: #080808ee;
|
| 507 |
border-bottom: 1px solid var(--line);
|
| 508 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 509 |
#viewer-zoom {
|
| 510 |
min-width: 4rem;
|
| 511 |
text-align: center;
|
|
@@ -513,6 +539,7 @@ input:focus, select:focus, textarea:focus { outline: none; border-color: var(--a
|
|
| 513 |
font-size: .85rem;
|
| 514 |
}
|
| 515 |
#viewer-stage {
|
|
|
|
| 516 |
flex: 1;
|
| 517 |
display: flex;
|
| 518 |
align-items: flex-start;
|
|
@@ -522,6 +549,22 @@ input:focus, select:focus, textarea:focus { outline: none; border-color: var(--a
|
|
| 522 |
cursor: grab;
|
| 523 |
touch-action: none;
|
| 524 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 525 |
#viewer-stage.dragging { cursor: grabbing; }
|
| 526 |
#viewer-img {
|
| 527 |
max-width: none;
|
|
|
|
| 91 |
}
|
| 92 |
.submit-grid {
|
| 93 |
display: grid;
|
| 94 |
+
grid-template-columns: minmax(14rem, 1fr) 6rem 8rem auto auto;
|
| 95 |
gap: .75rem;
|
| 96 |
align-items: end;
|
| 97 |
}
|
| 98 |
+
.check-field {
|
| 99 |
+
height: 42px;
|
| 100 |
+
display: flex;
|
| 101 |
+
align-items: center;
|
| 102 |
+
gap: .45rem;
|
| 103 |
+
color: var(--muted);
|
| 104 |
+
font-size: .78rem;
|
| 105 |
+
font-weight: 800;
|
| 106 |
+
white-space: nowrap;
|
| 107 |
+
}
|
| 108 |
+
.check-field input { width: auto; height: auto; accent-color: var(--accent); }
|
| 109 |
.form-group { min-width: 0; }
|
| 110 |
.form-group label {
|
| 111 |
display: block;
|
|
|
|
| 498 |
font-size: .72rem;
|
| 499 |
font-weight: 800;
|
| 500 |
}
|
| 501 |
+
.result-pages {
|
| 502 |
+
position: absolute;
|
| 503 |
+
left: .55rem;
|
| 504 |
+
bottom: .45rem;
|
| 505 |
+
color: var(--accent);
|
| 506 |
+
font-family: Consolas, "SFMono-Regular", monospace;
|
| 507 |
+
font-size: .72rem;
|
| 508 |
+
font-weight: 800;
|
| 509 |
+
}
|
| 510 |
#viewer {
|
| 511 |
position: fixed;
|
| 512 |
inset: 0;
|
|
|
|
| 526 |
background: #080808ee;
|
| 527 |
border-bottom: 1px solid var(--line);
|
| 528 |
}
|
| 529 |
+
#viewer-download {
|
| 530 |
+
display: inline-flex;
|
| 531 |
+
align-items: center;
|
| 532 |
+
padding-top: .12rem;
|
| 533 |
+
line-height: 1;
|
| 534 |
+
}
|
| 535 |
#viewer-zoom {
|
| 536 |
min-width: 4rem;
|
| 537 |
text-align: center;
|
|
|
|
| 539 |
font-size: .85rem;
|
| 540 |
}
|
| 541 |
#viewer-stage {
|
| 542 |
+
position: relative;
|
| 543 |
flex: 1;
|
| 544 |
display: flex;
|
| 545 |
align-items: flex-start;
|
|
|
|
| 549 |
cursor: grab;
|
| 550 |
touch-action: none;
|
| 551 |
}
|
| 552 |
+
.viewer-nav {
|
| 553 |
+
position: fixed;
|
| 554 |
+
top: 50%;
|
| 555 |
+
z-index: 1;
|
| 556 |
+
width: 46px;
|
| 557 |
+
height: 64px;
|
| 558 |
+
background: #080808dd;
|
| 559 |
+
border: 1px solid var(--line);
|
| 560 |
+
color: var(--text);
|
| 561 |
+
font-size: 2rem;
|
| 562 |
+
cursor: pointer;
|
| 563 |
+
}
|
| 564 |
+
.viewer-nav:hover { color: var(--accent); border-color: var(--accent); }
|
| 565 |
+
.viewer-nav:disabled { opacity: .3; cursor: default; }
|
| 566 |
+
#viewer-prev { left: 1rem; }
|
| 567 |
+
#viewer-next { right: 1rem; }
|
| 568 |
#viewer-stage.dragging { cursor: grabbing; }
|
| 569 |
#viewer-img {
|
| 570 |
max-width: none;
|