Neon-tech commited on
Commit
dc5edcb
·
verified ·
1 Parent(s): 6045381

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +71 -45
app.py CHANGED
@@ -1,49 +1,75 @@
1
- import os
2
- import gc
3
- from pathlib import Path
 
 
 
4
 
5
- TOK_DIR = "/data/tokenized"
6
- OUT_DIR = "/data/ready"
 
 
 
7
 
8
- CRAWL_MAP = {
9
- "2025-05": "fw-00001-of-00006.bin",
10
- "2025-08": "fw-00002-of-00006.bin",
11
- "2025-13": "fw-00003-of-00006.bin",
12
- "2025-18": "fw-00004-of-00006.bin",
13
- "2025-21": "fw-00005-of-00006.bin",
14
- "2025-28": "fw-00006-of-00006.bin",
15
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
 
17
- os.makedirs(OUT_DIR, exist_ok=True)
18
- print("Starting merge...")
19
- print(f"TOK_DIR sample: {list(Path(TOK_DIR).iterdir())[:5]}")
 
 
 
 
 
 
 
 
 
 
 
 
20
 
21
- index = {}
22
-
23
- for crawl_id, out_name in CRAWL_MAP.items():
24
- out_path = Path(OUT_DIR) / out_name
25
- if out_path.exists():
26
- print(f" SKIP {out_name}")
27
- continue
28
-
29
- shards = sorted(Path(TOK_DIR).glob(f"cc{crawl_id}_*.bin"))
30
- print(f" {crawl_id}: {len(shards)} shards")
31
- if not shards:
32
- continue
33
-
34
- total_tokens = 0
35
- with open(out_path, "wb") as out_f:
36
- for shard in shards:
37
- data = shard.read_bytes()
38
- out_f.write(data)
39
- total_tokens += len(data) // 2
40
- shard.unlink()
41
- del data
42
- gc.collect()
43
- print(f" ✓ {shard.name}")
44
-
45
- index[out_name] = {"crawl": crawl_id, "total_tokens": total_tokens}
46
- print(f" ✓ {out_name} | {total_tokens:,} tokens")
47
-
48
- Path("/data/index.json").write_text(__import__("json").dumps(index, indent=2))
49
- print("Done")
 
1
+ from fastapi import FastAPI
2
+ from fastapi.responses import FileResponse
3
+ import os, requests, time, subprocess
4
+ from bs4 import BeautifulSoup
5
+ from PIL import Image
6
+ from urllib.parse import quote
7
 
8
+ app = FastAPI()
9
+ BASE = "https://tcbscanmanga.com"
10
+ OUT = "data"
11
+ os.makedirs(OUT, exist_ok=True)
12
+ headers = {'User-Agent': 'Mozilla/5.0'}
13
 
14
+ @app.get("/search")
15
+ def search(q: str):
16
+ r = requests.get(f"{BASE}/search?search={quote(q)}", headers=headers)
17
+ soup = BeautifulSoup(r.content, 'html.parser')
18
+ results = []
19
+ for item in soup.select('.c-tabs-item__content'):
20
+ a = item.select_one('.post-title a')
21
+ if not a: continue
22
+ href = a.get('href', '')
23
+ title = a.get_text(strip=True)
24
+ slug = href.rstrip('/').split('/')[-1]
25
+ chap = item.select_one('.chapter a')
26
+ date = item.select_one('.post-on .font-meta')
27
+ rating = item.select_one('.score')
28
+ results.append({
29
+ "title": title,
30
+ "slug": slug,
31
+ "url": href,
32
+ "latest": chap.get_text(strip=True) if chap else None,
33
+ "date": date.get_text(strip=True) if date else None,
34
+ "rating": rating.get_text(strip=True) if rating else None
35
+ })
36
+ return results
37
 
38
+ @app.get("/pages/{slug}/{chapter}")
39
+ def pages(slug: str, chapter: str):
40
+ url = f"{BASE}/manga/{slug}/chapter-{chapter}"
41
+ r = requests.get(url, headers=headers, timeout=10)
42
+ r.raise_for_status()
43
+ soup = BeautifulSoup(r.content, 'html.parser')
44
+ imgs = soup.find_all('img', {'class': 'wp-manga-chapter-img'})
45
+ links = []
46
+ for img in imgs:
47
+ src = img.get('src') or img.get('data-src')
48
+ if not src: continue
49
+ if not src.startswith('http'):
50
+ src = BASE + src
51
+ links.append(src)
52
+ return {"slug": slug, "chapter": chapter, "pages": links}
53
 
54
+ @app.get("/download/{slug}/{chapter}")
55
+ def download(slug: str, chapter: str):
56
+ data = pages(slug, chapter)
57
+ if not data["pages"]:
58
+ return {"error": "no images"}
59
+ ch_dir = f"{OUT}/{slug}/ch{chapter}"
60
+ os.makedirs(ch_dir, exist_ok=True)
61
+ paths = []
62
+ for i, src in enumerate(data["pages"], 1):
63
+ ir = requests.get(src, headers=headers, timeout=10)
64
+ p = f"{ch_dir}/{i:03d}.png"
65
+ open(p, 'wb').write(ir.content)
66
+ paths.append(p)
67
+ time.sleep(0.2)
68
+ pil = [Image.open(p).convert('RGB') for p in paths]
69
+ pdf = f"{OUT}/{slug}_ch{chapter}.pdf"
70
+ pil[0].save(pdf, save_all=True, append_images=pil[1:])
71
+ z = pdf.replace('.pdf', '.7z')
72
+ subprocess.run(['7z', 'a', '-t7z', '-mx=9', z, pdf], check=True)
73
+ os.remove(pdf)
74
+ subprocess.run(['rm', '-rf', ch_dir], check=False)
75
+ return FileResponse(z, filename=f"{slug}_ch{chapter}.7z")