Spaces:
Running
Running
Download backend/main.py from iThors/PDF_Tokenizer: direct link, hf CLI and curl.
- Browser
- Download file 5.67 kB
-
https://huggingface.co/spaces/iThors/PDF_Tokenizer/resolve/main/backend/main.py
- Command line
-
hf download hf://spaces/iThors/PDF_Tokenizer/backend/main.py
-
curl -L -o main.py https://huggingface.co/spaces/iThors/PDF_Tokenizer/resolve/main/backend/main.py
5.67 kB
| import os | |
| import uuid | |
| import zipfile | |
| import shutil | |
| import sqlite3 | |
| from datetime import datetime, timedelta | |
| from fastapi import FastAPI, UploadFile, Form | |
| from fastapi.responses import FileResponse, JSONResponse | |
| from fastapi.middleware.cors import CORSMiddleware | |
| from fastapi.staticfiles import StaticFiles | |
| try: | |
| import tiktoken | |
| encoding = tiktoken.get_encoding("cl100k_base") | |
| except Exception: | |
| encoding = None | |
| BASE_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| DB_PATH = os.getenv("DB_PATH", "/data/usage.db") | |
| os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) | |
| UPLOAD_DIR = "/tmp/uploads" | |
| OUTPUT_DIR = "/tmp/outputs" | |
| JOB_EXPIRY_MINUTES = 15 | |
| os.makedirs(UPLOAD_DIR, exist_ok=True) | |
| os.makedirs(OUTPUT_DIR, exist_ok=True) | |
| # Initialize SQLite DB for counter | |
| def init_db(): | |
| conn = sqlite3.connect(DB_PATH) | |
| c = conn.cursor() | |
| c.execute(""" | |
| CREATE TABLE IF NOT EXISTS usage_count ( | |
| id INTEGER PRIMARY KEY, | |
| count INTEGER | |
| ) | |
| """) | |
| c.execute("INSERT OR IGNORE INTO usage_count (id, count) VALUES (1, 0)") | |
| conn.commit() | |
| conn.close() | |
| init_db() | |
| app = FastAPI() | |
| app.add_middleware( | |
| CORSMiddleware, | |
| allow_origins=["*"], | |
| allow_credentials=True, | |
| allow_methods=["*"], | |
| allow_headers=["*"], | |
| ) | |
| # SQLite-backed counter endpoints | |
| def read_db_count(): | |
| conn = sqlite3.connect(DB_PATH) | |
| c = conn.cursor() | |
| c.execute("SELECT count FROM usage_count WHERE id = 1") | |
| row = c.fetchone() | |
| conn.close() | |
| return int(row[0]) if row else 0 | |
| def write_db_count(n): | |
| conn = sqlite3.connect(DB_PATH) | |
| c = conn.cursor() | |
| c.execute("UPDATE usage_count SET count = ? WHERE id = 1", (int(n),)) | |
| conn.commit() | |
| conn.close() | |
| def get_count(): | |
| try: | |
| return {"count": read_db_count()} | |
| except Exception as e: | |
| return JSONResponse({"error": str(e)}, status_code=500) | |
| def increment_count(): | |
| try: | |
| current = read_db_count() | |
| current += 1 | |
| write_db_count(current) | |
| return {"count": current} | |
| except Exception as e: | |
| return JSONResponse({"error": str(e)}, status_code=500) | |
| # In-memory job store | |
| jobs = {} | |
| async def upload_files(files: list[UploadFile], token_size: int = Form(...)): | |
| job_id = str(uuid.uuid4()) | |
| merged_zip_path = os.path.join(OUTPUT_DIR, f"{job_id}.zip") | |
| with zipfile.ZipFile(merged_zip_path, "w") as merged_zip: | |
| for file in files: | |
| input_path = os.path.join(UPLOAD_DIR, f"{uuid.uuid4()}_{file.filename}") | |
| with open(input_path, "wb") as f: | |
| f.write(await file.read()) | |
| try: | |
| import fitz # PyMuPDF | |
| except Exception: | |
| return JSONResponse({"error": "PyMuPDF (fitz) not installed."}, status_code=500) | |
| doc = fitz.open(input_path) | |
| full_text = "".join([page.get_text() for page in doc]) | |
| doc.close() | |
| if encoding: | |
| tokens = encoding.encode(full_text) | |
| for i in range(0, len(tokens), token_size): | |
| chunk_tokens = tokens[i:i + token_size] | |
| chunk_text = encoding.decode(chunk_tokens) | |
| chunk_name = f"{os.path.splitext(file.filename)[0]}_part_{i // token_size + 1}.txt" | |
| merged_zip.writestr(chunk_name, chunk_text) | |
| else: | |
| chunk_chars = max(500, token_size * 4) | |
| for i in range(0, len(full_text), chunk_chars): | |
| chunk_text = full_text[i:i + chunk_chars] | |
| chunk_name = f"{os.path.splitext(file.filename)[0]}_part_{i // chunk_chars + 1}.txt" | |
| merged_zip.writestr(chunk_name, chunk_text) | |
| jobs[job_id] = { | |
| "filename": "multiple_files" if len(files) > 1 else files[0].filename, | |
| "zip_path": merged_zip_path, | |
| "timestamp": datetime.now() | |
| } | |
| return {"job_id": job_id, "filename": jobs[job_id]["filename"]} | |
| def download_job(job_id: str): | |
| job = jobs.get(job_id) | |
| if not job: | |
| return JSONResponse({"error": "Job not found"}, status_code=404) | |
| if datetime.now() - job["timestamp"] > timedelta(minutes=JOB_EXPIRY_MINUTES): | |
| return JSONResponse({"error": "Job expired"}, status_code=410) | |
| return FileResponse(job["zip_path"], filename=f"{os.path.splitext(job['filename'])[0]}_chunks.zip") | |
| def download_space(): | |
| repo_root = os.path.abspath(os.path.join(BASE_DIR, "..")) | |
| if repo_root in ("/", ""): | |
| repo_root = BASE_DIR | |
| tmp_zip_base = "/tmp/space_backup" | |
| tmp_zip_file = tmp_zip_base + ".zip" | |
| safe_temp_dir = "/tmp/space_export" | |
| if os.path.exists(safe_temp_dir): | |
| shutil.rmtree(safe_temp_dir) | |
| shutil.copytree(repo_root, safe_temp_dir, dirs_exist_ok=True, ignore=shutil.ignore_patterns("tmp", "outputs", "uploads", "__pycache__", ".git")) | |
| shutil.make_archive(tmp_zip_base, "zip", safe_temp_dir) | |
| return FileResponse(tmp_zip_file, filename="space_backup.zip") | |
| # Serve frontend static by copying frontend/ to backend/static | |
| possible_frontend = os.path.abspath(os.path.join(BASE_DIR, "..", "frontend")) | |
| static_target = os.path.join(BASE_DIR, "static") | |
| if os.path.isdir(possible_frontend): | |
| if os.path.exists(static_target): | |
| shutil.rmtree(static_target) | |
| shutil.copytree(possible_frontend, static_target) | |
| app.mount("/", StaticFiles(directory=static_target, html=True), name="static") | |
| else: | |
| if os.path.isdir(static_target): | |
| app.mount("/", StaticFiles(directory=static_target, html=True), name="static") | |