eng2pun / app.py
YADUUG's picture
refactor: Refactor DOCX processing to translate paragraphs instead of individual runs, simplifying logic and adding tests for formatting preservation.
bd4d14a
Raw History Blame Contribute Delete
20 kB
import asyncio
import json
import io
from fastapi import FastAPI, UploadFile, File, Header, HTTPException, Depends, Form
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import StreamingResponse, Response, FileResponse
from docx import Document
from deep_translator import GoogleTranslator
from deep_translator.exceptions import RequestError, TranslationNotFound
import random
import os
app = FastAPI()
# --- REAL FIREBASE CLIENT ---
import firebase_admin
from firebase_admin import credentials, firestore, auth
# Path to your Service Account Key (Local fallback)
CRED_PATH = "eng2pun-fc01a-firebase-adminsdk-fbsvc-4418817502.json"
try:
# 1. Try Environment Variable (For Production/Vercel)
if os.environ.get("FIREBASE_CREDENTIALS"):
# Expects the JSON content as a string
cred_dict = json.loads(os.environ.get("FIREBASE_CREDENTIALS"))
cred = credentials.Certificate(cred_dict)
print("Firebase Admin Initialized from Environment Variable")
# 2. Try Local File
elif os.path.exists(CRED_PATH):
cred = credentials.Certificate(CRED_PATH)
print("Firebase Admin Initialized from Local File")
else:
raise FileNotFoundError("No Firebase credentials found (Env or File)")
firebase_admin.initialize_app(cred)
except Exception as e:
print(f"Warning: Firebase Init Failed (might be already initialized): {e}")
class RealFirebaseClient:
def __init__(self):
self.db = firestore.client()
self.mock_tokens = {
"test-token-123": "user-1",
"user-token-456": "user-2"
}
# Ensure mock users exist in DB for testing if they don't exist
# self.ensure_mock_users()
def verify_token(self, token):
if not token or not token.startswith("Bearer "):
return None
token_str = token.split(" ")[1]
# 1. Try Mock Tokens (Transition/Dev Mode)
if token_str in self.mock_tokens:
uid = self.mock_tokens[token_str]
# Fetch from Real DB even for mock token
user_doc = self.get_user_doc(uid)
if not user_doc:
# Fallback implementation if DB validation fails for mock
return {"uid": uid, "email": "mock@example.com", "credits": 100, "isAdmin": False}
return {"uid": uid, **user_doc}
# 2. Verify Real Firebase ID Token
try:
decoded_token = auth.verify_id_token(token_str)
uid = decoded_token['uid']
# Fetch custom fields (credits) from Firestore
user_doc = self.get_user_doc(uid)
if not user_doc:
# Auto-create user doc if missing
self.db.collection('users').document(uid).set({
"email": decoded_token.get('email', ''),
"credits": 5000,
"isAdmin": False
})
user_doc = {"credits": 5000, "isAdmin": False, "email": decoded_token.get('email')}
return {"uid": uid, **user_doc}
except Exception as e:
print(f"Token Verification Failed: {e}")
return None
def get_user_doc(self, uid):
doc = self.db.collection("users").document(uid).get()
if doc.exists:
return doc.to_dict()
return None
def deduct_credits(self, uid, amount):
if uid.startswith("user-"):
return 1000 # Mock user never runs out of credits
ref = self.db.collection("users").document(uid)
# Transactional update is best, but generic update is okay for now
try:
# We want to decrement. Firestore 'increment' with negative value is atomic.
ref.update({"credits": firestore.Increment(-amount)})
# Fetch new balance
updated_doc = ref.get()
return updated_doc.to_dict().get("credits", 0)
except Exception as e:
print(f"Deduct Credits Failed: {e}")
return 0 # Fail gracefully, returning 0 credits if DB update fails
# Admin methods
def get_all_users(self):
docs = self.db.collection("users").stream()
users_list = []
for doc in docs:
users_list.append({"id": doc.id, **doc.to_dict()})
return users_list
def admin_add_credits(self, uid, amount):
ref = self.db.collection("users").document(uid)
try:
ref.update({"credits": firestore.Increment(amount)})
updated_doc = ref.get()
return {"id": uid, **updated_doc.to_dict()}
except Exception as e:
return None
# Initialize Client
firebase = RealFirebaseClient()
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
import time
# ... (Firebase init remains) ...
# Initialize Client
firebase = RealFirebaseClient()
app.add_middleware(
CORSMiddleware,
allow_origins=[
"http://localhost:8000",
"http://127.0.0.1:8000",
"https://eng2pun.vercel.app"
],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
app.mount("/static", StaticFiles(directory="static"), name="static")
async def get_current_user(authorization: str = Header(None)):
if not authorization:
raise HTTPException(status_code=401, detail="Missing Authorization Header")
user_claims = firebase.verify_token(authorization)
if not user_claims:
raise HTTPException(status_code=401, detail="Invalid Token")
return user_claims
@app.get("/")
async def read_index():
return FileResponse('index.html')
@app.get("/admin")
async def read_admin():
return FileResponse('admin.html')
class PaymentRequest(BaseModel):
amount: float
credits: int
utr: str
@app.post("/api/submit-payment")
async def submit_payment(req: PaymentRequest, user: dict = Depends(get_current_user)):
try:
payment_data = {
"uid": user['uid'],
"email": user.get('email', 'Unknown'),
"amount": req.amount,
"credits_requested": req.credits,
"utr": req.utr,
"status": "pending",
"timestamp": firestore.SERVER_TIMESTAMP
}
# Save to 'payment_requests' collection
firebase.db.collection("payment_requests").add(payment_data)
return {"status": "success", "message": "Payment request submitted"}
except Exception as e:
print(f"Payment Submit Error: {e}")
raise HTTPException(status_code=500, detail=str(e))
# Admin Endpoints (Mock)
@app.get("/api/admin/users")
async def get_users(user: dict = Depends(get_current_user)):
if not user.get('isAdmin'):
raise HTTPException(status_code=403, detail="Admin access required")
return firebase.get_all_users()
@app.post("/api/admin/add-credits")
async def add_credits(payload: dict, user: dict = Depends(get_current_user)):
if not user.get('isAdmin'):
raise HTTPException(status_code=403, detail="Admin access required")
target_id = payload.get('user_id')
amount = payload.get('amount')
updated = firebase.admin_add_credits(target_id, int(amount))
if not updated:
raise HTTPException(status_code=404, detail="User not found")
return updated
@app.get("/api/profile")
async def get_profile(user: dict = Depends(get_current_user)):
return user
# Use GoogleTranslator directly
def get_translator(source_lang='auto', dest_lang='pa'):
return GoogleTranslator(source=source_lang, target=dest_lang)
async def translate_text(text: str, source_lang='auto', dest_lang='pa', max_retries=3):
if not text or not text.strip():
return text
translator_instance = get_translator(source_lang, dest_lang)
for attempt in range(max_retries):
try:
loop = asyncio.get_event_loop()
# Using run_in_executor because deep-translator is sync
result = await loop.run_in_executor(
None,
lambda: translator_instance.translate(text)
)
# deep-translator sometimes returns Google's HTML error text as the translation
if result and ("Error 500" in result or "Error 400" in result or "That’s an error." in result):
raise Exception(f"Google Translate blocked/errored instead of translating: {result[:50]}...")
return result
except Exception as e:
print(f"Translation Error (Attempt {attempt+1}/{max_retries}): {e}")
if attempt < max_retries - 1:
# Exponential backoff with some jitter
sleep_time = (2 ** attempt) + random.uniform(0, 1)
await asyncio.sleep(sleep_time)
else:
print(f"Final Translation Failure for text: {text[:30]}...")
return None # Return None so we know it failed
# --- STREAMING LOGIC (For the Live View) ---
async def process_docx_stream(file_stream, user, source_lang='auto', dest_lang='pa'):
doc = Document(io.BytesIO(file_stream))
current_page = 1
para_count = 0
total_chars_processed = 0
uid = user['uid']
for para in doc.paragraphs:
original_text = para.text.strip()
if not original_text:
continue
# Check credits before processing paragraph
if user['credits'] <= 0:
yield json.dumps({"error": "Out of credits", "end": True}) + "\n"
break
is_graphic = False
if len(para.runs) > 0 and not original_text:
is_graphic = True
translated_text = ""
cost = 0
if is_graphic:
translated_text = "[GRAPH/DIAGRAM PRESERVED]"
original_text = "[GRAPH/DIAGRAM]"
else:
translated_result = await translate_text(original_text, source_lang=source_lang, dest_lang=dest_lang)
if translated_result is None:
# Translation failed
translated_text = original_text # Fallback
cost = 0 # Don't charge!
else:
translated_text = translated_result
cost = len(translated_text)
# Deduct Cost only if meaningful translation happened
if cost > 0:
new_balance = firebase.deduct_credits(uid, cost)
user['credits'] = new_balance # Update local copy for loop check
total_chars_processed += cost
data = {
"page": current_page,
"original": original_text,
"translated": translated_text if translated_text is not None else "[Translation Failed]",
"is_graphic": is_graphic,
"translation_failed": translated_text is None,
"remaining_credits": user['credits'] # Send back so frontend can update header
}
yield json.dumps(data) + "\n"
para_count += 1
if para_count >= 10:
current_page += 1
para_count = 0
await asyncio.sleep(0.05)
async def process_txt_stream(file_stream, user, source_lang='auto', dest_lang='pa'):
# Decode text, splitting into lines/paragraphs
text_content = file_stream.decode('utf-8', errors='replace')
lines = text_content.split('\n')
current_page = 1
para_count = 0
total_chars_processed = 0
uid = user['uid']
for line in lines:
original_text = line.strip()
if not original_text:
continue
# Check credits before processing
if user['credits'] <= 0:
yield json.dumps({"error": "Out of credits", "end": True}) + "\n"
break
translated_result = await translate_text(original_text, source_lang=source_lang, dest_lang=dest_lang)
if translated_result is None:
# Translation failed
translated_text = original_text # Fallback
cost = 0 # Don't charge!
else:
translated_text = translated_result
cost = len(translated_text)
# Deduct Cost only if meaningful translation happened
if cost > 0:
new_balance = firebase.deduct_credits(uid, cost)
user['credits'] = new_balance # Update local copy for loop check
total_chars_processed += cost
data = {
"page": current_page,
"original": original_text,
"translated": translated_text if translated_text is not None else "[Translation Failed]",
"is_graphic": False,
"translation_failed": translated_text is None,
"remaining_credits": user['credits'] # Send back so frontend can update header
}
yield json.dumps(data) + "\n"
para_count += 1
if para_count >= 15: # slightly more lines per page for txt
current_page += 1
para_count = 0
await asyncio.sleep(0.01)
# --- DOWNLOAD LOGIC (New) ---
async def process_docx_for_download(file_content, translation_list):
"""
Reads the file, replaces English text with provided translations from the list.
Preserves images by clearing text runs instead of deleting paragraph content.
Stops when translation_list is exhausted (partial download).
"""
source_stream = io.BytesIO(file_content)
doc = Document(source_stream)
trans_iter = iter(translation_list)
for para in doc.paragraphs:
original_text = para.text.strip()
if not original_text:
continue
# Logic: We only care about paragraphs that have text.
# Images usually are in paragraphs with text or empty paragraphs with runs.
# If original_text is empty, we skipped above.
# So we are looking at textual paragraphs here.
# Try to get next translation
try:
next_trans = next(trans_iter)
except StopIteration:
# No more translations available (ran out of credits or end of list)
# Stop replacing = keep original English for the rest
break
if not next_trans or next_trans == "[GRAPH/DIAGRAM PRESERVED]":
# If it's a preserved graphic marker, do nothing (keep original/image)
continue
# REPLACEMENT LOGIC WITH FORMATTING PRESERVATION
# 1. Identify runs that contain images
image_runs_indices = set()
for i, run in enumerate(para.runs):
# Check for standard drawing/image XML tags
if run.element.findall('.//w:drawing', namespaces=run.element.nsmap) or \
run.element.findall('.//w:pict', namespaces=run.element.nsmap) or \
run.element.findall('.//w:object', namespaces=run.element.nsmap):
image_runs_indices.add(i)
# 2. Extract original text runs and clear them
text_runs = []
original_lengths = []
for i, run in enumerate(para.runs):
if i in image_runs_indices:
continue # Skip image runs
run_text = run.text
if run_text:
text_runs.append(run)
original_lengths.append(len(run_text))
run.text = "" # Clear text
# 3. Distribute translation across runs
if text_runs and next_trans:
trans_words = next_trans.split(" ")
total_orig_len = sum(original_lengths)
if total_orig_len == 0:
text_runs[0].text = next_trans
else:
word_idx = 0
for i, run in enumerate(text_runs):
# Proportional allowance of words for this run
target_chars = (original_lengths[i] / total_orig_len) * len(next_trans)
current_chars = 0
run_words = []
while word_idx < len(trans_words) and (current_chars < target_chars or i == len(text_runs) - 1):
word = trans_words[word_idx]
run_words.append(word)
current_chars += len(word) + 1 # +1 for space
word_idx += 1
run.text = " ".join(run_words) + (" " if word_idx < len(trans_words) else "")
elif not text_runs and next_trans:
# If all runs were images (unlikely if para.text was not empty)
# or no runs exist, add a new one.
para.add_run(next_trans)
# Save to a new memory stream
output_stream = io.BytesIO()
doc.save(output_stream)
output_stream.seek(0)
return output_stream
async def process_txt_for_download(file_content, translation_list):
"""
Reads the text file, replaces lines with provided translations from the list.
"""
text_content = file_content.decode('utf-8', errors='replace')
lines = text_content.split('\n')
trans_iter = iter(translation_list)
output_lines = []
for line in lines:
if not line.strip():
output_lines.append(line) # preserve empty lines
continue
try:
next_trans = next(trans_iter)
output_lines.append(next_trans)
except StopIteration:
output_lines.append(line) # Ran out of translations, keep original
output_stream = io.BytesIO()
output_stream.write('\n'.join(output_lines).encode('utf-8'))
output_stream.seek(0)
return output_stream
@app.post("/translate-stream")
async def translate_document(
file: UploadFile = File(...),
source_lang: str = Form('auto'),
dest_lang: str = Form('pa'),
user: dict = Depends(get_current_user)
):
# Basic pre-check
filename_lower = file.filename.lower()
if not (filename_lower.endswith('.docx') or filename_lower.endswith('.txt')):
raise HTTPException(status_code=400, detail="Invalid file type. Only .docx and .txt files are supported.")
if user['credits'] <= 0:
raise HTTPException(status_code=402, detail="Insufficient credits")
file_content = await file.read()
if filename_lower.endswith('.docx'):
generator = process_docx_stream(file_content, user, source_lang, dest_lang)
else:
generator = process_txt_stream(file_content, user, source_lang, dest_lang)
return StreamingResponse(
generator,
media_type="application/x-ndjson"
)
@app.post("/download")
async def download_file(
file: UploadFile = File(...),
translations: str = Form(...), # JSON string of list of translated strings
user: dict = Depends(get_current_user)
):
# We trust the provided translations (since user paid for them in the stream)
# The 'translations' field is a JSON dump of a list of strings.
try:
translation_list = json.loads(translations)
except:
translation_list = []
file_content = await file.read()
filename_lower = file.filename.lower()
# Process using the provided list instead of re-translating
if filename_lower.endswith('.docx'):
output_io = await process_docx_for_download(file_content, translation_list)
media_type = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
elif filename_lower.endswith('.txt'):
output_io = await process_txt_for_download(file_content, translation_list)
media_type = "text/plain"
else:
raise HTTPException(status_code=400, detail="Unsupported file type for download")
return Response(
content=output_io.getvalue(),
media_type=media_type,
headers={"Content-Disposition": f"attachment; filename=translated_{file.filename}"}
)
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)