Spaces:
Runtime error
Runtime error
Download app.py from superhumania/lightweight-cil-proxy: direct link, hf CLI and curl.
- Browser
- Download file 13.4 kB
-
https://huggingface.co/spaces/superhumania/lightweight-cil-proxy/resolve/main/app.py
- Command line
-
hf download hf://spaces/superhumania/lightweight-cil-proxy/app.py
-
curl -L -o app.py https://huggingface.co/spaces/superhumania/lightweight-cil-proxy/resolve/main/app.py
13.4 kB
| import base64 | |
| import hashlib | |
| import io | |
| import json | |
| import os | |
| import tempfile | |
| import uuid | |
| from typing import Optional | |
| from fastapi import FastAPI, Depends, HTTPException, Header, Response | |
| from pydantic import BaseModel | |
| from huggingface_hub import ( | |
| batch_bucket_files, | |
| download_bucket_files, | |
| list_bucket_tree, | |
| ) | |
| from PIL import Image | |
| Image.MAX_IMAGE_PIXELS = 25_000_000 | |
| # --------------------------------------------------------------------------- | |
| # Configuration | |
| # --------------------------------------------------------------------------- | |
| BUCKET_ID = "superhumania/lightweight" | |
| HF_TOKEN = os.environ["HF_TOKEN"] | |
| PROXY_SECRET = os.environ["HF_PROXY_SECRET"] | |
| MAGIC_BYTES = { | |
| b"\x89PNG": "image/png", | |
| b"\xff\xd8\xff": "image/jpeg", | |
| b"RIFF": "image/webp", | |
| b"GIF8": "image/gif", | |
| } | |
| def _validate_magic_bytes(raw: bytes) -> bool: | |
| """Check that raw bytes start with a known image magic signature.""" | |
| for sig in MAGIC_BYTES: | |
| if raw[: len(sig)] == sig: | |
| return True | |
| return False | |
| app = FastAPI() | |
| # --------------------------------------------------------------------------- | |
| # Auth middleware | |
| # --------------------------------------------------------------------------- | |
| async def verify_token(authorization: str = Header(...)): | |
| """Verify Bearer token matches HF_PROXY_SECRET.""" | |
| if not authorization.startswith("Bearer "): | |
| raise HTTPException(401, "Missing bearer token") | |
| if authorization[7:] != PROXY_SECRET: | |
| raise HTTPException(401, "Invalid token") | |
| # --------------------------------------------------------------------------- | |
| # Pydantic request models | |
| # --------------------------------------------------------------------------- | |
| class UploadRequest(BaseModel): | |
| path: str | |
| content: str | |
| content_hash: Optional[str] = None | |
| class DownloadRequest(BaseModel): | |
| path: str | |
| class BatchOperation(BaseModel): | |
| action: str # only "upload" supported | |
| path: str | |
| content: str | |
| content_hash: Optional[str] = None | |
| class BatchRequest(BaseModel): | |
| operations: list[BatchOperation] | |
| class ListRequest(BaseModel): | |
| prefix: str | |
| class DeleteRequest(BaseModel): | |
| paths: list[str] | |
| class ImageRequest(BaseModel): | |
| session_id: str | |
| image_data: str | |
| media_type: str | |
| # --------------------------------------------------------------------------- | |
| # Hash index helpers (content-hash deduplication per PROXY-06) | |
| # --------------------------------------------------------------------------- | |
| def _hash_index_path(prefix: str) -> str: | |
| """Return the bucket path for a hash index file.""" | |
| return f"_hashes/{prefix}.json" | |
| def _load_hash_index(prefix: str) -> dict: | |
| """Download and parse the hash index for *prefix* from the bucket. | |
| Returns an empty dict if the index file does not exist yet. | |
| """ | |
| bucket_path = _hash_index_path(prefix) | |
| tmp = None | |
| try: | |
| tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json") | |
| tmp.close() | |
| download_bucket_files( | |
| BUCKET_ID, | |
| files=[(bucket_path, tmp.name)], | |
| token=HF_TOKEN, | |
| ) | |
| with open(tmp.name, "r", encoding="utf-8") as f: | |
| return json.load(f) | |
| except Exception: | |
| # File does not exist or is corrupted -- start fresh | |
| return {} | |
| finally: | |
| if tmp is not None and os.path.exists(tmp.name): | |
| os.unlink(tmp.name) | |
| def _save_hash_index(prefix: str, index: dict) -> None: | |
| """Serialize *index* to JSON and upload to the bucket.""" | |
| bucket_path = _hash_index_path(prefix) | |
| data = json.dumps(index, separators=(",", ":")).encode("utf-8") | |
| try: | |
| batch_bucket_files( | |
| BUCKET_ID, | |
| add=[(data, bucket_path)], | |
| token=HF_TOKEN, | |
| ) | |
| except Exception: | |
| # Non-critical -- dedup will just re-upload next time | |
| pass | |
| def _get_prefix(path: str) -> str: | |
| """Extract the first path segment (e.g. 'sessions' from 'sessions/u/123/file').""" | |
| return path.split("/")[0] if "/" in path else "default" | |
| # --------------------------------------------------------------------------- | |
| # Endpoint 1: GET /health (no auth -- used by keep-warm cron) | |
| # --------------------------------------------------------------------------- | |
| async def health(): | |
| return {"status": "ok", "bucket": BUCKET_ID} | |
| # --------------------------------------------------------------------------- | |
| # Endpoint 2: POST /upload (authed, per PROXY-01) | |
| # --------------------------------------------------------------------------- | |
| async def upload_file(request: UploadRequest): | |
| prefix = _get_prefix(request.path) | |
| # Content-hash dedup (PROXY-06) | |
| if request.content_hash: | |
| index = _load_hash_index(prefix) | |
| if index.get(request.path) == request.content_hash: | |
| return {"ok": True, "path": request.path, "skipped": True} | |
| try: | |
| batch_bucket_files( | |
| BUCKET_ID, | |
| add=[(request.content.encode("utf-8"), request.path)], | |
| token=HF_TOKEN, | |
| ) | |
| except Exception as e: | |
| raise HTTPException(500, f"Upload failed: {e}") | |
| # Update hash index after successful upload | |
| if request.content_hash: | |
| index = _load_hash_index(prefix) | |
| index[request.path] = request.content_hash | |
| _save_hash_index(prefix, index) | |
| return {"ok": True, "path": request.path, "skipped": False} | |
| # --------------------------------------------------------------------------- | |
| # Endpoint 3: POST /download (authed, per PROXY-01) | |
| # --------------------------------------------------------------------------- | |
| async def download_file(request: DownloadRequest): | |
| tmp_path = None | |
| try: | |
| tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".tmp") | |
| tmp_path = tmp.name | |
| tmp.close() | |
| download_bucket_files( | |
| BUCKET_ID, | |
| files=[(request.path, tmp_path)], | |
| token=HF_TOKEN, | |
| ) | |
| with open(tmp_path, "r", encoding="utf-8") as f: | |
| content = f.read() | |
| return {"ok": True, "content": content} | |
| except Exception as e: | |
| raise HTTPException(404, f"Download failed: {e}") | |
| finally: | |
| if tmp_path is not None and os.path.exists(tmp_path): | |
| os.unlink(tmp_path) | |
| # --------------------------------------------------------------------------- | |
| # Endpoint 4: POST /batch (authed, per PROXY-05) | |
| # --------------------------------------------------------------------------- | |
| async def batch_operations(request: BatchRequest): | |
| add_files: list[tuple[bytes, str]] = [] | |
| skipped_count = 0 | |
| # Group by prefix for hash-index lookups | |
| prefix_indexes: dict[str, dict] = {} | |
| for op in request.operations: | |
| if op.action != "upload": | |
| continue | |
| prefix = _get_prefix(op.path) | |
| # Content-hash dedup (PROXY-06) | |
| if op.content_hash: | |
| if prefix not in prefix_indexes: | |
| prefix_indexes[prefix] = _load_hash_index(prefix) | |
| if prefix_indexes[prefix].get(op.path) == op.content_hash: | |
| skipped_count += 1 | |
| continue | |
| add_files.append((op.content.encode("utf-8"), op.path)) | |
| if add_files: | |
| try: | |
| batch_bucket_files( | |
| BUCKET_ID, | |
| add=add_files, | |
| token=HF_TOKEN, | |
| ) | |
| except Exception as e: | |
| raise HTTPException(500, f"Batch upload failed: {e}") | |
| # Update hash indexes for all uploaded files | |
| updated_prefixes: set[str] = set() | |
| for op in request.operations: | |
| if op.action != "upload" or not op.content_hash: | |
| continue | |
| prefix = _get_prefix(op.path) | |
| # Only update if the file was actually uploaded (not skipped) | |
| if prefix not in prefix_indexes: | |
| prefix_indexes[prefix] = _load_hash_index(prefix) | |
| was_skipped = prefix_indexes[prefix].get(op.path) == op.content_hash | |
| if not was_skipped: | |
| prefix_indexes[prefix][op.path] = op.content_hash | |
| updated_prefixes.add(prefix) | |
| for prefix in updated_prefixes: | |
| _save_hash_index(prefix, prefix_indexes[prefix]) | |
| return {"ok": True, "uploaded": len(add_files), "skipped": skipped_count} | |
| # --------------------------------------------------------------------------- | |
| # Endpoint 5: POST /list (authed) | |
| # --------------------------------------------------------------------------- | |
| async def list_files(request: ListRequest): | |
| try: | |
| items = list( | |
| list_bucket_tree( | |
| BUCKET_ID, | |
| prefix=request.prefix, | |
| recursive=True, | |
| token=HF_TOKEN, | |
| ) | |
| ) | |
| files = [ | |
| {"path": item.path, "size": item.size} | |
| for item in items | |
| if item.type == "file" | |
| ] | |
| return {"ok": True, "files": files} | |
| except Exception as e: | |
| raise HTTPException(500, f"List failed: {e}") | |
| # --------------------------------------------------------------------------- | |
| # Endpoint 6: POST /delete (authed) | |
| # --------------------------------------------------------------------------- | |
| async def delete_files(request: DeleteRequest): | |
| try: | |
| batch_bucket_files( | |
| BUCKET_ID, | |
| delete=request.paths, | |
| token=HF_TOKEN, | |
| ) | |
| return {"ok": True, "deleted": len(request.paths)} | |
| except Exception as e: | |
| raise HTTPException(500, f"Delete failed: {e}") | |
| # --------------------------------------------------------------------------- | |
| # Endpoint 7: POST /image (authed, per CIO-03) | |
| # --------------------------------------------------------------------------- | |
| async def upload_image(request: ImageRequest): | |
| """Accept base64 image, validate, strip EXIF, convert to WebP, store in Bucket.""" | |
| raw = base64.b64decode(request.image_data) | |
| if len(raw) > 20_000_000: | |
| raise HTTPException(400, "Image too large: max 20MB") | |
| if not _validate_magic_bytes(raw): | |
| raise HTTPException( | |
| 400, "Invalid image format: only PNG, JPEG, WebP, GIF allowed" | |
| ) | |
| try: | |
| img = Image.open(io.BytesIO(raw)) | |
| except Exception as e: | |
| raise HTTPException(400, f"Cannot decode image: {e}") | |
| original_size_bytes = len(raw) | |
| # Strip EXIF metadata | |
| img.info.pop("exif", None) | |
| if img.mode not in ("RGB", "RGBA"): | |
| img = img.convert("RGB") | |
| # Resize if larger than 1920px on longest edge (preserve aspect ratio) | |
| img.thumbnail((1920, 1920), Image.LANCZOS) | |
| buf = io.BytesIO() | |
| img.save(buf, format="WEBP", quality=80, exif=b"") | |
| webp_bytes = buf.getvalue() | |
| final_w, final_h = img.size | |
| filename = f"images/{request.session_id}/{uuid.uuid4()}.webp" | |
| try: | |
| batch_bucket_files(BUCKET_ID, add=[(webp_bytes, filename)], token=HF_TOKEN) | |
| except Exception as e: | |
| raise HTTPException(500, f"Bucket upload failed: {e}") | |
| serve_url = f"/image/{filename}" | |
| return { | |
| "ok": True, | |
| "serve_url": serve_url, | |
| "width": final_w, | |
| "height": final_h, | |
| "size_bytes": len(webp_bytes), | |
| "original_size_bytes": original_size_bytes, | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Endpoint 8: GET /image/{path:path} (NO auth — LLMs fetch without Bearer) | |
| # --------------------------------------------------------------------------- | |
| async def serve_image(path: str): | |
| """Serve raw image bytes from Bucket. Public, no auth required.""" | |
| tmp_path = None | |
| try: | |
| tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".img") | |
| tmp_path = tmp.name | |
| tmp.close() | |
| download_bucket_files( | |
| BUCKET_ID, files=[(path, tmp_path)], token=HF_TOKEN | |
| ) | |
| with open(tmp_path, "rb") as f: | |
| file_bytes = f.read() | |
| if path.endswith(".webp"): | |
| content_type = "image/webp" | |
| elif path.endswith(".png"): | |
| content_type = "image/png" | |
| elif path.endswith((".jpg", ".jpeg")): | |
| content_type = "image/jpeg" | |
| else: | |
| content_type = "application/octet-stream" | |
| return Response( | |
| content=file_bytes, | |
| media_type=content_type, | |
| headers={"Cache-Control": "public, max-age=86400"}, | |
| ) | |
| except Exception: | |
| raise HTTPException(404, "Image not found") | |
| finally: | |
| if tmp_path and os.path.exists(tmp_path): | |
| os.unlink(tmp_path) | |