Spaces:
Running
Running
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache — ProcessPoolExecutor hangs on cpu-basic
d7013e6 Download src/services/upload_service.py from tisaconundrum/grassmann-cache: direct link, hf CLI and curl.
- Browser
- Download file 5.86 kB
-
https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/src/services/upload_service.py
- Command line
-
hf download hf://spaces/tisaconundrum/grassmann-cache/src/services/upload_service.py
-
curl -L -o upload_service.py https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/src/services/upload_service.py
5.86 kB
| """ | |
| UploadThing upload service — uploads files directly via signed URLs. | |
| Generates file keys locally (Sqids + djb2), signs the upload URL with HMAC-SHA256, | |
| and PUTs directly to the regional ingest endpoint. No server-side API call needed. | |
| Requires UPLOADTHING_TOKEN env var (base64-encoded JSON with apiKey, appId, regions). | |
| """ | |
| import base64 | |
| import hashlib | |
| import hmac | |
| import json | |
| import math | |
| import os | |
| import time | |
| import urllib.error | |
| import urllib.request | |
| import uuid | |
| from sqids import Sqids | |
| from sqids.constants import DEFAULT_ALPHABET | |
| def _djb2(s: str) -> int: | |
| """DJB2 hash matching UploadThing's JS implementation.""" | |
| h = 5381 | |
| for ch in reversed(s): | |
| h = (h * 33) ^ ord(ch) | |
| h &= 0xFFFFFFFF | |
| h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000) | |
| if h >= 0x80000000: | |
| h -= 0x100000000 | |
| return h | |
| def _shuffle(input_str: str, seed: str) -> str: | |
| """Shuffle alphabet deterministically using seed (matches UploadThing JS).""" | |
| chars = list(input_str) | |
| seed_num = _djb2(seed) | |
| for i in range(len(chars)): | |
| j = int(math.fmod(math.fmod(seed_num, i + 1) + i, len(chars))) | |
| chars[i], chars[j] = chars[j], chars[i] | |
| return "".join(chars) | |
| def _generate_file_key(app_id: str) -> str: | |
| """Generate a unique UploadThing file key using Sqids (matches JS SDK).""" | |
| alphabet = _shuffle(DEFAULT_ALPHABET, app_id) | |
| sqids = Sqids(alphabet=alphabet, min_length=12) | |
| encoded_app_id = sqids.encode([abs(_djb2(app_id))]) | |
| file_seed = base64.urlsafe_b64encode(uuid.uuid4().bytes).decode().rstrip("=") | |
| return encoded_app_id + file_seed | |
| def _pct_encode(s: str) -> str: | |
| """Percent-encode like JS encodeURIComponent.""" | |
| return urllib.request.quote(s, safe="") | |
| def _generate_signed_url( | |
| base_url: str, api_key: str, app_id: str, | |
| file_name: str, file_size: int, | |
| file_type: str = "application/octet-stream", | |
| ) -> str: | |
| """Generate HMAC-signed upload URL.""" | |
| expires = int(time.time() * 1000) + 3600 * 1000 # 1 hour | |
| params = ( | |
| f"?expires={expires}" | |
| f"&x-ut-identifier={_pct_encode(app_id)}" | |
| f"&x-ut-file-name={_pct_encode(file_name)}" | |
| f"&x-ut-file-size={file_size}" | |
| f"&x-ut-file-type={_pct_encode(file_type)}" | |
| f"&x-ut-content-disposition=inline" | |
| f"&x-ut-acl=public-read" | |
| ) | |
| url = base_url + params | |
| sig = hmac.new(api_key.encode(), url.encode(), hashlib.sha256).hexdigest() | |
| return f"{url}&signature=hmac-sha256={sig}" | |
| def _decode_token() -> tuple[str, str, str]: | |
| """Decode UPLOADTHING_TOKEN to get (api_key, app_id, region).""" | |
| token = os.environ.get("UPLOADTHING_TOKEN") | |
| if not token: | |
| raise RuntimeError("UPLOADTHING_TOKEN env var not set") | |
| decoded = json.loads(base64.b64decode(token)) | |
| return decoded["apiKey"], decoded["appId"], decoded.get("regions", ["sea1"])[0] | |
| def _build_multipart(file_name: str, file_data: bytes, content_type: str) -> tuple[bytes, str]: | |
| """Build multipart/form-data body manually (no requests dependency).""" | |
| boundary = uuid.uuid4().hex | |
| body = ( | |
| f"--{boundary}\r\n" | |
| f'Content-Disposition: form-data; name="file"; filename="{file_name}"\r\n' | |
| f"Content-Type: {content_type}\r\n" | |
| f"\r\n" | |
| ).encode() + file_data + f"\r\n--{boundary}--\r\n".encode() | |
| return body, f"multipart/form-data; boundary={boundary}" | |
| def upload_file(file_path: str, custom_id: str | None = None) -> str | None: | |
| """ | |
| Upload a single file to UploadThing. | |
| Returns the ufsUrl of the uploaded file, or None on failure. | |
| """ | |
| if not os.path.isfile(file_path): | |
| print(f" [upload] File not found: {file_path}", flush=True) | |
| return None | |
| api_key, app_id, region = _decode_token() | |
| file_name = os.path.basename(file_path) | |
| file_size = os.path.getsize(file_path) | |
| # Generate file key and signed URL | |
| file_key = _generate_file_key(app_id) | |
| ingest_base = f"https://{region}.ingest.uploadthing.com/{file_key}" | |
| signed_url = _generate_signed_url( | |
| ingest_base, api_key, app_id, file_name, file_size, | |
| ) | |
| print(f" [upload] Uploading {file_name} ({file_size / 1e6:.1f} MB)...", flush=True) | |
| with open(file_path, "rb") as f: | |
| file_data = f.read() | |
| # PUT with multipart form-data (matching the working Python/C# implementations) | |
| body, content_type = _build_multipart(file_name, file_data, "application/octet-stream") | |
| put_req = urllib.request.Request( | |
| signed_url, | |
| data=body, | |
| headers={ | |
| "Content-Type": content_type, | |
| "Range": "bytes=0-", | |
| }, | |
| method="PUT", | |
| ) | |
| try: | |
| with urllib.request.urlopen(put_req, timeout=300) as resp: | |
| resp_body = resp.read().decode() | |
| resp_json = json.loads(resp_body) | |
| ufs_url = resp_json.get("ufsUrl") or resp_json.get("url", "") | |
| print(f" [upload] Success: {ufs_url}", flush=True) | |
| return ufs_url | |
| except urllib.error.HTTPError as e: | |
| body_text = e.read().decode() if e.fp else "" | |
| print(f" [upload] PUT failed: {e.code} {body_text[:500]}", flush=True) | |
| return None | |
| except Exception as e: | |
| print(f" [upload] PUT error: {e}", flush=True) | |
| return None | |
| def upload_directory(dir_path: str, prefix: str = "") -> list[str]: | |
| """Upload all files in a directory. Returns list of uploaded URLs.""" | |
| urls = [] | |
| if not os.path.isdir(dir_path): | |
| print(f" [upload] Directory not found: {dir_path}", flush=True) | |
| return urls | |
| for fname in sorted(os.listdir(dir_path)): | |
| fpath = os.path.join(dir_path, fname) | |
| if os.path.isfile(fpath): | |
| url = upload_file(fpath, custom_id=f"{prefix}/{fname}" if prefix else fname) | |
| if url: | |
| urls.append(url) | |
| return urls | |