Spaces:
Running
Running
P0-R2 HF Bucket flag-OFF deployment
Browse files- huggingface/hf_bucket_store.py +380 -0
- huggingface/managed_auth.py +867 -0
- huggingface/requirements.txt +16 -0
- huggingface/task_executor.py +107 -19
huggingface/hf_bucket_store.py
ADDED
|
@@ -0,0 +1,380 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""Best-effort private Hugging Face Bucket storage for completed ZIP delivery.
|
| 3 |
+
|
| 4 |
+
This module is deliberately outside the paper-generation core. Bucket storage is
|
| 5 |
+
an optional recovery layer only: a Bucket error must never turn an already valid
|
| 6 |
+
local paper package into a failed generation or alter quota/accounting state.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import hashlib
|
| 11 |
+
import json
|
| 12 |
+
import logging
|
| 13 |
+
import os
|
| 14 |
+
import re
|
| 15 |
+
import tempfile
|
| 16 |
+
import zipfile
|
| 17 |
+
from dataclasses import dataclass
|
| 18 |
+
from datetime import datetime, timedelta, timezone
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
from typing import Any
|
| 21 |
+
|
| 22 |
+
_TRUE = {"1", "true", "yes", "on", "enabled"}
|
| 23 |
+
_DEFAULT_TTL_HOURS = 168
|
| 24 |
+
_DEFAULT_WARNING_GIB = 70.0
|
| 25 |
+
_DEFAULT_HARD_STOP_GIB = 80.0
|
| 26 |
+
_MAX_HARD_STOP_GIB = 80.0 # Free-only safety ceiling approved for Article Agent.
|
| 27 |
+
_GIB = 1024 ** 3
|
| 28 |
+
_TASK_RE = re.compile(r"^[A-Za-z0-9_.-]{1,160}$")
|
| 29 |
+
_BUCKET_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@dataclass(frozen=True)
|
| 33 |
+
class BucketConfig:
|
| 34 |
+
enabled: bool
|
| 35 |
+
bucket_id: str
|
| 36 |
+
token: str
|
| 37 |
+
ttl_hours: int
|
| 38 |
+
warning_bytes: int
|
| 39 |
+
hard_stop_bytes: int
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _truthy(value: Any) -> bool:
|
| 43 |
+
return str(value or "").strip().casefold() in _TRUE
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _bounded_float(name: str, default: float, low: float, high: float) -> float:
|
| 47 |
+
try:
|
| 48 |
+
value = float(os.environ.get(name, str(default)))
|
| 49 |
+
except (TypeError, ValueError):
|
| 50 |
+
value = default
|
| 51 |
+
return max(low, min(high, value))
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _bounded_int(name: str, default: int, low: int, high: int) -> int:
|
| 55 |
+
try:
|
| 56 |
+
value = int(os.environ.get(name, str(default)))
|
| 57 |
+
except (TypeError, ValueError):
|
| 58 |
+
value = default
|
| 59 |
+
return max(low, min(high, value))
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def config() -> BucketConfig:
|
| 63 |
+
enabled = _truthy(os.environ.get("SMARTPAPER_HF_ARTIFACTS_ENABLED"))
|
| 64 |
+
bucket_id = str(os.environ.get("SMARTPAPER_HF_ARTIFACT_BUCKET") or "").strip()
|
| 65 |
+
token = str(os.environ.get("SMARTPAPER_HF_BUCKET_TOKEN") or "").strip()
|
| 66 |
+
ttl_hours = _bounded_int("SMARTPAPER_HF_ARTIFACT_TTL_HOURS", _DEFAULT_TTL_HOURS, 1, _DEFAULT_TTL_HOURS)
|
| 67 |
+
requested_hard = _bounded_float(
|
| 68 |
+
"SMARTPAPER_HF_ARTIFACT_HARD_STOP_GIB", _DEFAULT_HARD_STOP_GIB, 1.0, _MAX_HARD_STOP_GIB
|
| 69 |
+
)
|
| 70 |
+
requested_warning = _bounded_float(
|
| 71 |
+
"SMARTPAPER_HF_ARTIFACT_WARNING_GIB", _DEFAULT_WARNING_GIB, 1.0, requested_hard
|
| 72 |
+
)
|
| 73 |
+
return BucketConfig(
|
| 74 |
+
enabled=enabled,
|
| 75 |
+
bucket_id=bucket_id,
|
| 76 |
+
token=token,
|
| 77 |
+
ttl_hours=ttl_hours,
|
| 78 |
+
warning_bytes=int(requested_warning * _GIB),
|
| 79 |
+
hard_stop_bytes=int(requested_hard * _GIB),
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def _new_api(token: str):
|
| 84 |
+
# huggingface_hub reads these at import time. Keep recovery bounded without
|
| 85 |
+
# importing the optional dependency when the feature is disabled.
|
| 86 |
+
os.environ.setdefault("HF_HUB_ETAG_TIMEOUT", "8")
|
| 87 |
+
os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "8")
|
| 88 |
+
from huggingface_hub import HfApi # lazy: absence must not break normal generation
|
| 89 |
+
|
| 90 |
+
return HfApi(token=token)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _validate_config(cfg: BucketConfig) -> None:
|
| 94 |
+
if not cfg.enabled:
|
| 95 |
+
raise RuntimeError("hf_bucket_disabled")
|
| 96 |
+
if not cfg.bucket_id or not _BUCKET_RE.fullmatch(cfg.bucket_id):
|
| 97 |
+
raise RuntimeError("hf_bucket_id_invalid")
|
| 98 |
+
if not cfg.token:
|
| 99 |
+
raise RuntimeError("hf_bucket_token_missing")
|
| 100 |
+
if cfg.hard_stop_bytes > int(_MAX_HARD_STOP_GIB * _GIB):
|
| 101 |
+
raise RuntimeError("hf_bucket_hard_stop_exceeds_free_safety_ceiling")
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def _trusted_key_parts(user_id: int | str, task_id: str) -> tuple[str, str]:
|
| 105 |
+
try:
|
| 106 |
+
trusted_user_id = str(int(user_id))
|
| 107 |
+
except (TypeError, ValueError) as exc:
|
| 108 |
+
raise RuntimeError("hf_bucket_user_id_invalid") from exc
|
| 109 |
+
trusted_task_id = str(task_id or "").strip()
|
| 110 |
+
if not _TASK_RE.fullmatch(trusted_task_id):
|
| 111 |
+
raise RuntimeError("hf_bucket_task_id_invalid")
|
| 112 |
+
return trusted_user_id, trusted_task_id
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def object_paths(user_id: int | str, task_id: str) -> tuple[str, str]:
|
| 116 |
+
trusted_user_id, trusted_task_id = _trusted_key_parts(user_id, task_id)
|
| 117 |
+
prefix = f"v1/users/{trusted_user_id}/tasks/{trusted_task_id}"
|
| 118 |
+
return f"{prefix}/final.zip", f"{prefix}/final.json"
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def _safe_output_filename(value: str) -> str:
|
| 122 |
+
text = str(value or "").strip()
|
| 123 |
+
name = os.path.basename(text)
|
| 124 |
+
if not name or name != text or not name.lower().endswith(".zip") or len(name) > 240:
|
| 125 |
+
raise RuntimeError("hf_bucket_output_filename_invalid")
|
| 126 |
+
return name
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _sha256_file(path: Path) -> tuple[str, int]:
|
| 130 |
+
digest = hashlib.sha256()
|
| 131 |
+
size = 0
|
| 132 |
+
with path.open("rb") as handle:
|
| 133 |
+
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
| 134 |
+
size += len(block)
|
| 135 |
+
digest.update(block)
|
| 136 |
+
return digest.hexdigest(), size
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _private_bucket_info(api, cfg: BucketConfig):
|
| 140 |
+
info = api.bucket_info(cfg.bucket_id)
|
| 141 |
+
if getattr(info, "private", None) is not True:
|
| 142 |
+
raise RuntimeError("hf_bucket_not_private")
|
| 143 |
+
return info
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def _file_map(api, bucket_id: str, paths: list[str]) -> dict[str, Any]:
|
| 147 |
+
return {str(item.path): item for item in api.get_bucket_paths_info(bucket_id, paths)}
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def _metadata_bytes(*, user_id: str, task_id: str, filename: str, sha256: str, size_bytes: int, uploaded_at: datetime) -> bytes:
|
| 151 |
+
payload = {
|
| 152 |
+
"schema_version": "articleagent-hf-bucket-v1",
|
| 153 |
+
"user_id": user_id,
|
| 154 |
+
"task_id": task_id,
|
| 155 |
+
"output_filename": filename,
|
| 156 |
+
"sha256": sha256,
|
| 157 |
+
"size_bytes": int(size_bytes),
|
| 158 |
+
"uploaded_at": uploaded_at.astimezone(timezone.utc).isoformat(),
|
| 159 |
+
}
|
| 160 |
+
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def _parse_uploaded_at(value: Any) -> datetime:
|
| 164 |
+
text = str(value or "").strip()
|
| 165 |
+
if not text:
|
| 166 |
+
raise RuntimeError("hf_bucket_metadata_time_missing")
|
| 167 |
+
try:
|
| 168 |
+
observed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
| 169 |
+
except ValueError as exc:
|
| 170 |
+
raise RuntimeError("hf_bucket_metadata_time_invalid") from exc
|
| 171 |
+
if observed.tzinfo is None:
|
| 172 |
+
observed = observed.replace(tzinfo=timezone.utc)
|
| 173 |
+
return observed.astimezone(timezone.utc)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def _download_metadata(api, bucket_id: str, remote_path: str) -> dict[str, Any]:
|
| 177 |
+
with tempfile.TemporaryDirectory(prefix="sp_hf_meta_") as temp_dir:
|
| 178 |
+
target = Path(temp_dir) / "final.json"
|
| 179 |
+
api.download_bucket_files(bucket_id, [(remote_path, target)], raise_on_missing_files=True)
|
| 180 |
+
raw = target.read_bytes()
|
| 181 |
+
if len(raw) > 64 * 1024:
|
| 182 |
+
raise RuntimeError("hf_bucket_metadata_too_large")
|
| 183 |
+
value = json.loads(raw.decode("utf-8"))
|
| 184 |
+
if not isinstance(value, dict):
|
| 185 |
+
raise RuntimeError("hf_bucket_metadata_invalid")
|
| 186 |
+
return value
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def cleanup_expired(*, api, cfg: BucketConfig, now: datetime | None = None) -> dict[str, Any]:
|
| 190 |
+
"""Delete only files older than the approved 7-day retention window.
|
| 191 |
+
|
| 192 |
+
Cleanup itself is best-effort at the call site. This function raises on API
|
| 193 |
+
failure so callers can log it, while never guessing that a younger object is
|
| 194 |
+
safe to delete.
|
| 195 |
+
"""
|
| 196 |
+
current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc)
|
| 197 |
+
cutoff = current - timedelta(hours=cfg.ttl_hours)
|
| 198 |
+
expired: list[str] = []
|
| 199 |
+
for item in api.list_bucket_tree(cfg.bucket_id, prefix="v1/users/", recursive=True):
|
| 200 |
+
if str(getattr(item, "type", "")) != "file":
|
| 201 |
+
continue
|
| 202 |
+
mtime = getattr(item, "mtime", None)
|
| 203 |
+
if not isinstance(mtime, datetime):
|
| 204 |
+
continue
|
| 205 |
+
if mtime.tzinfo is None:
|
| 206 |
+
mtime = mtime.replace(tzinfo=timezone.utc)
|
| 207 |
+
if mtime.astimezone(timezone.utc) < cutoff:
|
| 208 |
+
expired.append(str(item.path))
|
| 209 |
+
if expired:
|
| 210 |
+
api.batch_bucket_files(cfg.bucket_id, delete=expired)
|
| 211 |
+
return {"deleted": len(expired), "paths": expired}
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def archive_completed_zip(
|
| 215 |
+
*,
|
| 216 |
+
user_id: int | str,
|
| 217 |
+
task_id: str,
|
| 218 |
+
output_filename: str,
|
| 219 |
+
output_path: str | Path,
|
| 220 |
+
now: datetime | None = None,
|
| 221 |
+
) -> dict[str, Any]:
|
| 222 |
+
"""Best-effort archive primitive. Caller must catch all errors.
|
| 223 |
+
|
| 224 |
+
This function never modifies task/quota state and should only be called after
|
| 225 |
+
the authoritative Worker has accepted the completed task transition.
|
| 226 |
+
"""
|
| 227 |
+
cfg = config()
|
| 228 |
+
if not cfg.enabled:
|
| 229 |
+
return {"ok": True, "stored": False, "reason": "disabled"}
|
| 230 |
+
_validate_config(cfg)
|
| 231 |
+
trusted_user_id, trusted_task_id = _trusted_key_parts(user_id, task_id)
|
| 232 |
+
filename = _safe_output_filename(output_filename)
|
| 233 |
+
local = Path(output_path).resolve()
|
| 234 |
+
if not local.is_file() or local.name != filename:
|
| 235 |
+
raise RuntimeError("hf_bucket_local_zip_missing")
|
| 236 |
+
sha256, size_bytes = _sha256_file(local)
|
| 237 |
+
if size_bytes <= 0:
|
| 238 |
+
raise RuntimeError("hf_bucket_local_zip_empty")
|
| 239 |
+
|
| 240 |
+
api = _new_api(cfg.token)
|
| 241 |
+
_private_bucket_info(api, cfg)
|
| 242 |
+
|
| 243 |
+
cleanup_warning = ""
|
| 244 |
+
try:
|
| 245 |
+
cleanup_expired(api=api, cfg=cfg, now=now)
|
| 246 |
+
except Exception as exc: # cleanup must never destroy local delivery
|
| 247 |
+
cleanup_warning = type(exc).__name__
|
| 248 |
+
logging.warning("[SP_BUCKET] cleanup_failed task_id=%s error=%s", trusted_task_id, cleanup_warning)
|
| 249 |
+
|
| 250 |
+
zip_path, metadata_path = object_paths(trusted_user_id, trusted_task_id)
|
| 251 |
+
remote = _file_map(api, cfg.bucket_id, [zip_path, metadata_path])
|
| 252 |
+
if zip_path in remote or metadata_path in remote:
|
| 253 |
+
if zip_path in remote and metadata_path in remote:
|
| 254 |
+
existing = _download_metadata(api, cfg.bucket_id, metadata_path)
|
| 255 |
+
same = (
|
| 256 |
+
str(existing.get("user_id") or "") == trusted_user_id
|
| 257 |
+
and str(existing.get("task_id") or "") == trusted_task_id
|
| 258 |
+
and str(existing.get("output_filename") or "") == filename
|
| 259 |
+
and str(existing.get("sha256") or "") == sha256
|
| 260 |
+
and int(existing.get("size_bytes") or -1) == size_bytes
|
| 261 |
+
and int(getattr(remote[zip_path], "size", -1) or -1) == size_bytes
|
| 262 |
+
)
|
| 263 |
+
if same:
|
| 264 |
+
return {
|
| 265 |
+
"ok": True,
|
| 266 |
+
"stored": True,
|
| 267 |
+
"idempotent": True,
|
| 268 |
+
"sha256": sha256,
|
| 269 |
+
"size_bytes": size_bytes,
|
| 270 |
+
"cleanup_warning": cleanup_warning,
|
| 271 |
+
}
|
| 272 |
+
raise RuntimeError("hf_bucket_artifact_conflict")
|
| 273 |
+
|
| 274 |
+
info = _private_bucket_info(api, cfg)
|
| 275 |
+
current_size = int(getattr(info, "size", 0) or 0)
|
| 276 |
+
if current_size >= cfg.warning_bytes:
|
| 277 |
+
logging.warning("[SP_BUCKET] capacity_warning bytes=%s", current_size)
|
| 278 |
+
uploaded_at = (now or datetime.now(timezone.utc)).astimezone(timezone.utc)
|
| 279 |
+
metadata = _metadata_bytes(
|
| 280 |
+
user_id=trusted_user_id,
|
| 281 |
+
task_id=trusted_task_id,
|
| 282 |
+
filename=filename,
|
| 283 |
+
sha256=sha256,
|
| 284 |
+
size_bytes=size_bytes,
|
| 285 |
+
uploaded_at=uploaded_at,
|
| 286 |
+
)
|
| 287 |
+
projected = current_size + size_bytes + len(metadata)
|
| 288 |
+
if projected >= cfg.hard_stop_bytes:
|
| 289 |
+
raise RuntimeError("hf_bucket_hard_stop_reached")
|
| 290 |
+
|
| 291 |
+
api.batch_bucket_files(
|
| 292 |
+
cfg.bucket_id,
|
| 293 |
+
add=[(local, zip_path), (metadata, metadata_path)],
|
| 294 |
+
)
|
| 295 |
+
verified = _file_map(api, cfg.bucket_id, [zip_path, metadata_path])
|
| 296 |
+
if zip_path not in verified or metadata_path not in verified:
|
| 297 |
+
raise RuntimeError("hf_bucket_upload_incomplete")
|
| 298 |
+
if int(getattr(verified[zip_path], "size", -1) or -1) != size_bytes:
|
| 299 |
+
raise RuntimeError("hf_bucket_upload_size_mismatch")
|
| 300 |
+
existing = _download_metadata(api, cfg.bucket_id, metadata_path)
|
| 301 |
+
if (
|
| 302 |
+
str(existing.get("sha256") or "") != sha256
|
| 303 |
+
or int(existing.get("size_bytes") or -1) != size_bytes
|
| 304 |
+
or str(existing.get("task_id") or "") != trusted_task_id
|
| 305 |
+
or str(existing.get("user_id") or "") != trusted_user_id
|
| 306 |
+
):
|
| 307 |
+
raise RuntimeError("hf_bucket_upload_metadata_mismatch")
|
| 308 |
+
return {
|
| 309 |
+
"ok": True,
|
| 310 |
+
"stored": True,
|
| 311 |
+
"idempotent": False,
|
| 312 |
+
"sha256": sha256,
|
| 313 |
+
"size_bytes": size_bytes,
|
| 314 |
+
"cleanup_warning": cleanup_warning,
|
| 315 |
+
}
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def restore_completed_zip(
|
| 319 |
+
*,
|
| 320 |
+
user_id: int | str,
|
| 321 |
+
task_id: str,
|
| 322 |
+
output_filename: str,
|
| 323 |
+
output_path: str | Path,
|
| 324 |
+
now: datetime | None = None,
|
| 325 |
+
) -> bool:
|
| 326 |
+
"""Restore a completed ZIP from a private bucket, with ownership/integrity checks."""
|
| 327 |
+
cfg = config()
|
| 328 |
+
if not cfg.enabled:
|
| 329 |
+
return False
|
| 330 |
+
_validate_config(cfg)
|
| 331 |
+
trusted_user_id, trusted_task_id = _trusted_key_parts(user_id, task_id)
|
| 332 |
+
filename = _safe_output_filename(output_filename)
|
| 333 |
+
target = Path(output_path).resolve()
|
| 334 |
+
if target.is_file():
|
| 335 |
+
return True
|
| 336 |
+
if target.name != filename:
|
| 337 |
+
raise RuntimeError("hf_bucket_restore_target_invalid")
|
| 338 |
+
|
| 339 |
+
api = _new_api(cfg.token)
|
| 340 |
+
_private_bucket_info(api, cfg)
|
| 341 |
+
zip_path, metadata_path = object_paths(trusted_user_id, trusted_task_id)
|
| 342 |
+
remote = _file_map(api, cfg.bucket_id, [zip_path, metadata_path])
|
| 343 |
+
if zip_path not in remote or metadata_path not in remote:
|
| 344 |
+
return False
|
| 345 |
+
|
| 346 |
+
metadata = _download_metadata(api, cfg.bucket_id, metadata_path)
|
| 347 |
+
uploaded_at = _parse_uploaded_at(metadata.get("uploaded_at"))
|
| 348 |
+
current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc)
|
| 349 |
+
if uploaded_at > current + timedelta(minutes=5):
|
| 350 |
+
raise RuntimeError("hf_bucket_metadata_time_future")
|
| 351 |
+
if current - uploaded_at > timedelta(hours=cfg.ttl_hours):
|
| 352 |
+
return False
|
| 353 |
+
expected_size = int(metadata.get("size_bytes") or -1)
|
| 354 |
+
expected_sha = str(metadata.get("sha256") or "")
|
| 355 |
+
if (
|
| 356 |
+
str(metadata.get("schema_version") or "") != "articleagent-hf-bucket-v1"
|
| 357 |
+
or str(metadata.get("user_id") or "") != trusted_user_id
|
| 358 |
+
or str(metadata.get("task_id") or "") != trusted_task_id
|
| 359 |
+
or str(metadata.get("output_filename") or "") != filename
|
| 360 |
+
or expected_size <= 0
|
| 361 |
+
or not re.fullmatch(r"[0-9a-f]{64}", expected_sha)
|
| 362 |
+
or int(getattr(remote[zip_path], "size", -1) or -1) != expected_size
|
| 363 |
+
):
|
| 364 |
+
raise RuntimeError("hf_bucket_restore_metadata_mismatch")
|
| 365 |
+
|
| 366 |
+
target.parent.mkdir(parents=True, exist_ok=True)
|
| 367 |
+
temporary = target.with_suffix(target.suffix + ".bucket.tmp")
|
| 368 |
+
try:
|
| 369 |
+
temporary.unlink(missing_ok=True)
|
| 370 |
+
api.download_bucket_files(cfg.bucket_id, [(zip_path, temporary)], raise_on_missing_files=True)
|
| 371 |
+
observed_sha, observed_size = _sha256_file(temporary)
|
| 372 |
+
if observed_size != expected_size or observed_sha != expected_sha:
|
| 373 |
+
raise RuntimeError("hf_bucket_restore_integrity_mismatch")
|
| 374 |
+
with zipfile.ZipFile(temporary, "r") as archive:
|
| 375 |
+
if archive.testzip() is not None:
|
| 376 |
+
raise RuntimeError("hf_bucket_restore_zip_invalid")
|
| 377 |
+
os.replace(temporary, target)
|
| 378 |
+
finally:
|
| 379 |
+
temporary.unlink(missing_ok=True)
|
| 380 |
+
return True
|
huggingface/managed_auth.py
ADDED
|
@@ -0,0 +1,867 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""SmartPaper edge-first authentication and browser task/payment bridge.
|
| 3 |
+
|
| 4 |
+
Cloudflare owns the durable browser session, quota, tasks, and payments. The
|
| 5 |
+
Hugging Face process verifies the signed edge assertion but does not call the
|
| 6 |
+
Worker over the server-to-server network. Authenticated business operations are
|
| 7 |
+
executed by the user's browser through the same-origin invisible component.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import base64
|
| 12 |
+
import hashlib
|
| 13 |
+
import hmac
|
| 14 |
+
import html
|
| 15 |
+
import io
|
| 16 |
+
import json
|
| 17 |
+
import logging
|
| 18 |
+
import os
|
| 19 |
+
import re
|
| 20 |
+
import secrets
|
| 21 |
+
import time
|
| 22 |
+
from pathlib import Path
|
| 23 |
+
from typing import Any, Callable
|
| 24 |
+
from urllib.parse import unquote, urlsplit
|
| 25 |
+
|
| 26 |
+
import qrcode
|
| 27 |
+
import streamlit as st
|
| 28 |
+
|
| 29 |
+
from config import APP_CONFIG
|
| 30 |
+
from i18n import tr
|
| 31 |
+
from smartpaper_restore_bridge import edge_bridge
|
| 32 |
+
|
| 33 |
+
EDGE_ASSERTION_COOKIE_NAME = "smartpaper_edge_assertion"
|
| 34 |
+
EDGE_ASSERTION_AUDIENCE = "smartpaper-streamlit"
|
| 35 |
+
ASSERTION_CLOCK_SKEW_SECONDS = 30
|
| 36 |
+
def _runtime_value(name: str) -> str:
|
| 37 |
+
value = os.environ.get(name, "")
|
| 38 |
+
if not value:
|
| 39 |
+
try:
|
| 40 |
+
value = st.secrets.get(name, "")
|
| 41 |
+
except Exception:
|
| 42 |
+
value = ""
|
| 43 |
+
return str(value or "").strip()
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _api_base() -> str:
|
| 47 |
+
value = str(st.session_state.get("sp_auth_api_origin") or "").strip().rstrip("/")
|
| 48 |
+
if not value:
|
| 49 |
+
value = _runtime_value("SMARTPAPER_AUTH_API").rstrip("/")
|
| 50 |
+
parsed = urlsplit(value)
|
| 51 |
+
if (
|
| 52 |
+
parsed.scheme != "https"
|
| 53 |
+
or not parsed.netloc
|
| 54 |
+
or parsed.username is not None
|
| 55 |
+
or parsed.password is not None
|
| 56 |
+
or parsed.path not in ("", "/")
|
| 57 |
+
or parsed.query
|
| 58 |
+
or parsed.fragment
|
| 59 |
+
):
|
| 60 |
+
raise RuntimeError("SMARTPAPER_AUTH_API must be an HTTPS origin.")
|
| 61 |
+
return f"https://{parsed.netloc}"
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _service_key() -> str:
|
| 65 |
+
value = _runtime_value("SMARTPAPER_SERVICE_KEY")
|
| 66 |
+
if not value:
|
| 67 |
+
raise RuntimeError("SMARTPAPER_SERVICE_KEY is not configured.")
|
| 68 |
+
return value
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _context_cookie(name: str) -> str:
|
| 72 |
+
try:
|
| 73 |
+
cookies = st.context.cookies
|
| 74 |
+
value = cookies.get(name, "")
|
| 75 |
+
if value:
|
| 76 |
+
return unquote(str(value).strip())
|
| 77 |
+
target = name.lower()
|
| 78 |
+
for key, item in cookies.items():
|
| 79 |
+
if str(key).lower() == target:
|
| 80 |
+
return unquote(str(item or "").strip())
|
| 81 |
+
except Exception:
|
| 82 |
+
return ""
|
| 83 |
+
return ""
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _decode_b64url(value: str) -> bytes:
|
| 87 |
+
return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _verify_edge_assertion(token: str) -> dict[str, Any]:
|
| 91 |
+
try:
|
| 92 |
+
encoded, supplied_signature = token.split(".", 1)
|
| 93 |
+
except ValueError as exc:
|
| 94 |
+
raise RuntimeError("invalid_edge_assertion") from exc
|
| 95 |
+
expected_signature = hmac.new(
|
| 96 |
+
_service_key().encode("utf-8"),
|
| 97 |
+
encoded.encode("ascii"),
|
| 98 |
+
hashlib.sha256,
|
| 99 |
+
).hexdigest()
|
| 100 |
+
if not hmac.compare_digest(expected_signature, supplied_signature):
|
| 101 |
+
raise RuntimeError("invalid_edge_assertion_signature")
|
| 102 |
+
try:
|
| 103 |
+
payload = json.loads(_decode_b64url(encoded).decode("utf-8"))
|
| 104 |
+
except Exception as exc:
|
| 105 |
+
raise RuntimeError("invalid_edge_assertion_payload") from exc
|
| 106 |
+
now = int(time.time())
|
| 107 |
+
if payload.get("ver") != 1 or payload.get("aud") != EDGE_ASSERTION_AUDIENCE:
|
| 108 |
+
raise RuntimeError("invalid_edge_assertion_claims")
|
| 109 |
+
api_origin = str(payload.get("api_origin") or "").strip().rstrip("/")
|
| 110 |
+
parsed = urlsplit(api_origin)
|
| 111 |
+
if (
|
| 112 |
+
parsed.scheme != "https"
|
| 113 |
+
or not parsed.netloc
|
| 114 |
+
or parsed.username is not None
|
| 115 |
+
or parsed.password is not None
|
| 116 |
+
or parsed.path not in ("", "/")
|
| 117 |
+
or parsed.query
|
| 118 |
+
or parsed.fragment
|
| 119 |
+
):
|
| 120 |
+
raise RuntimeError("invalid_edge_assertion_origin")
|
| 121 |
+
issued_at = int(payload.get("iat") or 0)
|
| 122 |
+
expires_at = int(payload.get("exp") or 0)
|
| 123 |
+
if issued_at > now + ASSERTION_CLOCK_SKEW_SECONDS or expires_at <= now - ASSERTION_CLOCK_SKEW_SECONDS:
|
| 124 |
+
raise RuntimeError("expired_edge_assertion")
|
| 125 |
+
if expires_at - issued_at > 9 * 60 * 60:
|
| 126 |
+
raise RuntimeError("invalid_edge_assertion_lifetime")
|
| 127 |
+
user = payload.get("user") or {}
|
| 128 |
+
if not isinstance(user, dict) or not user.get("id") or user.get("status") != "active":
|
| 129 |
+
raise RuntimeError("invalid_edge_assertion_identity")
|
| 130 |
+
payload["api_origin"] = f"https://{parsed.netloc}"
|
| 131 |
+
return payload
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def _clear_generation_session_state() -> None:
|
| 135 |
+
for key in (
|
| 136 |
+
"sp_generation_task", "sp_tracked_task_id", "sp_current_node",
|
| 137 |
+
"sp_task_recovery_initialized", "sp_last_ui_task_status",
|
| 138 |
+
"sp_generation_idempotency_key", "zip_buffer", "zip_filename",
|
| 139 |
+
"is_running", "start_pipeline", "resume_pipeline", "run_mode",
|
| 140 |
+
"rag_engine", "up_amj_ref", "up_amj_data", "up_asq_ref",
|
| 141 |
+
"up_asq_data", "sp_payment_required", "sp_payment_order",
|
| 142 |
+
"sp_payment_request_id", "sp_payment_package_sku",
|
| 143 |
+
):
|
| 144 |
+
st.session_state.pop(key, None)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _clear_session() -> None:
|
| 148 |
+
for key in (
|
| 149 |
+
"sp_session_expires_at", "sp_user",
|
| 150 |
+
"sp_last_verified_at", "sp_edge_assertion", "sp_auth_api_origin",
|
| 151 |
+
"sp_product_ui_projection", "sp_product_tier",
|
| 152 |
+
):
|
| 153 |
+
st.session_state.pop(key, None)
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def require_alipay_login(language: str = "zh") -> dict[str, Any]:
|
| 157 |
+
"""Return the user authenticated by the Cloudflare edge gateway."""
|
| 158 |
+
token = _context_cookie(EDGE_ASSERTION_COOKIE_NAME)
|
| 159 |
+
if token and token == st.session_state.get("sp_edge_assertion") and st.session_state.get("sp_user"):
|
| 160 |
+
return dict(st.session_state["sp_user"])
|
| 161 |
+
if not token:
|
| 162 |
+
_clear_session()
|
| 163 |
+
st.stop()
|
| 164 |
+
try:
|
| 165 |
+
payload = _verify_edge_assertion(token)
|
| 166 |
+
except Exception:
|
| 167 |
+
_clear_session()
|
| 168 |
+
st.stop()
|
| 169 |
+
user = dict(payload["user"])
|
| 170 |
+
st.session_state["sp_edge_assertion"] = token
|
| 171 |
+
st.session_state["sp_session_expires_at"] = payload["exp"]
|
| 172 |
+
st.session_state["sp_user"] = user
|
| 173 |
+
st.session_state["sp_auth_api_origin"] = str(payload["api_origin"]).rstrip("/")
|
| 174 |
+
st.session_state["sp_last_verified_at"] = time.time()
|
| 175 |
+
return user
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _bridge_key(prefix: str, value: str) -> str:
|
| 179 |
+
digest = hashlib.sha256(str(value).encode("utf-8")).hexdigest()[:20]
|
| 180 |
+
return f"{prefix}_{digest}"
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def _edge_call(
|
| 184 |
+
action: str,
|
| 185 |
+
command_id: str,
|
| 186 |
+
payload: dict[str, Any] | None = None,
|
| 187 |
+
*,
|
| 188 |
+
key_prefix: str = "sp_edge",
|
| 189 |
+
) -> dict[str, Any] | None:
|
| 190 |
+
return edge_bridge(
|
| 191 |
+
action=action,
|
| 192 |
+
command_id=command_id,
|
| 193 |
+
api_origin=_api_base(),
|
| 194 |
+
payload=payload or {},
|
| 195 |
+
key=_bridge_key(key_prefix, command_id),
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def _raise_edge_error(result: dict[str, Any], fallback: str) -> None:
|
| 200 |
+
message = (
|
| 201 |
+
result.get("message")
|
| 202 |
+
or result.get("reason")
|
| 203 |
+
or result.get("error")
|
| 204 |
+
or fallback
|
| 205 |
+
)
|
| 206 |
+
raise RuntimeError(str(message))
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def _merge_quota_into_session(result: dict[str, Any]) -> None:
|
| 210 |
+
"""Refresh the approved account card immediately after a quota mutation."""
|
| 211 |
+
quota = result.get("quota")
|
| 212 |
+
if not isinstance(quota, dict):
|
| 213 |
+
return
|
| 214 |
+
user = dict(st.session_state.get("sp_user") or {})
|
| 215 |
+
for key in (
|
| 216 |
+
"quota_date",
|
| 217 |
+
"daily_free_limit",
|
| 218 |
+
"daily_free_reserved",
|
| 219 |
+
"daily_free_used",
|
| 220 |
+
"daily_free_remaining",
|
| 221 |
+
"bonus_papers",
|
| 222 |
+
):
|
| 223 |
+
if key in quota:
|
| 224 |
+
user[key] = quota[key]
|
| 225 |
+
if user:
|
| 226 |
+
st.session_state["sp_user"] = user
|
| 227 |
+
st.session_state["sp_account_revision"] = int(
|
| 228 |
+
st.session_state.get("sp_account_revision") or 0
|
| 229 |
+
) + 1
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _legacy_product_ui_projection() -> dict[str, Any]:
|
| 233 |
+
return {
|
| 234 |
+
"schema_version": "a5-product-ui-v1",
|
| 235 |
+
"mode": "legacy",
|
| 236 |
+
"product_tier_selection_required": False,
|
| 237 |
+
"products": {},
|
| 238 |
+
}
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def _normalize_product_ui_projection(value: Any) -> dict[str, Any]:
|
| 242 |
+
"""Accept only the Worker-owned A5 UI projection; never reinterpret legacy quota fields."""
|
| 243 |
+
if not isinstance(value, dict):
|
| 244 |
+
return _legacy_product_ui_projection()
|
| 245 |
+
mode = str(value.get("mode") or "legacy").strip().lower()
|
| 246 |
+
if mode == "legacy":
|
| 247 |
+
return _legacy_product_ui_projection()
|
| 248 |
+
if mode != "a5_product":
|
| 249 |
+
raise RuntimeError("product_ui_projection_invalid_mode")
|
| 250 |
+
|
| 251 |
+
raw_products = value.get("products")
|
| 252 |
+
raw_entitlement = value.get("entitlement")
|
| 253 |
+
if not isinstance(raw_products, dict) or not isinstance(raw_entitlement, dict):
|
| 254 |
+
raise RuntimeError("product_ui_projection_incomplete")
|
| 255 |
+
|
| 256 |
+
products: dict[str, dict[str, Any]] = {}
|
| 257 |
+
for tier in ("STANDARD", "FLAGSHIP"):
|
| 258 |
+
raw = raw_products.get(tier)
|
| 259 |
+
if not isinstance(raw, dict):
|
| 260 |
+
raise RuntimeError("product_ui_projection_missing_tier")
|
| 261 |
+
try:
|
| 262 |
+
minimum = int(raw.get("min_body_words"))
|
| 263 |
+
maximum = int(raw.get("max_body_words"))
|
| 264 |
+
except (TypeError, ValueError) as exc:
|
| 265 |
+
raise RuntimeError("product_ui_projection_invalid_word_policy") from exc
|
| 266 |
+
if minimum <= 0 or maximum < minimum:
|
| 267 |
+
raise RuntimeError("product_ui_projection_invalid_word_policy")
|
| 268 |
+
try:
|
| 269 |
+
amount_fen = int(raw.get("amount_fen"))
|
| 270 |
+
credits_granted = int(raw.get("credits_granted"))
|
| 271 |
+
except (TypeError, ValueError) as exc:
|
| 272 |
+
raise RuntimeError("product_ui_projection_invalid_billing_policy") from exc
|
| 273 |
+
products[tier] = {
|
| 274 |
+
"product_tier": tier,
|
| 275 |
+
"product_policy_version": str(raw.get("product_policy_version") or ""),
|
| 276 |
+
"word_count_policy_version": str(raw.get("word_count_policy_version") or ""),
|
| 277 |
+
"body_metric": str(raw.get("body_metric") or ""),
|
| 278 |
+
"min_body_words": minimum,
|
| 279 |
+
"max_body_words": maximum,
|
| 280 |
+
"counting_contract_version": str(raw.get("counting_contract_version") or ""),
|
| 281 |
+
"package_sku": str(raw.get("package_sku") or ""),
|
| 282 |
+
"billing_policy_version": str(raw.get("billing_policy_version") or ""),
|
| 283 |
+
"amount_fen": amount_fen,
|
| 284 |
+
"currency": str(raw.get("currency") or ""),
|
| 285 |
+
"credits_granted": credits_granted,
|
| 286 |
+
}
|
| 287 |
+
if not all((
|
| 288 |
+
products[tier]["product_policy_version"],
|
| 289 |
+
products[tier]["word_count_policy_version"],
|
| 290 |
+
products[tier]["body_metric"],
|
| 291 |
+
products[tier]["counting_contract_version"],
|
| 292 |
+
products[tier]["package_sku"],
|
| 293 |
+
products[tier]["billing_policy_version"],
|
| 294 |
+
products[tier]["currency"],
|
| 295 |
+
)) or amount_fen <= 0 or credits_granted <= 0:
|
| 296 |
+
raise RuntimeError("product_ui_projection_incomplete")
|
| 297 |
+
|
| 298 |
+
try:
|
| 299 |
+
standard_remaining = int(raw_entitlement.get("standard_remaining"))
|
| 300 |
+
flagship_remaining = int(raw_entitlement.get("flagship_remaining"))
|
| 301 |
+
except (TypeError, ValueError) as exc:
|
| 302 |
+
raise RuntimeError("product_ui_projection_invalid_entitlement") from exc
|
| 303 |
+
if standard_remaining < 0 or flagship_remaining < 0:
|
| 304 |
+
raise RuntimeError("product_ui_projection_invalid_entitlement")
|
| 305 |
+
|
| 306 |
+
return {
|
| 307 |
+
"schema_version": str(value.get("schema_version") or "a5-product-ui-v1"),
|
| 308 |
+
"mode": "a5_product",
|
| 309 |
+
"product_tier_selection_required": True,
|
| 310 |
+
"products": products,
|
| 311 |
+
"entitlement": {
|
| 312 |
+
"standard_remaining": standard_remaining,
|
| 313 |
+
"flagship_remaining": flagship_remaining,
|
| 314 |
+
},
|
| 315 |
+
}
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def get_product_ui_projection() -> dict[str, Any]:
|
| 319 |
+
value = st.session_state.get("sp_product_ui_projection")
|
| 320 |
+
return _normalize_product_ui_projection(value)
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
def get_account_snapshot(*, block: bool = False) -> dict[str, Any] | None:
|
| 324 |
+
bucket = int(time.time() // 30)
|
| 325 |
+
revision = int(st.session_state.get("sp_account_revision") or 0)
|
| 326 |
+
command_id = f"account:{revision}:{bucket}"
|
| 327 |
+
result = _edge_call("account", command_id, {}, key_prefix="sp_account")
|
| 328 |
+
if result is None:
|
| 329 |
+
if block:
|
| 330 |
+
st.stop()
|
| 331 |
+
return None
|
| 332 |
+
if not result.get("ok"):
|
| 333 |
+
if block:
|
| 334 |
+
_raise_edge_error(result, "account_query_failed")
|
| 335 |
+
return None
|
| 336 |
+
user = result.get("user")
|
| 337 |
+
if isinstance(user, dict):
|
| 338 |
+
st.session_state["sp_user"] = dict(user)
|
| 339 |
+
projection = _normalize_product_ui_projection(result.get("product_ui"))
|
| 340 |
+
st.session_state["sp_product_ui_projection"] = projection
|
| 341 |
+
if projection["mode"] != "a5_product":
|
| 342 |
+
st.session_state.pop("sp_product_tier", None)
|
| 343 |
+
# R3.1 single-writer rule: account refresh owns identity/quota only.
|
| 344 |
+
# Task UI state is initialized by recover_generation_task() and thereafter
|
| 345 |
+
# updated only by generation_task_status()/explicit task actions.
|
| 346 |
+
return result
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def start_generation_task(
|
| 350 |
+
*,
|
| 351 |
+
idempotency_key: str,
|
| 352 |
+
mode: str,
|
| 353 |
+
interface_language: str,
|
| 354 |
+
target_words: int,
|
| 355 |
+
product_tier: str | None = None,
|
| 356 |
+
) -> dict[str, Any]:
|
| 357 |
+
normalized = str(idempotency_key or "").strip()
|
| 358 |
+
if not normalized:
|
| 359 |
+
raise RuntimeError("idempotency_key_required")
|
| 360 |
+
projection = get_product_ui_projection()
|
| 361 |
+
normalized_tier = str(product_tier or "").strip().upper()
|
| 362 |
+
product_mode = projection.get("mode") == "a5_product"
|
| 363 |
+
if product_mode:
|
| 364 |
+
if normalized_tier not in projection.get("products", {}):
|
| 365 |
+
raise RuntimeError("product_tier_required")
|
| 366 |
+
elif normalized_tier:
|
| 367 |
+
raise RuntimeError("product_tier_not_available_in_legacy_mode")
|
| 368 |
+
payload = {
|
| 369 |
+
"idempotency_key": normalized,
|
| 370 |
+
"mode": str(mode or "").strip().lower(),
|
| 371 |
+
"interface_language": str(interface_language or "zh").strip(),
|
| 372 |
+
"target_words": int(target_words),
|
| 373 |
+
}
|
| 374 |
+
if product_mode:
|
| 375 |
+
# Product tier is user intent only. Worker/D1 resolves entitlement and
|
| 376 |
+
# freezes the trusted TaskPolicySnapshot; browser state is never authority.
|
| 377 |
+
payload["product_tier"] = normalized_tier
|
| 378 |
+
result = _edge_call(
|
| 379 |
+
"task_start",
|
| 380 |
+
normalized,
|
| 381 |
+
payload,
|
| 382 |
+
key_prefix="sp_task_start",
|
| 383 |
+
)
|
| 384 |
+
if result is None:
|
| 385 |
+
st.stop()
|
| 386 |
+
if not result.get("ok"):
|
| 387 |
+
_raise_edge_error(result, "task_start_failed")
|
| 388 |
+
_merge_quota_into_session(result)
|
| 389 |
+
started_task = result.get("task")
|
| 390 |
+
if isinstance(started_task, dict) and started_task.get("task_id"):
|
| 391 |
+
st.session_state["sp_tracked_task_id"] = str(started_task["task_id"])
|
| 392 |
+
return dict(result)
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
def _completed_output_path(task: dict[str, Any]) -> Path | None:
|
| 396 |
+
user = dict(st.session_state.get("sp_user") or {})
|
| 397 |
+
user_key = re.sub(r"[^A-Za-z0-9_.-]", "_", str(user.get("id") or "").strip())[:100]
|
| 398 |
+
task_key = re.sub(r"[^A-Za-z0-9_.-]", "_", str((task or {}).get("task_id") or "").strip())[:100]
|
| 399 |
+
filename = os.path.basename(str((task or {}).get("output_filename") or "").strip())
|
| 400 |
+
if not user_key or not task_key or not filename or not filename.lower().endswith(".zip"):
|
| 401 |
+
return None
|
| 402 |
+
workspace = Path(str(APP_CONFIG.get("workspace_dir", ".workspace")))
|
| 403 |
+
return workspace / "users" / user_key / "tasks" / task_key / "outputs" / filename
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def _restore_completed_artifact_if_needed(task: dict[str, Any]) -> None:
|
| 407 |
+
"""Best-effort restore for a completed task when this runtime lost its local ZIP."""
|
| 408 |
+
if str((task or {}).get("status") or "").strip().lower() != "completed":
|
| 409 |
+
return
|
| 410 |
+
output_path = _completed_output_path(task)
|
| 411 |
+
if output_path is None or output_path.is_file():
|
| 412 |
+
return
|
| 413 |
+
user = dict(st.session_state.get("sp_user") or {})
|
| 414 |
+
numeric_user_id = user.get("numeric_id")
|
| 415 |
+
if numeric_user_id in (None, ""):
|
| 416 |
+
logging.warning("[SP_AUTH] bucket_restore_skipped task_id=%s reason=numeric_user_missing", str(task.get("task_id") or ""))
|
| 417 |
+
return
|
| 418 |
+
try:
|
| 419 |
+
from hf_bucket_store import restore_completed_zip
|
| 420 |
+
|
| 421 |
+
restored = restore_completed_zip(
|
| 422 |
+
user_id=numeric_user_id,
|
| 423 |
+
task_id=str(task.get("task_id") or ""),
|
| 424 |
+
output_filename=str(task.get("output_filename") or ""),
|
| 425 |
+
output_path=output_path,
|
| 426 |
+
)
|
| 427 |
+
if restored:
|
| 428 |
+
logging.info("[SP_AUTH] bucket_restore_ok task_id=%s", str(task.get("task_id") or ""))
|
| 429 |
+
except Exception as exc:
|
| 430 |
+
# Recovery storage is never authoritative for task state or quota.
|
| 431 |
+
logging.warning(
|
| 432 |
+
"[SP_AUTH] bucket_restore_nonfatal task_id=%s error=%s",
|
| 433 |
+
str((task or {}).get("task_id") or ""), type(exc).__name__,
|
| 434 |
+
)
|
| 435 |
+
|
| 436 |
+
|
| 437 |
+
def recover_generation_task() -> dict[str, Any]:
|
| 438 |
+
"""Initialize current task state once from the authoritative Worker.
|
| 439 |
+
|
| 440 |
+
Account polling is intentionally forbidden from writing task state. This
|
| 441 |
+
recovery call is the only fresh-session initializer; later changes come
|
| 442 |
+
from generation_task_status() or explicit start/cancel operations.
|
| 443 |
+
"""
|
| 444 |
+
command_id = f"recover:{int(time.time() // 10)}"
|
| 445 |
+
result = _edge_call("task_recover", command_id, {}, key_prefix="sp_task_recover")
|
| 446 |
+
if result is None:
|
| 447 |
+
st.stop()
|
| 448 |
+
if not result.get("ok"):
|
| 449 |
+
_raise_edge_error(result, "task_recover_failed")
|
| 450 |
+
_merge_quota_into_session(result)
|
| 451 |
+
task = result.get("task")
|
| 452 |
+
if isinstance(task, dict) and task.get("task_id"):
|
| 453 |
+
_restore_completed_artifact_if_needed(task)
|
| 454 |
+
st.session_state["sp_generation_task"] = dict(task)
|
| 455 |
+
st.session_state["sp_tracked_task_id"] = str(task["task_id"])
|
| 456 |
+
st.session_state["sp_last_ui_task_status"] = str(task.get("status") or "")
|
| 457 |
+
else:
|
| 458 |
+
st.session_state["sp_generation_task"] = None
|
| 459 |
+
st.session_state["sp_last_ui_task_status"] = ""
|
| 460 |
+
st.session_state.pop("sp_tracked_task_id", None)
|
| 461 |
+
return dict(result)
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
def generation_task_status(task_id: str) -> dict[str, Any]:
|
| 465 |
+
normalized = str(task_id or "").strip()
|
| 466 |
+
command_id = f"status:{normalized}:{int(time.time() // 5)}"
|
| 467 |
+
result = _edge_call(
|
| 468 |
+
"task_status", command_id, {"task_id": normalized}, key_prefix="sp_task_status"
|
| 469 |
+
)
|
| 470 |
+
if result is None:
|
| 471 |
+
st.stop()
|
| 472 |
+
if not result.get("ok"):
|
| 473 |
+
_raise_edge_error(result, "task_status_failed")
|
| 474 |
+
_merge_quota_into_session(result)
|
| 475 |
+
task = result.get("task")
|
| 476 |
+
if isinstance(task, dict):
|
| 477 |
+
st.session_state["sp_generation_task"] = dict(task)
|
| 478 |
+
if task.get("task_id"):
|
| 479 |
+
st.session_state["sp_tracked_task_id"] = str(task["task_id"])
|
| 480 |
+
return dict(result)
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
def cancel_generation_task(task_id: str | None = None) -> dict[str, Any]:
|
| 484 |
+
"""Cancel synchronously so quota release is confirmed before local cleanup."""
|
| 485 |
+
normalized = str(task_id or "").strip()
|
| 486 |
+
command_id = f"cancel:{normalized or 'latest'}"
|
| 487 |
+
result = _edge_call(
|
| 488 |
+
"task_cancel",
|
| 489 |
+
command_id,
|
| 490 |
+
{"task_id": normalized} if normalized else {},
|
| 491 |
+
key_prefix="sp_task_cancel",
|
| 492 |
+
)
|
| 493 |
+
if result is None:
|
| 494 |
+
st.stop()
|
| 495 |
+
if not result.get("ok"):
|
| 496 |
+
_raise_edge_error(result, "task_cancel_failed")
|
| 497 |
+
reservation = result.get("reservation")
|
| 498 |
+
if result.get("cancelled") and reservation and str(reservation.get("status") or "").lower() != "released":
|
| 499 |
+
raise RuntimeError("task_cancel_quota_not_released")
|
| 500 |
+
_merge_quota_into_session(result)
|
| 501 |
+
return dict(result)
|
| 502 |
+
|
| 503 |
+
|
| 504 |
+
def logout_alipay_session(on_logout: Callable[[str, str], None] | None = None) -> None:
|
| 505 |
+
user_id = str((st.session_state.get("sp_user") or {}).get("id") or "")
|
| 506 |
+
task = st.session_state.get("sp_generation_task") or {}
|
| 507 |
+
task_id = str(task.get("task_id") or "")
|
| 508 |
+
if on_logout is not None:
|
| 509 |
+
try:
|
| 510 |
+
on_logout(user_id, task_id)
|
| 511 |
+
except Exception:
|
| 512 |
+
pass
|
| 513 |
+
_clear_session()
|
| 514 |
+
_clear_generation_session_state()
|
| 515 |
+
|
| 516 |
+
|
| 517 |
+
def create_payment_order(
|
| 518 |
+
idempotency_key: str,
|
| 519 |
+
package_sku: str | None = None,
|
| 520 |
+
) -> dict[str, Any]:
|
| 521 |
+
normalized = str(idempotency_key or "").strip()
|
| 522 |
+
payload: dict[str, Any] = {"idempotency_key": normalized}
|
| 523 |
+
normalized_sku = str(package_sku or "").strip()
|
| 524 |
+
if normalized_sku:
|
| 525 |
+
payload["package_sku"] = normalized_sku
|
| 526 |
+
result = _edge_call(
|
| 527 |
+
"pay_create",
|
| 528 |
+
normalized,
|
| 529 |
+
payload,
|
| 530 |
+
key_prefix="sp_pay_create",
|
| 531 |
+
)
|
| 532 |
+
if result is None:
|
| 533 |
+
st.stop()
|
| 534 |
+
if not result.get("ok"):
|
| 535 |
+
_raise_edge_error(result, "payment_create_failed")
|
| 536 |
+
return dict(result)
|
| 537 |
+
|
| 538 |
+
|
| 539 |
+
def payment_order_status(merchant_order_no: str) -> dict[str, Any]:
|
| 540 |
+
order_no = str(merchant_order_no or "").strip()
|
| 541 |
+
command_id = f"pay-status:{order_no}:{int(time.time() // 3)}"
|
| 542 |
+
result = _edge_call(
|
| 543 |
+
"pay_status",
|
| 544 |
+
command_id,
|
| 545 |
+
{"merchant_order_no": order_no},
|
| 546 |
+
key_prefix="sp_pay_status",
|
| 547 |
+
)
|
| 548 |
+
if result is None:
|
| 549 |
+
st.stop()
|
| 550 |
+
if not result.get("ok"):
|
| 551 |
+
_raise_edge_error(result, "payment_status_failed")
|
| 552 |
+
return dict(result)
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
def _qr_png_bytes(value: str) -> bytes:
|
| 556 |
+
image = qrcode.make(str(value))
|
| 557 |
+
buffer = io.BytesIO()
|
| 558 |
+
image.save(buffer, format="PNG")
|
| 559 |
+
return buffer.getvalue()
|
| 560 |
+
|
| 561 |
+
|
| 562 |
+
def _render_payment_purchase_body(language: str) -> None:
|
| 563 |
+
projection = get_product_ui_projection()
|
| 564 |
+
product_mode = projection.get("mode") == "a5_product"
|
| 565 |
+
selected_product: dict[str, Any] | None = None
|
| 566 |
+
package_sku = ""
|
| 567 |
+
if product_mode:
|
| 568 |
+
selected_tier = str(st.session_state.get("sp_product_tier") or "STANDARD").strip().upper()
|
| 569 |
+
raw_product = (projection.get("products") or {}).get(selected_tier)
|
| 570 |
+
if not isinstance(raw_product, dict):
|
| 571 |
+
raise RuntimeError("product_payment_projection_missing_tier")
|
| 572 |
+
selected_product = dict(raw_product)
|
| 573 |
+
package_sku = str(selected_product.get("package_sku") or "").strip()
|
| 574 |
+
amount_fen = int(selected_product.get("amount_fen") or 0)
|
| 575 |
+
credits = int(selected_product.get("credits_granted") or 0)
|
| 576 |
+
if not package_sku or amount_fen <= 0 or credits <= 0:
|
| 577 |
+
raise RuntimeError("product_payment_projection_invalid")
|
| 578 |
+
amount_text = f"{amount_fen / 100:.2f}".rstrip("0").rstrip(".")
|
| 579 |
+
currency = str(selected_product.get("currency") or "").strip()
|
| 580 |
+
if not currency:
|
| 581 |
+
raise RuntimeError("product_payment_projection_invalid_currency")
|
| 582 |
+
tier_name = tr(language, f"product_tier_{selected_tier.lower()}")
|
| 583 |
+
st.warning(
|
| 584 |
+
tr(
|
| 585 |
+
language,
|
| 586 |
+
"payment_notice_product",
|
| 587 |
+
tier=tier_name,
|
| 588 |
+
amount=amount_text,
|
| 589 |
+
currency=currency,
|
| 590 |
+
credits=credits,
|
| 591 |
+
)
|
| 592 |
+
)
|
| 593 |
+
existing_order = st.session_state.get("sp_payment_order")
|
| 594 |
+
existing_sku = str(st.session_state.get("sp_payment_package_sku") or "").strip()
|
| 595 |
+
order_sku = str((existing_order or {}).get("package_sku") or "").strip() if isinstance(existing_order, dict) else ""
|
| 596 |
+
if (existing_sku and existing_sku != package_sku) or (existing_order and order_sku != package_sku):
|
| 597 |
+
st.session_state["sp_payment_order"] = None
|
| 598 |
+
st.session_state["sp_payment_request_id"] = ""
|
| 599 |
+
else:
|
| 600 |
+
st.warning(tr(language, "payment_notice"))
|
| 601 |
+
|
| 602 |
+
if st.button(tr(language, "payment_create"), key="sp_create_payment_order"):
|
| 603 |
+
st.session_state["sp_payment_request_id"] = (
|
| 604 |
+
f"pay_{int(time.time() * 1000)}_{secrets.token_urlsafe(12)}"
|
| 605 |
+
)
|
| 606 |
+
st.session_state["sp_payment_package_sku"] = package_sku
|
| 607 |
+
|
| 608 |
+
request_id = str(st.session_state.get("sp_payment_request_id") or "").strip()
|
| 609 |
+
order = st.session_state.get("sp_payment_order")
|
| 610 |
+
if request_id and not order:
|
| 611 |
+
st.info(tr(language, "payment_creating"))
|
| 612 |
+
order_result = create_payment_order(request_id, package_sku or None)
|
| 613 |
+
order = dict(order_result.get("order") or {})
|
| 614 |
+
if not order.get("merchant_order_no") or not order.get("qr_code"):
|
| 615 |
+
raise RuntimeError("payment_order_missing_qr_code")
|
| 616 |
+
if product_mode and str(order.get("package_sku") or "") != package_sku:
|
| 617 |
+
raise RuntimeError("payment_order_package_mismatch")
|
| 618 |
+
st.session_state["sp_payment_order"] = order
|
| 619 |
+
projected = order_result.get("product_ui")
|
| 620 |
+
if isinstance(projected, dict):
|
| 621 |
+
st.session_state["sp_product_ui_projection"] = _normalize_product_ui_projection(projected)
|
| 622 |
+
|
| 623 |
+
order = st.session_state.get("sp_payment_order") or {}
|
| 624 |
+
qr_code = str(order.get("qr_code") or "").strip()
|
| 625 |
+
if qr_code:
|
| 626 |
+
st.image(_qr_png_bytes(qr_code), width=280)
|
| 627 |
+
order_no = str(order.get("merchant_order_no") or "").strip()
|
| 628 |
+
if order_no:
|
| 629 |
+
st.caption(tr(language, "payment_order_no", order_no=order_no))
|
| 630 |
+
st.code(order_no)
|
| 631 |
+
payment_provider = str(order.get("payment_provider") or "").strip().upper()
|
| 632 |
+
if payment_provider == "MANUAL_ALIPAY":
|
| 633 |
+
st.caption(tr(language, "payment_scan_manual"))
|
| 634 |
+
else:
|
| 635 |
+
st.caption(tr(language, "payment_scan"))
|
| 636 |
+
support = order.get("payment_support") if isinstance(order.get("payment_support"), dict) else {}
|
| 637 |
+
support_wechat = str((support or {}).get("wechat") or "").strip()
|
| 638 |
+
support_whatsapp = str((support or {}).get("whatsapp") or "").strip()
|
| 639 |
+
if support_wechat or support_whatsapp:
|
| 640 |
+
st.info(tr(language, "payment_support_prompt"))
|
| 641 |
+
if support_wechat:
|
| 642 |
+
st.caption(tr(language, "payment_support_wechat", value=support_wechat))
|
| 643 |
+
if support_whatsapp:
|
| 644 |
+
st.caption(tr(language, "payment_support_whatsapp", value=support_whatsapp))
|
| 645 |
+
if st.button(tr(language, "payment_check"), key="sp_check_payment_status"):
|
| 646 |
+
status_result = payment_order_status(order.get("merchant_order_no", ""))
|
| 647 |
+
updated = dict(status_result.get("order") or {})
|
| 648 |
+
st.session_state["sp_payment_order"] = updated
|
| 649 |
+
projected = status_result.get("product_ui")
|
| 650 |
+
if isinstance(projected, dict):
|
| 651 |
+
st.session_state["sp_product_ui_projection"] = _normalize_product_ui_projection(projected)
|
| 652 |
+
if updated.get("status") == "paid" and updated.get("credit_applied"):
|
| 653 |
+
user = status_result.get("user")
|
| 654 |
+
if isinstance(user, dict):
|
| 655 |
+
st.session_state["sp_user"] = dict(user)
|
| 656 |
+
st.session_state["sp_account_revision"] = int(
|
| 657 |
+
st.session_state.get("sp_account_revision") or 0
|
| 658 |
+
) + 1
|
| 659 |
+
st.session_state["sp_payment_required"] = False
|
| 660 |
+
st.session_state["sp_payment_order"] = None
|
| 661 |
+
st.session_state["sp_payment_request_id"] = ""
|
| 662 |
+
st.session_state["sp_payment_package_sku"] = ""
|
| 663 |
+
if product_mode and selected_product is not None:
|
| 664 |
+
tier_name = tr(language, f"product_tier_{str(selected_product.get('product_tier') or '').lower()}")
|
| 665 |
+
st.success(
|
| 666 |
+
tr(
|
| 667 |
+
language,
|
| 668 |
+
"payment_paid_product",
|
| 669 |
+
tier=tier_name,
|
| 670 |
+
credits=int(selected_product.get("credits_granted") or 0),
|
| 671 |
+
)
|
| 672 |
+
)
|
| 673 |
+
else:
|
| 674 |
+
st.success(tr(language, "payment_paid"))
|
| 675 |
+
elif updated.get("status") in {"closed", "failed"}:
|
| 676 |
+
st.session_state["sp_payment_order"] = None
|
| 677 |
+
st.session_state["sp_payment_request_id"] = ""
|
| 678 |
+
st.session_state["sp_payment_package_sku"] = ""
|
| 679 |
+
st.warning(tr(language, "payment_expired"))
|
| 680 |
+
|
| 681 |
+
|
| 682 |
+
def render_payment_purchase(language: str) -> None:
|
| 683 |
+
"""Show payment in a modal so the approved normal-page layout is unchanged."""
|
| 684 |
+
dialog_factory = getattr(st, "dialog", None)
|
| 685 |
+
if callable(dialog_factory):
|
| 686 |
+
@dialog_factory(tr(language, "payment_dialog_title"))
|
| 687 |
+
def _payment_dialog():
|
| 688 |
+
_render_payment_purchase_body(language)
|
| 689 |
+
_payment_dialog()
|
| 690 |
+
return
|
| 691 |
+
# Compatibility fallback for older Streamlit builds.
|
| 692 |
+
_render_payment_purchase_body(language)
|
| 693 |
+
|
| 694 |
+
|
| 695 |
+
def render_logged_in_account(
|
| 696 |
+
language: str,
|
| 697 |
+
user: dict[str, Any] | None = None,
|
| 698 |
+
on_logout: Callable[[str, str], None] | None = None,
|
| 699 |
+
) -> None:
|
| 700 |
+
"""Render the approved responsive account summary with live quota values."""
|
| 701 |
+
live = get_account_snapshot(block=False)
|
| 702 |
+
if live and isinstance(live.get("user"), dict):
|
| 703 |
+
user = dict(live["user"])
|
| 704 |
+
else:
|
| 705 |
+
user = user or st.session_state.get("sp_user", {})
|
| 706 |
+
user_id = user.get("id", "—")
|
| 707 |
+
projection = get_product_ui_projection()
|
| 708 |
+
product_mode = projection.get("mode") == "a5_product"
|
| 709 |
+
if product_mode:
|
| 710 |
+
entitlement = projection["entitlement"]
|
| 711 |
+
standard_remaining = entitlement["standard_remaining"]
|
| 712 |
+
flagship_remaining = entitlement["flagship_remaining"]
|
| 713 |
+
else:
|
| 714 |
+
free_remaining = user.get("daily_free_remaining", user.get("daily_free_limit", 3))
|
| 715 |
+
bonus_papers = user.get("bonus_papers", 0)
|
| 716 |
+
|
| 717 |
+
st.markdown(
|
| 718 |
+
"""
|
| 719 |
+
<style>
|
| 720 |
+
section[data-testid="stSidebar"] {
|
| 721 |
+
display: none !important;
|
| 722 |
+
}
|
| 723 |
+
|
| 724 |
+
[data-testid="collapsedControl"] {
|
| 725 |
+
display: none !important;
|
| 726 |
+
}
|
| 727 |
+
|
| 728 |
+
.sp-account-card {
|
| 729 |
+
display: flex;
|
| 730 |
+
align-items: center;
|
| 731 |
+
justify-content: center;
|
| 732 |
+
gap: clamp(0.3rem, 0.55vw, 0.58rem);
|
| 733 |
+
width: 100%;
|
| 734 |
+
min-width: 0;
|
| 735 |
+
min-height: 2.85rem;
|
| 736 |
+
padding: 0.48rem clamp(0.55rem, 0.85vw, 0.9rem);
|
| 737 |
+
border: 1px solid rgba(148, 163, 184, 0.30);
|
| 738 |
+
border-radius: 13px;
|
| 739 |
+
background: rgba(30, 41, 59, 0.58);
|
| 740 |
+
color: var(--text-color, #f3f4f6);
|
| 741 |
+
font-size: clamp(0.76rem, 0.84vw, 0.92rem);
|
| 742 |
+
font-weight: 500;
|
| 743 |
+
line-height: 1.25;
|
| 744 |
+
white-space: nowrap;
|
| 745 |
+
overflow: visible;
|
| 746 |
+
box-sizing: border-box;
|
| 747 |
+
}
|
| 748 |
+
|
| 749 |
+
.sp-account-card span {
|
| 750 |
+
display: inline-block;
|
| 751 |
+
white-space: nowrap;
|
| 752 |
+
}
|
| 753 |
+
|
| 754 |
+
.sp-account-separator {
|
| 755 |
+
flex: 0 0 auto;
|
| 756 |
+
opacity: 0.48;
|
| 757 |
+
}
|
| 758 |
+
|
| 759 |
+
div[data-testid="stMarkdownContainer"]:has(.sp-logout-link) {
|
| 760 |
+
width: 100%;
|
| 761 |
+
}
|
| 762 |
+
|
| 763 |
+
div[data-testid="stMarkdownContainer"]:has(.sp-logout-link) p {
|
| 764 |
+
width: 100%;
|
| 765 |
+
margin: 0;
|
| 766 |
+
}
|
| 767 |
+
|
| 768 |
+
.sp-logout-link {
|
| 769 |
+
display: flex;
|
| 770 |
+
align-items: center;
|
| 771 |
+
justify-content: center;
|
| 772 |
+
width: 100%;
|
| 773 |
+
min-width: 6.8rem;
|
| 774 |
+
min-height: 2.85rem;
|
| 775 |
+
padding: 0.45rem 0.72rem;
|
| 776 |
+
border: 1px solid rgba(148, 163, 184, 0.30);
|
| 777 |
+
border-radius: 0.5rem;
|
| 778 |
+
background: transparent;
|
| 779 |
+
color: var(--text-color, #f3f4f6) !important;
|
| 780 |
+
text-decoration: none !important;
|
| 781 |
+
white-space: nowrap !important;
|
| 782 |
+
overflow-wrap: normal !important;
|
| 783 |
+
word-break: keep-all !important;
|
| 784 |
+
line-height: 1.15;
|
| 785 |
+
font-size: clamp(0.78rem, 0.82vw, 0.92rem);
|
| 786 |
+
font-weight: 400;
|
| 787 |
+
box-sizing: border-box;
|
| 788 |
+
cursor: pointer;
|
| 789 |
+
transition: border-color 120ms ease, background-color 120ms ease;
|
| 790 |
+
}
|
| 791 |
+
|
| 792 |
+
.sp-logout-link:hover {
|
| 793 |
+
border-color: rgba(255, 75, 75, 0.78);
|
| 794 |
+
background: rgba(255, 255, 255, 0.035);
|
| 795 |
+
color: var(--text-color, #f3f4f6) !important;
|
| 796 |
+
}
|
| 797 |
+
|
| 798 |
+
.sp-logout-link:focus-visible {
|
| 799 |
+
outline: 2px solid rgba(255, 75, 75, 0.85);
|
| 800 |
+
outline-offset: 2px;
|
| 801 |
+
}
|
| 802 |
+
|
| 803 |
+
@media (max-width: 1150px) {
|
| 804 |
+
.sp-account-card {
|
| 805 |
+
font-size: 0.75rem;
|
| 806 |
+
gap: 0.3rem;
|
| 807 |
+
padding-left: 0.48rem;
|
| 808 |
+
padding-right: 0.48rem;
|
| 809 |
+
}
|
| 810 |
+
|
| 811 |
+
.sp-logout-link {
|
| 812 |
+
min-width: 5.8rem;
|
| 813 |
+
font-size: 0.78rem;
|
| 814 |
+
padding-left: 0.46rem;
|
| 815 |
+
padding-right: 0.46rem;
|
| 816 |
+
}
|
| 817 |
+
}
|
| 818 |
+
|
| 819 |
+
@media (max-width: 760px) {
|
| 820 |
+
.sp-account-card {
|
| 821 |
+
justify-content: flex-start;
|
| 822 |
+
overflow-x: auto;
|
| 823 |
+
scrollbar-width: none;
|
| 824 |
+
}
|
| 825 |
+
|
| 826 |
+
.sp-account-card::-webkit-scrollbar {
|
| 827 |
+
display: none;
|
| 828 |
+
}
|
| 829 |
+
}
|
| 830 |
+
</style>
|
| 831 |
+
""",
|
| 832 |
+
unsafe_allow_html=True,
|
| 833 |
+
)
|
| 834 |
+
|
| 835 |
+
info_col, logout_col = st.columns([5.9, 1.65])
|
| 836 |
+
with info_col:
|
| 837 |
+
if product_mode:
|
| 838 |
+
balance_markup = (
|
| 839 |
+
f"<span>{html.escape(tr(language, 'account_chip_standard', value=standard_remaining))}</span>"
|
| 840 |
+
f'<span class="sp-account-separator">•</span>'
|
| 841 |
+
f"<span>{html.escape(tr(language, 'account_chip_flagship', value=flagship_remaining))}</span>"
|
| 842 |
+
)
|
| 843 |
+
else:
|
| 844 |
+
balance_markup = (
|
| 845 |
+
f"<span>{html.escape(tr(language, 'account_chip_free', value=free_remaining))}</span>"
|
| 846 |
+
f'<span class="sp-account-separator">•</span>'
|
| 847 |
+
f"<span>{html.escape(tr(language, 'account_chip_bonus', value=bonus_papers))}</span>"
|
| 848 |
+
)
|
| 849 |
+
st.markdown(
|
| 850 |
+
f"""
|
| 851 |
+
<div class="sp-account-card">
|
| 852 |
+
<span>{html.escape(tr(language, 'account_chip_user', value=user_id))}</span>
|
| 853 |
+
<span class="sp-account-separator">•</span>
|
| 854 |
+
{balance_markup}
|
| 855 |
+
</div>
|
| 856 |
+
""",
|
| 857 |
+
unsafe_allow_html=True,
|
| 858 |
+
)
|
| 859 |
+
with logout_col:
|
| 860 |
+
logout_label = html.escape(tr(language, "logout_short"))
|
| 861 |
+
logout_help = html.escape(tr(language, "logout"), quote=True)
|
| 862 |
+
st.markdown(
|
| 863 |
+
f'<a class="sp-logout-link" href="/edge/logout" target="_self" '
|
| 864 |
+
f'rel="nofollow" role="button" aria-label="{logout_help}" '
|
| 865 |
+
f'title="{logout_help}">{logout_label}</a>',
|
| 866 |
+
unsafe_allow_html=True,
|
| 867 |
+
)
|
huggingface/requirements.txt
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
streamlit==1.38.0
|
| 2 |
+
requests>=2.31,<3
|
| 3 |
+
python-docx>=1.1,<2
|
| 4 |
+
pandas>=2.1,<3
|
| 5 |
+
numpy>=1.26,<3
|
| 6 |
+
scikit-learn>=1.4,<2
|
| 7 |
+
tiktoken>=0.7,<1
|
| 8 |
+
langdetect==1.0.9
|
| 9 |
+
mistune>=3,<4
|
| 10 |
+
beautifulsoup4>=4.12,<5
|
| 11 |
+
PyMuPDF>=1.24,<2
|
| 12 |
+
openpyxl>=3.1,<4
|
| 13 |
+
tabulate>=0.9,<1
|
| 14 |
+
qrcode[pil]>=8,<9
|
| 15 |
+
|
| 16 |
+
huggingface_hub>=1.16,<2
|
huggingface/task_executor.py
CHANGED
|
@@ -29,6 +29,7 @@ import requests
|
|
| 29 |
|
| 30 |
from config import APP_CONFIG
|
| 31 |
from core.checkpoint_store import TaskCheckpointStore, checkpoint_authority_lock
|
|
|
|
| 32 |
|
| 33 |
_log_handlers = [logging.StreamHandler(sys.stdout)]
|
| 34 |
_executor_log_path = str(os.environ.get("SMARTPAPER_EXECUTOR_LOG_PATH") or "").strip()
|
|
@@ -125,34 +126,69 @@ def _is_local_checkpoint_authority_failure(exc: Exception) -> bool:
|
|
| 125 |
return isinstance(exc, RuntimeError) and str(exc or "").startswith("checkpoint_authority_")
|
| 126 |
|
| 127 |
|
| 128 |
-
def
|
| 129 |
-
"""Classify
|
| 130 |
-
|
| 131 |
-
This deliberately avoids importing the generation pipeline at module import
|
| 132 |
-
time. Provider/model exception class names are inspected only after
|
| 133 |
-
claim/bootstrap has succeeded, so future provider adapters do not require
|
| 134 |
-
executor imports or vendor-specific branching.
|
| 135 |
-
"""
|
| 136 |
if isinstance(exc, ExecutorApiError):
|
| 137 |
-
return exc.status_code == 429 or exc.status_code >= 500
|
| 138 |
class_name = exc.__class__.__name__
|
|
|
|
|
|
|
| 139 |
if class_name in {
|
| 140 |
-
"DeepSeekAuthenticationError", "DeepSeekBalanceError",
|
| 141 |
-
"ModelAuthenticationError", "ModelBalanceError",
|
| 142 |
-
"ModelProviderUnavailableError", "ModelGatewayError",
|
| 143 |
}:
|
| 144 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
if isinstance(exc, (requests.exceptions.Timeout, requests.exceptions.ConnectionError)):
|
| 146 |
-
return
|
| 147 |
-
if
|
| 148 |
-
|
| 149 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
transient_markers = (
|
| 151 |
"connection", "timed out", "timeout", "temporarily unavailable",
|
| 152 |
"streaming failed after", "api failed after", "stream was interrupted",
|
| 153 |
"insufficient eligible references",
|
| 154 |
)
|
| 155 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
|
| 157 |
|
| 158 |
class ExecutorApi:
|
|
@@ -415,6 +451,23 @@ def _snapshot_nullable_version(value):
|
|
| 415 |
return text or None
|
| 416 |
|
| 417 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 418 |
def _trusted_task_policy_snapshot_envelope(snapshot: dict, resolved: dict) -> dict:
|
| 419 |
return {
|
| 420 |
"task_id": str(snapshot.get("task_id") or ""),
|
|
@@ -486,6 +539,7 @@ def _validate_trusted_task_policy_snapshot(claim: dict, task: dict, request_data
|
|
| 486 |
if _snapshot_nullable_version(resolved.get(field)) != _snapshot_nullable_version(snapshot.get(field)):
|
| 487 |
raise RuntimeError("task_policy_snapshot_policy_version_mismatch")
|
| 488 |
|
|
|
|
| 489 |
if str(resolved.get("manuscript_mode") or "").strip().lower() != str(task.get("mode") or "").strip().lower():
|
| 490 |
raise RuntimeError("task_policy_snapshot_mode_mismatch")
|
| 491 |
target_words = int(task.get("target_words") or 0)
|
|
@@ -675,6 +729,7 @@ def main() -> int:
|
|
| 675 |
state.stop_heartbeat()
|
| 676 |
return 2
|
| 677 |
|
|
|
|
| 678 |
try:
|
| 679 |
# Heavy paper modules are deliberately imported only after D1 claim and
|
| 680 |
# the local claim handshake. Any import/init failure is now observable
|
|
@@ -700,6 +755,7 @@ def main() -> int:
|
|
| 700 |
logging.warning("[SP_EXECUTOR] final_heartbeat_error task_id=%s error=%s", args.task_id, _safe(exc, 180))
|
| 701 |
if state.cancelled():
|
| 702 |
raise RuntimeError("task_cancel_requested")
|
|
|
|
| 703 |
api.finish(
|
| 704 |
{
|
| 705 |
"task_id": args.task_id,
|
|
@@ -708,7 +764,7 @@ def main() -> int:
|
|
| 708 |
"executor_instance_id": instance_id,
|
| 709 |
"status": "completed",
|
| 710 |
"current_node": "package_validated",
|
| 711 |
-
"output_filename":
|
| 712 |
# R14-TQ2-PQ1-R3-R1: completed tasks may carry a nonfatal,
|
| 713 |
# backend-trusted quality warning. Worker/D1 already persists
|
| 714 |
# safe_error_code/detail for completed terminal transitions;
|
|
@@ -717,6 +773,28 @@ def main() -> int:
|
|
| 717 |
"safe_error_detail": str(result.get("safe_error_detail") or "")[:800],
|
| 718 |
}
|
| 719 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 720 |
logging.info("[SP_EXECUTOR] completed task_id=%s epoch=%s", args.task_id, epoch)
|
| 721 |
return 0
|
| 722 |
except Exception as exc:
|
|
@@ -742,6 +820,15 @@ def main() -> int:
|
|
| 742 |
logging.exception("[SP_EXECUTOR] recoverable_interruption task_id=%s epoch=%s error=%s", args.task_id, epoch, _safe(exc, 240))
|
| 743 |
return 75
|
| 744 |
logging.exception("[SP_EXECUTOR] failed task_id=%s epoch=%s error=%s", args.task_id, epoch, _safe(exc, 240))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 745 |
try:
|
| 746 |
api.finish(
|
| 747 |
{
|
|
@@ -753,6 +840,7 @@ def main() -> int:
|
|
| 753 |
"safe_error_code": "E_GENERATION_FAILED",
|
| 754 |
"safe_error_detail": _safe(exc, 800),
|
| 755 |
"current_node": "generation_failed",
|
|
|
|
| 756 |
}
|
| 757 |
)
|
| 758 |
except Exception as finish_exc:
|
|
|
|
| 29 |
|
| 30 |
from config import APP_CONFIG
|
| 31 |
from core.checkpoint_store import TaskCheckpointStore, checkpoint_authority_lock
|
| 32 |
+
from core.advanced_method_registry import METHOD_FAMILIES, method_capability
|
| 33 |
|
| 34 |
_log_handlers = [logging.StreamHandler(sys.stdout)]
|
| 35 |
_executor_log_path = str(os.environ.get("SMARTPAPER_EXECUTOR_LOG_PATH") or "").strip()
|
|
|
|
| 126 |
return isinstance(exc, RuntimeError) and str(exc or "").startswith("checkpoint_authority_")
|
| 127 |
|
| 128 |
|
| 129 |
+
def _generation_failure_family(exc: Exception) -> str:
|
| 130 |
+
"""Classify post-claim execution failures for the P0 delivery-first contract."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
if isinstance(exc, ExecutorApiError):
|
| 132 |
+
return "TRANSPORT" if exc.status_code == 429 or exc.status_code >= 500 else "AUTH_POLICY_SECURITY"
|
| 133 |
class_name = exc.__class__.__name__
|
| 134 |
+
text = str(exc or "").casefold()
|
| 135 |
+
|
| 136 |
if class_name in {
|
| 137 |
+
"DeepSeekAuthenticationError", "DeepSeekBalanceError",
|
| 138 |
+
"ModelAuthenticationError", "ModelBalanceError",
|
|
|
|
| 139 |
}:
|
| 140 |
+
return "AUTH_POLICY_SECURITY"
|
| 141 |
+
stable_policy_markers = (
|
| 142 |
+
"trusted_product_policy", "trusted_product_route_", "request_hash_mismatch",
|
| 143 |
+
"policy_snapshot", "entitlement", "authentication", "balance is insufficient",
|
| 144 |
+
"rejected the request format (400)", "not found (404)", "parameters are invalid (422)",
|
| 145 |
+
"unknown or unconfigured deepseek model role", "no deepseek model configured",
|
| 146 |
+
)
|
| 147 |
+
if any(marker in text for marker in stable_policy_markers):
|
| 148 |
+
return "AUTH_POLICY_SECURITY"
|
| 149 |
if isinstance(exc, (requests.exceptions.Timeout, requests.exceptions.ConnectionError)):
|
| 150 |
+
return "TRANSPORT"
|
| 151 |
+
if isinstance(exc, requests.exceptions.HTTPError):
|
| 152 |
+
response = getattr(exc, "response", None)
|
| 153 |
+
status = int(getattr(response, "status_code", 0) or 0)
|
| 154 |
+
if status in {408, 409, 425, 429, 500, 502, 503, 504}:
|
| 155 |
+
return "TRANSPORT"
|
| 156 |
+
if class_name in {"ModelProviderUnavailableError", "DeepSeekAPIError"}:
|
| 157 |
+
return "TRANSPORT"
|
| 158 |
+
if class_name == "DeepSeekRequestError":
|
| 159 |
+
content_validation_markers = (
|
| 160 |
+
"final abstract", "front matter", "translation", "citation", "hypothesis",
|
| 161 |
+
"manuscript", "qualitative", "glossary", "marker mismatch", "semantic",
|
| 162 |
+
"must not contain", "empty after validation", "structured validation",
|
| 163 |
+
)
|
| 164 |
+
return "CONTENT_VALIDATION" if any(marker in text for marker in content_validation_markers) else "AUTH_POLICY_SECURITY"
|
| 165 |
+
if class_name == "ModelRequestError":
|
| 166 |
+
# Gateway request errors are trusted route/request-shape failures; provider
|
| 167 |
+
# availability has its own recoverable exception class.
|
| 168 |
+
return "AUTH_POLICY_SECURITY"
|
| 169 |
+
if class_name == "ModelGatewayError":
|
| 170 |
+
transient_markers = ("unavailable", "timeout", "timed out", "connection", "provider")
|
| 171 |
+
return "TRANSPORT" if any(marker in text for marker in transient_markers) else "CONTENT_VALIDATION"
|
| 172 |
transient_markers = (
|
| 173 |
"connection", "timed out", "timeout", "temporarily unavailable",
|
| 174 |
"streaming failed after", "api failed after", "stream was interrupted",
|
| 175 |
"insufficient eligible references",
|
| 176 |
)
|
| 177 |
+
if any(marker in text for marker in transient_markers):
|
| 178 |
+
return "TRANSPORT"
|
| 179 |
+
return "TERMINAL_INTEGRITY"
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def _is_recoverable_generation_failure(exc: Exception) -> bool:
|
| 183 |
+
"""Return True only for failures that should resume instead of refund/fail.
|
| 184 |
+
|
| 185 |
+
CONTENT_VALIDATION reaches this layer only after the runner's local repair/
|
| 186 |
+
fallback paths were unable to complete the current attempt. A new executor
|
| 187 |
+
attempt is still preferable to destroying the paper because model output is
|
| 188 |
+
nondeterministic; stable auth/policy/request-shape faults remain terminal.
|
| 189 |
+
"""
|
| 190 |
+
return _generation_failure_family(exc) in {"TRANSPORT", "CONTENT_VALIDATION"}
|
| 191 |
+
|
| 192 |
|
| 193 |
|
| 194 |
class ExecutorApi:
|
|
|
|
| 451 |
return text or None
|
| 452 |
|
| 453 |
|
| 454 |
+
def _validate_request_method_family(request_data: dict, task: dict) -> None:
|
| 455 |
+
"""Validate new U1-R2 method identity while preserving legacy requests."""
|
| 456 |
+
raw_family = str((request_data or {}).get("method_family") or "").strip().casefold()
|
| 457 |
+
if not raw_family:
|
| 458 |
+
return
|
| 459 |
+
if raw_family not in METHOD_FAMILIES:
|
| 460 |
+
raise RuntimeError("request_method_family_invalid")
|
| 461 |
+
capability = method_capability(raw_family)
|
| 462 |
+
expected_lane = str(capability.get("legacy_policy_lane") or "").strip().casefold()
|
| 463 |
+
request_lane = str((request_data or {}).get("derived_policy_lane") or "").strip().casefold()
|
| 464 |
+
task_lane = str((task or {}).get("mode") or "").strip().casefold()
|
| 465 |
+
if expected_lane not in {"amj", "asq"} or request_lane != expected_lane or task_lane != expected_lane:
|
| 466 |
+
raise RuntimeError("request_method_family_lane_mismatch")
|
| 467 |
+
if str(capability.get("capability_status") or "").strip().casefold() != "production_supported":
|
| 468 |
+
raise RuntimeError("request_method_family_capability_blocked")
|
| 469 |
+
|
| 470 |
+
|
| 471 |
def _trusted_task_policy_snapshot_envelope(snapshot: dict, resolved: dict) -> dict:
|
| 472 |
return {
|
| 473 |
"task_id": str(snapshot.get("task_id") or ""),
|
|
|
|
| 539 |
if _snapshot_nullable_version(resolved.get(field)) != _snapshot_nullable_version(snapshot.get(field)):
|
| 540 |
raise RuntimeError("task_policy_snapshot_policy_version_mismatch")
|
| 541 |
|
| 542 |
+
_validate_request_method_family(request_data, task)
|
| 543 |
if str(resolved.get("manuscript_mode") or "").strip().lower() != str(task.get("mode") or "").strip().lower():
|
| 544 |
raise RuntimeError("task_policy_snapshot_mode_mismatch")
|
| 545 |
target_words = int(task.get("target_words") or 0)
|
|
|
|
| 729 |
state.stop_heartbeat()
|
| 730 |
return 2
|
| 731 |
|
| 732 |
+
runner = None
|
| 733 |
try:
|
| 734 |
# Heavy paper modules are deliberately imported only after D1 claim and
|
| 735 |
# the local claim handshake. Any import/init failure is now observable
|
|
|
|
| 755 |
logging.warning("[SP_EXECUTOR] final_heartbeat_error task_id=%s error=%s", args.task_id, _safe(exc, 180))
|
| 756 |
if state.cancelled():
|
| 757 |
raise RuntimeError("task_cancel_requested")
|
| 758 |
+
output_filename = str(result.get("output_filename") or "")[:240]
|
| 759 |
api.finish(
|
| 760 |
{
|
| 761 |
"task_id": args.task_id,
|
|
|
|
| 764 |
"executor_instance_id": instance_id,
|
| 765 |
"status": "completed",
|
| 766 |
"current_node": "package_validated",
|
| 767 |
+
"output_filename": output_filename,
|
| 768 |
# R14-TQ2-PQ1-R3-R1: completed tasks may carry a nonfatal,
|
| 769 |
# backend-trusted quality warning. Worker/D1 already persists
|
| 770 |
# safe_error_code/detail for completed terminal transitions;
|
|
|
|
| 773 |
"safe_error_detail": str(result.get("safe_error_detail") or "")[:800],
|
| 774 |
}
|
| 775 |
)
|
| 776 |
+
# P0-R2: private HF Bucket storage is a best-effort recovery layer only.
|
| 777 |
+
# The authoritative completed/quota transition above must succeed first;
|
| 778 |
+
# any storage failure below is nonfatal and cannot alter task accounting.
|
| 779 |
+
try:
|
| 780 |
+
from hf_bucket_store import archive_completed_zip
|
| 781 |
+
|
| 782 |
+
archive_result = archive_completed_zip(
|
| 783 |
+
user_id=args.user_id,
|
| 784 |
+
task_id=args.task_id,
|
| 785 |
+
output_filename=output_filename,
|
| 786 |
+
output_path=str(result.get("output_path") or ""),
|
| 787 |
+
)
|
| 788 |
+
if archive_result.get("stored"):
|
| 789 |
+
logging.info(
|
| 790 |
+
"[SP_EXECUTOR] bucket_backup_ok task_id=%s bytes=%s idempotent=%s",
|
| 791 |
+
args.task_id, archive_result.get("size_bytes"), bool(archive_result.get("idempotent")),
|
| 792 |
+
)
|
| 793 |
+
except Exception as exc:
|
| 794 |
+
logging.warning(
|
| 795 |
+
"[SP_EXECUTOR] bucket_backup_nonfatal task_id=%s error=%s",
|
| 796 |
+
args.task_id, type(exc).__name__,
|
| 797 |
+
)
|
| 798 |
logging.info("[SP_EXECUTOR] completed task_id=%s epoch=%s", args.task_id, epoch)
|
| 799 |
return 0
|
| 800 |
except Exception as exc:
|
|
|
|
| 820 |
logging.exception("[SP_EXECUTOR] recoverable_interruption task_id=%s epoch=%s error=%s", args.task_id, epoch, _safe(exc, 240))
|
| 821 |
return 75
|
| 822 |
logging.exception("[SP_EXECUTOR] failed task_id=%s epoch=%s error=%s", args.task_id, epoch, _safe(exc, 240))
|
| 823 |
+
recovery = {}
|
| 824 |
+
if runner is not None:
|
| 825 |
+
try:
|
| 826 |
+
recovery = dict(runner.build_recovery_artifact(exc) or {})
|
| 827 |
+
except Exception as recovery_exc:
|
| 828 |
+
logging.warning(
|
| 829 |
+
"[SP_EXECUTOR] recovery_package_failed task_id=%s error=%s",
|
| 830 |
+
args.task_id, _safe(recovery_exc, 220),
|
| 831 |
+
)
|
| 832 |
try:
|
| 833 |
api.finish(
|
| 834 |
{
|
|
|
|
| 840 |
"safe_error_code": "E_GENERATION_FAILED",
|
| 841 |
"safe_error_detail": _safe(exc, 800),
|
| 842 |
"current_node": "generation_failed",
|
| 843 |
+
"output_filename": str(recovery.get("output_filename") or "")[:240],
|
| 844 |
}
|
| 845 |
)
|
| 846 |
except Exception as finish_exc:
|