Spaces:
Running
Running
Upload 91 files
Browse files- app.py +178 -104
- comment_store.py +101 -0
- db_utils.py +130 -33
- models.py +11 -0
- owned_thread.py +59 -17
- periodic_tasks.py +59 -0
- profile_io.py +106 -0
- proxy_resources.py +15 -0
- rate_limit.py +13 -4
- resource_deletion.py +35 -6
- resource_publication.py +2 -1
- router_comments.py +2 -2
- router_items.py +32 -27
- router_messages.py +9 -16
- router_posts.py +25 -37
- router_prompts.py +23 -34
- router_proxy.py +9 -5
- router_tasks.py +7 -24
- router_users_profile.py +23 -3
- router_wallet.py +12 -10
- safe_http_transport.py +3 -1
- script_execution.py +160 -0
- task_projection.py +52 -23
- task_repair.py +112 -106
- user_store.py +18 -0
- zip_validation.py +2 -1
- 云端_定时版本检测引擎.py +50 -19
- 数据库连接.py +96 -31
app.py
CHANGED
|
@@ -20,6 +20,11 @@ import threading
|
|
| 20 |
import 数据库连接 as db
|
| 21 |
import asyncio
|
| 22 |
from 安全认证 import require_auth
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
|
| 24 |
|
| 25 |
# ==========================================
|
|
@@ -228,61 +233,78 @@ def detailed_health_check():
|
|
| 228 |
@app.on_event("startup")
|
| 229 |
async def on_startup():
|
| 230 |
logger.info("🚀 ComfyUI-Ranking API 启动中...")
|
| 231 |
-
|
| 232 |
-
|
|
|
|
|
|
|
|
|
|
| 233 |
try:
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
# 不抛出异常,允许降级运行
|
| 239 |
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
|
| 287 |
|
| 288 |
@app.on_event("shutdown")
|
|
@@ -290,33 +312,43 @@ async def on_shutdown():
|
|
| 290 |
"""优雅关闭,清理资源"""
|
| 291 |
logger.info("🛑 ComfyUI-Ranking API 正在关闭...")
|
| 292 |
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 320 |
|
| 321 |
# 🛡️ CORS 配置:默认通配符(插件前端运行于用户本地 ComfyUI 页面,origin 为
|
| 322 |
# 127.0.0.1/localhost/局域网 IP/自定义端口等无法穷举;本 API 通过 Authorization 头
|
|
@@ -418,7 +450,7 @@ def _is_admin(account: str) -> bool:
|
|
| 418 |
# 配置存储文件
|
| 419 |
SYSTEM_CONFIG_FILE = "/tmp/system_config.json" # 保留作为降级方案
|
| 420 |
|
| 421 |
-
def _load_system_config_from_db(db: Session) -> dict:
|
| 422 |
"""从数据库加载所有系统配置"""
|
| 423 |
try:
|
| 424 |
configs = db.query(SystemConfig).all()
|
|
@@ -431,10 +463,18 @@ def _load_system_config_from_db(db: Session) -> dict:
|
|
| 431 |
return result
|
| 432 |
except Exception as e:
|
| 433 |
logger.warning(f"从数据库加载系统配置失败,降级到文件: error_type={type(e).__name__}")
|
|
|
|
|
|
|
| 434 |
return _load_system_config_from_file()
|
| 435 |
|
| 436 |
-
def _load_system_config_from_file() -> dict:
|
| 437 |
-
"""降级方案
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 438 |
try:
|
| 439 |
if os.path.exists(SYSTEM_CONFIG_FILE):
|
| 440 |
with open(SYSTEM_CONFIG_FILE, "r") as f:
|
|
@@ -443,18 +483,25 @@ def _load_system_config_from_file() -> dict:
|
|
| 443 |
logger.debug(f"加载系统配置文件失败: error_type={type(e).__name__}")
|
| 444 |
return {}
|
| 445 |
|
| 446 |
-
def _save_system_config_to_db(db: Session, config_key: str, value, updated_by: str = None):
|
| 447 |
"""保存单个配置项到数据库"""
|
| 448 |
try:
|
| 449 |
cfg = db.query(SystemConfig).filter(SystemConfig.config_key == config_key).first()
|
| 450 |
if not cfg:
|
| 451 |
cfg = SystemConfig(config_key=config_key)
|
| 452 |
db.add(cfg)
|
| 453 |
-
cfg.config_value = json.dumps(value, ensure_ascii=False)
|
| 454 |
cfg.updated_by = updated_by
|
| 455 |
cfg.updated_at = datetime.datetime.utcnow()
|
| 456 |
db.commit()
|
| 457 |
except Exception as e:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 458 |
db.rollback()
|
| 459 |
logger.warning(f"保存系统配置到数据库失败,降级到文件: error_type={type(e).__name__}")
|
| 460 |
# 降级:保存到文件
|
|
@@ -467,8 +514,16 @@ def _save_system_config_to_db(db: Session, config_key: str, value, updated_by: s
|
|
| 467 |
logger.warning(f"降级保存系统配置文件也失败: error_type={type(fe).__name__}")
|
| 468 |
|
| 469 |
# 兼容旧调用的包装函数
|
| 470 |
-
def _load_system_config(db: Session = None) -> dict:
|
| 471 |
-
"""加载系统配置(优先数据库,降级文件)"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 472 |
if db:
|
| 473 |
return _load_system_config_from_db(db)
|
| 474 |
# 无 db session 时(如公开端点),创建临时 session
|
|
@@ -504,8 +559,15 @@ async def get_system_config(config_key: str, current_user: str = Depends(require
|
|
| 504 |
if not _is_admin(current_user):
|
| 505 |
raise HTTPException(status_code=403, detail="无权访问系统配置,仅管理员可操作")
|
| 506 |
|
| 507 |
-
|
| 508 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 509 |
|
| 510 |
|
| 511 |
# 🏷️ 版本配置默认值
|
|
@@ -528,20 +590,29 @@ async def get_public_project_version(db: Session = Depends(get_db)):
|
|
| 528 |
@app.get("/api/public/banner_config")
|
| 529 |
async def get_public_banner_config(db: Session = Depends(get_db)):
|
| 530 |
"""公开接口:获取广告横幅配置(所有已登录/未登录用户均可访问)"""
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 545 |
|
| 546 |
|
| 547 |
@app.put("/api/admin/config/{config_key}")
|
|
@@ -550,7 +621,10 @@ async def set_system_config(config_key: str, value: dict, current_user: str = De
|
|
| 550 |
# 🔒 权限验证
|
| 551 |
if not _is_admin(current_user):
|
| 552 |
raise HTTPException(status_code=403, detail="无权修改系统配置,仅管理员可操作")
|
| 553 |
-
|
|
|
|
|
|
|
|
|
|
| 554 |
return {"status": "success", "message": "配置已更新"}
|
| 555 |
|
| 556 |
|
|
|
|
| 20 |
import 数据库连接 as db
|
| 21 |
import asyncio
|
| 22 |
from 安全认证 import require_auth
|
| 23 |
+
from profile_io import start_profile_io, close_profile_io
|
| 24 |
+
from script_execution import start_admin_scripts, stop_admin_scripts, close_admin_scripts
|
| 25 |
+
from comment_store import start_comment_reads, stop_comment_reads, close_comment_reads
|
| 26 |
+
from periodic_tasks import start_periodic_task, request_periodic_stop, wait_periodic_tasks
|
| 27 |
+
from owned_thread import run_owned_thread
|
| 28 |
|
| 29 |
|
| 30 |
# ==========================================
|
|
|
|
| 233 |
@app.on_event("startup")
|
| 234 |
async def on_startup():
|
| 235 |
logger.info("🚀 ComfyUI-Ranking API 启动中...")
|
| 236 |
+
if getattr(app.state, "_ranking_starting", False) or getattr(app.state, "_ranking_closers", 0):
|
| 237 |
+
raise RuntimeError("application lifecycle is already in progress")
|
| 238 |
+
if getattr(app.state, "_ranking_started", False):
|
| 239 |
+
return
|
| 240 |
+
app.state._ranking_starting = True
|
| 241 |
try:
|
| 242 |
+
start_profile_io()
|
| 243 |
+
try:
|
| 244 |
+
start_comment_reads()
|
| 245 |
+
start_admin_scripts()
|
|
|
|
| 246 |
|
| 247 |
+
# ========== 预热检查:SQL 数据库 ==========
|
| 248 |
+
try:
|
| 249 |
+
init_sql_db()
|
| 250 |
+
logger.info("✅ SQL 数据库初始化完成")
|
| 251 |
+
except Exception as e:
|
| 252 |
+
logger.error(f"❌ SQL 数据库初始化失败: error_type={type(e).__name__}")
|
| 253 |
+
# 不抛出异常,允许降级运行
|
| 254 |
+
|
| 255 |
+
# ========== 加载第三方插件注册表(依赖 SQL 数据库) ==========
|
| 256 |
+
try:
|
| 257 |
+
from router_open_api import load_plugins_from_db
|
| 258 |
+
load_plugins_from_db()
|
| 259 |
+
logger.info("✅ 第三方插件注册表已加载")
|
| 260 |
+
except Exception as e:
|
| 261 |
+
logger.warning(f"⚠️ 第三方插件注册表加载失败: error_type={type(e).__name__}")
|
| 262 |
+
|
| 263 |
+
# ========== 预热检查:JSON 数据库 ==========
|
| 264 |
+
try:
|
| 265 |
+
db.load_data("users.json", default_data={})
|
| 266 |
+
db.load_data("items.json", default_data=[])
|
| 267 |
+
logger.info("✅ JSON 数据库访问正常")
|
| 268 |
+
except Exception as e:
|
| 269 |
+
logger.warning(f"⚠️ JSON 数据库访问异常: error_type={type(e).__name__}")
|
| 270 |
+
|
| 271 |
+
# SQL资金已冻结但JSON发布失败的任务,重启后继续投影;持有句柄供停机收尾。
|
| 272 |
+
from task_publication import publication_worker
|
| 273 |
+
from database_sql import engine
|
| 274 |
+
from db_utils import sort_cache
|
| 275 |
+
start_periodic_task(app.state, "task_publication",
|
| 276 |
+
lambda stop: publication_worker(engine, db, sort_cache.invalidate, stop))
|
| 277 |
+
|
| 278 |
+
from resource_publication import publication_worker as resource_publication_worker
|
| 279 |
+
start_periodic_task(app.state, "resource_publication",
|
| 280 |
+
lambda stop: resource_publication_worker(engine, db, sort_cache.invalidate, stop))
|
| 281 |
+
|
| 282 |
+
from resource_deletion import resource_deletion_worker
|
| 283 |
+
start_periodic_task(app.state, "resource_deletion",
|
| 284 |
+
lambda stop: resource_deletion_worker(db, stop, sort_cache.invalidate))
|
| 285 |
+
|
| 286 |
+
# ========== 启动后台任务 ==========
|
| 287 |
+
start_periodic_task(app.state, "daily_version", daily_version_check_task)
|
| 288 |
+
logger.info("✅ 定时版本检测任务已挂载")
|
| 289 |
+
|
| 290 |
+
start_periodic_task(app.state, "daily_views", reset_daily_views_task)
|
| 291 |
+
logger.info("✅ daily_views 每日重置任务已挂载")
|
| 292 |
+
|
| 293 |
+
# ========== 启动 HF 批量同步定时器 ==========
|
| 294 |
+
db.start_batch_sync()
|
| 295 |
+
logger.info("✅ HF 同步调度初始化已完成")
|
| 296 |
+
|
| 297 |
+
app.state._ranking_started = True
|
| 298 |
+
logger.info("🎉 ComfyUI-Ranking API 启动完成!")
|
| 299 |
+
except BaseException:
|
| 300 |
+
# 启动可能只登记了部分句柄;正常关闭按已登记对象收尾。
|
| 301 |
+
try:
|
| 302 |
+
await on_shutdown()
|
| 303 |
+
except BaseException as error:
|
| 304 |
+
logger.error("启动收尾失败: error_type=%s", type(error).__name__)
|
| 305 |
+
raise
|
| 306 |
+
finally:
|
| 307 |
+
app.state._ranking_starting = False
|
| 308 |
|
| 309 |
|
| 310 |
@app.on_event("shutdown")
|
|
|
|
| 312 |
"""优雅关闭,清理资源"""
|
| 313 |
logger.info("🛑 ComfyUI-Ranking API 正在关闭...")
|
| 314 |
|
| 315 |
+
app.state._ranking_closers = getattr(app.state, "_ranking_closers", 0) + 1
|
| 316 |
+
try:
|
| 317 |
+
app.state._ranking_started = False
|
| 318 |
+
stop_admin_scripts()
|
| 319 |
+
stop_comment_reads()
|
| 320 |
+
workers = request_periodic_stop(app.state, (
|
| 321 |
+
"daily_version", "daily_views", "task_publication", "resource_publication", "resource_deletion"))
|
| 322 |
+
|
| 323 |
+
# 关闭等待被取消也先收尾全部所属工作,之后传播取消;不提前最终同步。
|
| 324 |
+
cancelled = None
|
| 325 |
+
try:
|
| 326 |
+
await close_profile_io()
|
| 327 |
+
except asyncio.CancelledError as error:
|
| 328 |
+
cancelled = error
|
| 329 |
+
try:
|
| 330 |
+
await close_admin_scripts()
|
| 331 |
+
except asyncio.CancelledError as error:
|
| 332 |
+
cancelled = error
|
| 333 |
+
try:
|
| 334 |
+
await close_comment_reads()
|
| 335 |
+
except asyncio.CancelledError as error:
|
| 336 |
+
cancelled = error
|
| 337 |
+
try:
|
| 338 |
+
await wait_periodic_tasks(workers)
|
| 339 |
+
except asyncio.CancelledError as error:
|
| 340 |
+
cancelled = error
|
| 341 |
+
if cancelled is not None:
|
| 342 |
+
raise cancelled
|
| 343 |
+
|
| 344 |
+
# ========== 关闭前同步所有脏文件 ==========
|
| 345 |
+
await run_owned_thread(db.flush_sync)
|
| 346 |
+
logger.info("✅ HF 关闭前同步尝试已结束")
|
| 347 |
+
|
| 348 |
+
# 这里可以添加其他清理逻辑(如关闭连接池等)
|
| 349 |
+
logger.info("✅ 关闭完成")
|
| 350 |
+
finally:
|
| 351 |
+
app.state._ranking_closers -= 1
|
| 352 |
|
| 353 |
# 🛡️ CORS 配置:默认通配符(插件前端运行于用户本地 ComfyUI 页面,origin 为
|
| 354 |
# 127.0.0.1/localhost/局域网 IP/自定义端口等无法穷举;本 API 通过 Authorization 头
|
|
|
|
| 450 |
# 配置存储文件
|
| 451 |
SYSTEM_CONFIG_FILE = "/tmp/system_config.json" # 保留作为降级方案
|
| 452 |
|
| 453 |
+
def _load_system_config_from_db(db: Session, *, strict: bool = False) -> dict:
|
| 454 |
"""从数据库加载所有系统配置"""
|
| 455 |
try:
|
| 456 |
configs = db.query(SystemConfig).all()
|
|
|
|
| 463 |
return result
|
| 464 |
except Exception as e:
|
| 465 |
logger.warning(f"从数据库加载系统配置失败,降级到文件: error_type={type(e).__name__}")
|
| 466 |
+
if strict:
|
| 467 |
+
return _load_system_config_from_file(strict=True)
|
| 468 |
return _load_system_config_from_file()
|
| 469 |
|
| 470 |
+
def _load_system_config_from_file(*, strict: bool = False) -> dict:
|
| 471 |
+
"""降级方案;严格读取不把故障或缺失文件冒充空配置。"""
|
| 472 |
+
if strict:
|
| 473 |
+
with open(SYSTEM_CONFIG_FILE, "r", encoding="utf-8") as stream:
|
| 474 |
+
config = json.load(stream)
|
| 475 |
+
if not isinstance(config, dict):
|
| 476 |
+
raise ValueError("invalid system configuration root")
|
| 477 |
+
return config
|
| 478 |
try:
|
| 479 |
if os.path.exists(SYSTEM_CONFIG_FILE):
|
| 480 |
with open(SYSTEM_CONFIG_FILE, "r") as f:
|
|
|
|
| 483 |
logger.debug(f"加载系统配置文件失败: error_type={type(e).__name__}")
|
| 484 |
return {}
|
| 485 |
|
| 486 |
+
def _save_system_config_to_db(db: Session, config_key: str, value, updated_by: str = None, *, strict: bool = False):
|
| 487 |
"""保存单个配置项到数据库"""
|
| 488 |
try:
|
| 489 |
cfg = db.query(SystemConfig).filter(SystemConfig.config_key == config_key).first()
|
| 490 |
if not cfg:
|
| 491 |
cfg = SystemConfig(config_key=config_key)
|
| 492 |
db.add(cfg)
|
| 493 |
+
cfg.config_value = json.dumps(value, ensure_ascii=False, allow_nan=not strict)
|
| 494 |
cfg.updated_by = updated_by
|
| 495 |
cfg.updated_at = datetime.datetime.utcnow()
|
| 496 |
db.commit()
|
| 497 |
except Exception as e:
|
| 498 |
+
if strict:
|
| 499 |
+
# commit异常可能发生在真实提交之后;回滚不能证明尚未保存。
|
| 500 |
+
try:
|
| 501 |
+
db.rollback()
|
| 502 |
+
except Exception:
|
| 503 |
+
logger.warning("SYSTEM_CONFIG_ROLLBACK_UNCONFIRMED")
|
| 504 |
+
raise
|
| 505 |
db.rollback()
|
| 506 |
logger.warning(f"保存系统配置到数据库失败,降级到文件: error_type={type(e).__name__}")
|
| 507 |
# 降级:保存到文件
|
|
|
|
| 514 |
logger.warning(f"降级保存系统配置文件也失败: error_type={type(fe).__name__}")
|
| 515 |
|
| 516 |
# 兼容旧调用的包装函数
|
| 517 |
+
def _load_system_config(db: Session = None, *, strict: bool = False) -> dict:
|
| 518 |
+
"""加载系统配置(优先数据库,降级文件);严格模式保留故障。"""
|
| 519 |
+
if strict:
|
| 520 |
+
if db is not None:
|
| 521 |
+
return _load_system_config_from_db(db, strict=True)
|
| 522 |
+
temp_db = SessionLocal()
|
| 523 |
+
try:
|
| 524 |
+
return _load_system_config_from_db(temp_db, strict=True)
|
| 525 |
+
finally:
|
| 526 |
+
temp_db.close()
|
| 527 |
if db:
|
| 528 |
return _load_system_config_from_db(db)
|
| 529 |
# 无 db session 时(如公开端点),创建临时 session
|
|
|
|
| 559 |
if not _is_admin(current_user):
|
| 560 |
raise HTTPException(status_code=403, detail="无权访问系统配置,仅管理员可操作")
|
| 561 |
|
| 562 |
+
def reject_constant(_value):
|
| 563 |
+
raise ValueError("invalid configuration constant")
|
| 564 |
+
try:
|
| 565 |
+
# 管理回读用于确认SQL保存结果,不能用另一个来源的旧值代替。
|
| 566 |
+
row = db.query(SystemConfig).filter(SystemConfig.config_key == config_key).first()
|
| 567 |
+
value = None if row is None else json.loads(row.config_value, parse_constant=reject_constant)
|
| 568 |
+
except Exception:
|
| 569 |
+
raise HTTPException(status_code=503, detail="config_read_unavailable") from None
|
| 570 |
+
return {"status": "success", "data": value}
|
| 571 |
|
| 572 |
|
| 573 |
# 🏷️ 版本配置默认值
|
|
|
|
| 590 |
@app.get("/api/public/banner_config")
|
| 591 |
async def get_public_banner_config(db: Session = Depends(get_db)):
|
| 592 |
"""公开接口:获取广告横幅配置(所有已登录/未登录用户均可访问)"""
|
| 593 |
+
try:
|
| 594 |
+
config = _load_system_config(db, strict=True)
|
| 595 |
+
if not isinstance(config, dict):
|
| 596 |
+
raise ValueError("invalid system configuration root")
|
| 597 |
+
banner = config.get("banner_config")
|
| 598 |
+
if banner is None:
|
| 599 |
+
return {"status": "success", "data": None}
|
| 600 |
+
if not isinstance(banner, dict) or type(banner.get("enabled")) is not bool:
|
| 601 |
+
raise ValueError("invalid banner configuration")
|
| 602 |
+
if not banner["enabled"]:
|
| 603 |
+
return {"status": "success", "data": None}
|
| 604 |
+
# 只投影展示字段,异常配置不能触发浏览器清除旧横幅。
|
| 605 |
+
result = {"enabled": True}
|
| 606 |
+
for field in ("bannerImage", "title", "description", "detailImage", "detailContent", "link"):
|
| 607 |
+
value = banner.get(field, "")
|
| 608 |
+
if not isinstance(value, str):
|
| 609 |
+
raise ValueError("invalid banner display field")
|
| 610 |
+
result[field] = value
|
| 611 |
+
if not result["bannerImage"].strip():
|
| 612 |
+
raise ValueError("missing banner image")
|
| 613 |
+
return {"status": "success", "data": result}
|
| 614 |
+
except Exception:
|
| 615 |
+
raise HTTPException(status_code=503, detail="banner_config_unavailable") from None
|
| 616 |
|
| 617 |
|
| 618 |
@app.put("/api/admin/config/{config_key}")
|
|
|
|
| 621 |
# 🔒 权限验证
|
| 622 |
if not _is_admin(current_user):
|
| 623 |
raise HTTPException(status_code=403, detail="无权修改系统配置,仅管理员可操作")
|
| 624 |
+
try:
|
| 625 |
+
_save_system_config_to_db(db, config_key, value, updated_by=current_user, strict=True)
|
| 626 |
+
except Exception:
|
| 627 |
+
raise HTTPException(status_code=503, detail="config_save_unconfirmed") from None
|
| 628 |
return {"status": "success", "message": "配置已更新"}
|
| 629 |
|
| 630 |
|
comment_store.py
CHANGED
|
@@ -1,12 +1,59 @@
|
|
| 1 |
"""评论记录的锁内修改;路由负责认证、内容存在性及响应契约。"""
|
| 2 |
import copy
|
|
|
|
| 3 |
import logging
|
|
|
|
|
|
|
| 4 |
from fastapi import HTTPException
|
| 5 |
import 数据库连接 as db
|
| 6 |
from resource_deletion import resource_mutation_lock, assert_comment_target_active
|
| 7 |
from resource_store import ResourceStoreError
|
| 8 |
|
| 9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
def _item_comments(data, item_id):
|
| 11 |
if not isinstance(data, dict):
|
| 12 |
raise HTTPException(status_code=503, detail="评论数据暂不可用")
|
|
@@ -100,6 +147,60 @@ def _visible_count(records, depth=0):
|
|
| 100 |
return count
|
| 101 |
|
| 102 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
def attach_comment_counts(records):
|
| 104 |
"""只校对响应副本,以评论主记录恢复展示计数,不写内容或资金数据。"""
|
| 105 |
try:
|
|
|
|
| 1 |
"""评论记录的锁内修改;路由负责认证、内容存在性及响应契约。"""
|
| 2 |
import copy
|
| 3 |
+
import asyncio
|
| 4 |
import logging
|
| 5 |
+
import threading
|
| 6 |
+
from owned_thread import run_owned_thread
|
| 7 |
from fastapi import HTTPException
|
| 8 |
import 数据库连接 as db
|
| 9 |
from resource_deletion import resource_mutation_lock, assert_comment_target_active
|
| 10 |
from resource_store import ResourceStoreError
|
| 11 |
|
| 12 |
|
| 13 |
+
COMMENT_MAX_LENGTH = 5000
|
| 14 |
+
MAX_COMMENT_READS = 8
|
| 15 |
+
_comment_read_capacity = threading.BoundedSemaphore(MAX_COMMENT_READS)
|
| 16 |
+
_comment_read_gate = threading.Lock()
|
| 17 |
+
_comment_read_accepting = True
|
| 18 |
+
_comment_read_pending = 0
|
| 19 |
+
_comment_read_closers = 0
|
| 20 |
+
_comment_read_drained = threading.Event()
|
| 21 |
+
_comment_read_drained.set()
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def start_comment_reads():
|
| 25 |
+
global _comment_read_accepting
|
| 26 |
+
with _comment_read_gate:
|
| 27 |
+
if _comment_read_pending or _comment_read_closers:
|
| 28 |
+
raise RuntimeError("comment reads are still pending")
|
| 29 |
+
_comment_read_accepting = True
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def stop_comment_reads():
|
| 33 |
+
global _comment_read_accepting
|
| 34 |
+
with _comment_read_gate:
|
| 35 |
+
_comment_read_accepting = False
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
async def close_comment_reads():
|
| 39 |
+
global _comment_read_accepting, _comment_read_closers
|
| 40 |
+
with _comment_read_gate:
|
| 41 |
+
_comment_read_accepting = False
|
| 42 |
+
_comment_read_closers += 1
|
| 43 |
+
try:
|
| 44 |
+
cancelled = False
|
| 45 |
+
while not _comment_read_drained.is_set():
|
| 46 |
+
try:
|
| 47 |
+
await asyncio.sleep(.01)
|
| 48 |
+
except asyncio.CancelledError:
|
| 49 |
+
cancelled = True
|
| 50 |
+
if cancelled:
|
| 51 |
+
raise asyncio.CancelledError
|
| 52 |
+
finally:
|
| 53 |
+
with _comment_read_gate:
|
| 54 |
+
_comment_read_closers -= 1
|
| 55 |
+
|
| 56 |
+
|
| 57 |
def _item_comments(data, item_id):
|
| 58 |
if not isinstance(data, dict):
|
| 59 |
raise HTTPException(status_code=503, detail="评论数据暂不可用")
|
|
|
|
| 147 |
return count
|
| 148 |
|
| 149 |
|
| 150 |
+
def _validate_comment_text(records):
|
| 151 |
+
for record in records:
|
| 152 |
+
if not isinstance(record.get("author"), str) or not isinstance(record.get("content"), str):
|
| 153 |
+
raise ValueError("Invalid comment text")
|
| 154 |
+
_validate_comment_text(record.get("replies", []))
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def read_comment_thread(item_id):
|
| 158 |
+
"""严格读取评论快照;存储故障不冒充成功空列表,不改历史主记录。"""
|
| 159 |
+
try:
|
| 160 |
+
snapshot = db.load_data("comments.json", default_data={}, skip_cache=True, strict=True)
|
| 161 |
+
records = _item_comments(snapshot, item_id)
|
| 162 |
+
_visible_count(records)
|
| 163 |
+
_validate_comment_text(records)
|
| 164 |
+
return copy.deepcopy(records)
|
| 165 |
+
except Exception:
|
| 166 |
+
logging.getLogger(__name__).warning("评论列表无法核实")
|
| 167 |
+
raise HTTPException(status_code=503, detail="评论暂时无法读取") from None
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _read_comment_view(item_id):
|
| 171 |
+
records = read_comment_thread(item_id)
|
| 172 |
+
users = db.load_data("users.json", default_data={})
|
| 173 |
+
result = []
|
| 174 |
+
for record in records:
|
| 175 |
+
author_info = users.get(record["author"], {})
|
| 176 |
+
result.append({**record, "author_name": author_info.get("name", record["author"]),
|
| 177 |
+
"author_avatar": author_info.get("avatarDataUrl", "")})
|
| 178 |
+
return result
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
async def read_comment_view(item_id):
|
| 182 |
+
"""整次评论读取在所属后台工作中执行;繁忙不提交线程池队列。"""
|
| 183 |
+
global _comment_read_pending
|
| 184 |
+
with _comment_read_gate:
|
| 185 |
+
if not _comment_read_accepting or not _comment_read_capacity.acquire(blocking=False):
|
| 186 |
+
raise HTTPException(status_code=503, detail="评论暂时无法读取")
|
| 187 |
+
_comment_read_pending += 1
|
| 188 |
+
_comment_read_drained.clear()
|
| 189 |
+
try:
|
| 190 |
+
return await run_owned_thread(_read_comment_view, item_id)
|
| 191 |
+
except HTTPException:
|
| 192 |
+
raise
|
| 193 |
+
except Exception:
|
| 194 |
+
logging.getLogger(__name__).warning("评论展示无法核实")
|
| 195 |
+
raise HTTPException(status_code=503, detail="评论暂时无法读取") from None
|
| 196 |
+
finally:
|
| 197 |
+
with _comment_read_gate:
|
| 198 |
+
_comment_read_capacity.release()
|
| 199 |
+
_comment_read_pending -= 1
|
| 200 |
+
if not _comment_read_pending:
|
| 201 |
+
_comment_read_drained.set()
|
| 202 |
+
|
| 203 |
+
|
| 204 |
def attach_comment_counts(records):
|
| 205 |
"""只校对响应副本,以评论主记录恢复展示计数,不写内容或资金数据。"""
|
| 206 |
try:
|
db_utils.py
CHANGED
|
@@ -12,6 +12,8 @@
|
|
| 12 |
|
| 13 |
from typing import Any, Dict, List, Optional, Callable, Union
|
| 14 |
import copy
|
|
|
|
|
|
|
| 15 |
import 数据库连接 as db
|
| 16 |
|
| 17 |
|
|
@@ -19,6 +21,36 @@ import 数据库连接 as db
|
|
| 19 |
# 📖 查询工具函数
|
| 20 |
# ==========================================
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
def get_by_id(file_name: str, item_id: str, id_field: str = "id") -> Optional[Dict]:
|
| 23 |
"""
|
| 24 |
根据 ID 获取单个记录
|
|
@@ -478,17 +510,38 @@ class SortCache:
|
|
| 478 |
sort_cache.invalidate("items")
|
| 479 |
"""
|
| 480 |
|
| 481 |
-
def __init__(self, ttl: int = 300):
|
| 482 |
"""
|
| 483 |
初始化排序缓存
|
| 484 |
|
| 485 |
参数:
|
| 486 |
ttl: 缓存过期时间(秒),默认 300 秒(5分钟)
|
|
|
|
|
|
|
| 487 |
"""
|
| 488 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 489 |
self._ttl = ttl
|
| 490 |
-
|
| 491 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 492 |
"""
|
| 493 |
获取排序后的数据(带缓存)
|
| 494 |
|
|
@@ -496,30 +549,65 @@ class SortCache:
|
|
| 496 |
cache_key: 缓存键,应包含数据文件和排序参数
|
| 497 |
items: 原始数据列表(来自 load_data 的最新数据)
|
| 498 |
sort_fn: 排序函数,接收 items 列表并原地排序
|
|
|
|
|
|
|
| 499 |
|
| 500 |
返回:
|
| 501 |
排序后的 items 列表
|
| 502 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 503 |
now = time.time()
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 523 |
def invalidate(self, prefix: str = ""):
|
| 524 |
"""
|
| 525 |
清除缓存
|
|
@@ -527,23 +615,32 @@ class SortCache:
|
|
| 527 |
参数:
|
| 528 |
prefix: 缓存键前缀,如果指定则只清除匹配的缓存,否则清除所有缓存
|
| 529 |
"""
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 537 |
def get_stats(self) -> Dict[str, Any]:
|
| 538 |
"""
|
| 539 |
获取缓存统计信息(用于调试)
|
| 540 |
"""
|
| 541 |
now = time.time()
|
| 542 |
-
|
|
|
|
|
|
|
| 543 |
return {
|
| 544 |
-
"total_cached": len(
|
| 545 |
"valid": valid_count,
|
| 546 |
-
"expired": len(
|
| 547 |
"ttl": self._ttl
|
| 548 |
}
|
| 549 |
|
|
|
|
| 12 |
|
| 13 |
from typing import Any, Dict, List, Optional, Callable, Union
|
| 14 |
import copy
|
| 15 |
+
from collections import OrderedDict
|
| 16 |
+
import threading
|
| 17 |
import 数据库连接 as db
|
| 18 |
|
| 19 |
|
|
|
|
| 21 |
# 📖 查询工具函数
|
| 22 |
# ==========================================
|
| 23 |
|
| 24 |
+
def read_public_source(store, file_name, default_data):
|
| 25 |
+
"""公开读取拒绝故障默认值及异常集合;保留已验证文件缓存。"""
|
| 26 |
+
try:
|
| 27 |
+
data = store.load_data(file_name, default_data=default_data, strict=True)
|
| 28 |
+
if type(data) is not type(default_data):
|
| 29 |
+
raise ValueError("invalid public source root")
|
| 30 |
+
if isinstance(data, list):
|
| 31 |
+
if any(not isinstance(row, dict) or not row for row in data):
|
| 32 |
+
raise ValueError("invalid public source records")
|
| 33 |
+
elif file_name == "users.json":
|
| 34 |
+
if any(not isinstance(key, str) or not key or not isinstance(row, dict)
|
| 35 |
+
for key, row in data.items()):
|
| 36 |
+
raise ValueError("invalid public user records")
|
| 37 |
+
elif file_name == "comments.json":
|
| 38 |
+
if any(not isinstance(rows, list) or
|
| 39 |
+
any(not isinstance(row, dict) or not row for row in rows)
|
| 40 |
+
for rows in data.values()):
|
| 41 |
+
raise ValueError("invalid public comment records")
|
| 42 |
+
elif file_name == "versions.json":
|
| 43 |
+
if any(row is not None and not isinstance(row, (str, dict)) or
|
| 44 |
+
isinstance(row, dict) and row.get("hash") is not None and
|
| 45 |
+
not isinstance(row.get("hash"), str)
|
| 46 |
+
for row in data.values()):
|
| 47 |
+
raise ValueError("invalid public version records")
|
| 48 |
+
return data
|
| 49 |
+
except Exception:
|
| 50 |
+
from fastapi import HTTPException
|
| 51 |
+
raise HTTPException(status_code=503, detail="ranking_data_unavailable") from None
|
| 52 |
+
|
| 53 |
+
|
| 54 |
def get_by_id(file_name: str, item_id: str, id_field: str = "id") -> Optional[Dict]:
|
| 55 |
"""
|
| 56 |
根据 ID 获取单个记录
|
|
|
|
| 510 |
sort_cache.invalidate("items")
|
| 511 |
"""
|
| 512 |
|
| 513 |
+
def __init__(self, ttl: int = 300, max_entries: int = 128, max_ids: int = 100000):
|
| 514 |
"""
|
| 515 |
初始化排序缓存
|
| 516 |
|
| 517 |
参数:
|
| 518 |
ttl: 缓存过期时间(秒),默认 300 秒(5分钟)
|
| 519 |
+
max_entries: 最多保留的缓存条目,0 表示不保留
|
| 520 |
+
max_ids: 所有缓存合计保留的 ID 数量,0 表示不保留
|
| 521 |
"""
|
| 522 |
+
if any(type(value) is not int or value < 0 for value in (max_entries, max_ids)):
|
| 523 |
+
raise ValueError("sort cache limits must be nonnegative integers")
|
| 524 |
+
self._cache = OrderedDict() # {cache_key: (sorted_ids, timestamp)}
|
| 525 |
+
self._max_entries = max_entries
|
| 526 |
+
self._max_ids = max_ids
|
| 527 |
+
self._cached_ids = 0
|
| 528 |
self._ttl = ttl
|
| 529 |
+
self._lock = threading.Lock()
|
| 530 |
+
self._pending_sorts: Dict[str, object] = {}
|
| 531 |
+
self._read_epoch = object()
|
| 532 |
+
|
| 533 |
+
def capture_read_epoch(self):
|
| 534 |
+
"""在读取排序源数据前捕获;只标识本进程失效代次,不是文件版本。"""
|
| 535 |
+
with self._lock:
|
| 536 |
+
return self._read_epoch
|
| 537 |
+
|
| 538 |
+
def _remove_cached(self, key: str):
|
| 539 |
+
# 调用者持有 _lock;失效、过期和容量淘汰共用计数维护。
|
| 540 |
+
cached = self._cache.pop(key, None)
|
| 541 |
+
if cached is not None:
|
| 542 |
+
self._cached_ids -= len(cached[0])
|
| 543 |
+
|
| 544 |
+
def get_sorted(self, cache_key: str, items: List[Dict], sort_fn: Callable[[List[Dict]], None], *, read_epoch=None, fresh: bool = False) -> List[Dict]:
|
| 545 |
"""
|
| 546 |
获取排序后的数据(带缓存)
|
| 547 |
|
|
|
|
| 549 |
cache_key: 缓存键,应包含数据文件和排序参数
|
| 550 |
items: 原始数据列表(来自 load_data 的最新数据)
|
| 551 |
sort_fn: 排序函数,接收 items 列表并原地排序
|
| 552 |
+
read_epoch: 读取前捕获的代次;已失效输入只排序返回,不读写缓存
|
| 553 |
+
fresh: 按本次输入直接排序,不读写排序缓存或发布标记
|
| 554 |
|
| 555 |
返回:
|
| 556 |
排序后的 items 列表
|
| 557 |
"""
|
| 558 |
+
if fresh:
|
| 559 |
+
# 公开榜单源数据可能由其它进程更新,旧ID顺序不可作为当前排序依据。
|
| 560 |
+
# 直接排序比逐条序列化/摘要校验更轻;不触碰其它调用的缓存与代次。
|
| 561 |
+
sort_fn(items)
|
| 562 |
+
return items
|
| 563 |
now = time.time()
|
| 564 |
+
with self._lock:
|
| 565 |
+
stale_read = read_epoch is not None and read_epoch is not self._read_epoch
|
| 566 |
+
hit = False
|
| 567 |
+
if not stale_read:
|
| 568 |
+
cached = self._cache.get(cache_key)
|
| 569 |
+
hit = cached is not None and now - cached[1] < self._ttl
|
| 570 |
+
if hit:
|
| 571 |
+
self._cache.move_to_end(cache_key)
|
| 572 |
+
else:
|
| 573 |
+
# 未命中时机会性回收过期ID列表,不新增后台线程。
|
| 574 |
+
for key, (_, cached_time) in list(self._cache.items()):
|
| 575 |
+
if now - cached_time >= self._ttl:
|
| 576 |
+
self._remove_cached(key)
|
| 577 |
+
token = object()
|
| 578 |
+
# 同键后开始的排序取得发布权;旧计算只能返回自己的结果。
|
| 579 |
+
self._pending_sorts[cache_key] = token
|
| 580 |
+
if stale_read:
|
| 581 |
+
# 失效前读取的输入不能命中新代缓存,也不能撤销新代发布标记。
|
| 582 |
+
sort_fn(items)
|
| 583 |
+
return items
|
| 584 |
+
|
| 585 |
+
if hit:
|
| 586 |
+
sorted_ids, cached_time = cached
|
| 587 |
+
id_order = {id_: idx for idx, id_ in enumerate(sorted_ids)}
|
| 588 |
+
return sorted(items, key=lambda x: id_order.get(x.get("id"), float('inf')))
|
| 589 |
+
|
| 590 |
+
completed = False
|
| 591 |
+
try:
|
| 592 |
+
# 耗时排序及业务回调均在锁外,失效/其它键不会被阻塞。
|
| 593 |
+
sort_fn(items)
|
| 594 |
+
# 超预算仍返回原排序结果,不构造/保留额外ID列表,也不淘汰有效旧缓存。
|
| 595 |
+
sorted_ids = ([item.get("id") for item in items]
|
| 596 |
+
if self._max_entries and len(items) <= self._max_ids else [])
|
| 597 |
+
completed = True
|
| 598 |
+
return items
|
| 599 |
+
finally:
|
| 600 |
+
with self._lock:
|
| 601 |
+
if self._pending_sorts.get(cache_key) is token:
|
| 602 |
+
del self._pending_sorts[cache_key]
|
| 603 |
+
if completed and sorted_ids:
|
| 604 |
+
self._remove_cached(cache_key)
|
| 605 |
+
while (len(self._cache) >= self._max_entries or
|
| 606 |
+
self._cached_ids + len(sorted_ids) > self._max_ids):
|
| 607 |
+
self._remove_cached(next(iter(self._cache)))
|
| 608 |
+
self._cache[cache_key] = (sorted_ids, now)
|
| 609 |
+
self._cached_ids += len(sorted_ids)
|
| 610 |
+
|
| 611 |
def invalidate(self, prefix: str = ""):
|
| 612 |
"""
|
| 613 |
清除缓存
|
|
|
|
| 615 |
参数:
|
| 616 |
prefix: 缓存键前缀,如果指定则只清除匹配的缓存,否则清除所有缓存
|
| 617 |
"""
|
| 618 |
+
with self._lock:
|
| 619 |
+
self._read_epoch = object()
|
| 620 |
+
if prefix:
|
| 621 |
+
for key in list(self._cache):
|
| 622 |
+
if key.startswith(prefix):
|
| 623 |
+
self._remove_cached(key)
|
| 624 |
+
for key in list(self._pending_sorts):
|
| 625 |
+
if key.startswith(prefix):
|
| 626 |
+
del self._pending_sorts[key]
|
| 627 |
+
else:
|
| 628 |
+
self._cache.clear()
|
| 629 |
+
self._cached_ids = 0
|
| 630 |
+
self._pending_sorts.clear()
|
| 631 |
+
|
| 632 |
def get_stats(self) -> Dict[str, Any]:
|
| 633 |
"""
|
| 634 |
获取缓存统计信息(用于调试)
|
| 635 |
"""
|
| 636 |
now = time.time()
|
| 637 |
+
with self._lock:
|
| 638 |
+
snapshot = tuple(self._cache.values())
|
| 639 |
+
valid_count = sum(1 for _, ts in snapshot if now - ts < self._ttl)
|
| 640 |
return {
|
| 641 |
+
"total_cached": len(snapshot),
|
| 642 |
"valid": valid_count,
|
| 643 |
+
"expired": len(snapshot) - valid_count,
|
| 644 |
"ttl": self._ttl
|
| 645 |
}
|
| 646 |
|
models.py
CHANGED
|
@@ -69,6 +69,17 @@ class UserUpdate(BaseModel):
|
|
| 69 |
avatarDataUrl: Optional[str] = None
|
| 70 |
bannerUrl: Optional[str] = None # 🚀 个人资料卡背景图云端URL
|
| 71 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
# 🧹 P2清理:移除零引用的 PasswordReset 模型(重置密码实际走 UserUpdate/专用端点)
|
| 73 |
|
| 74 |
class CommentCreate(BaseModel):
|
|
|
|
| 69 |
avatarDataUrl: Optional[str] = None
|
| 70 |
bannerUrl: Optional[str] = None # 🚀 个人资料卡背景图云端URL
|
| 71 |
|
| 72 |
+
class SettingsPrivacy(BaseModel):
|
| 73 |
+
model_config = {"extra": "forbid"}
|
| 74 |
+
follows: bool = Field(strict=True)
|
| 75 |
+
likes: bool = Field(strict=True)
|
| 76 |
+
favorites: bool = Field(strict=True)
|
| 77 |
+
downloads: bool = Field(strict=True)
|
| 78 |
+
|
| 79 |
+
class UserSettings(UserUpdate):
|
| 80 |
+
model_config = {"extra": "forbid"}
|
| 81 |
+
privacy: SettingsPrivacy
|
| 82 |
+
|
| 83 |
# 🧹 P2清理:移除零引用的 PasswordReset 模型(重置密码实际走 UserUpdate/专用端点)
|
| 84 |
|
| 85 |
class CommentCreate(BaseModel):
|
owned_thread.py
CHANGED
|
@@ -1,33 +1,75 @@
|
|
| 1 |
-
"""等待所属同步工作
|
| 2 |
import asyncio
|
|
|
|
| 3 |
|
| 4 |
|
| 5 |
async def run_owned_thread(function, *args, **kwargs):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
async def finish():
|
| 7 |
-
# 被取消的shield在部分Python版本会立即记录内层异常正文。
|
| 8 |
-
# 先把常规线程结果封装为数据,所属调用再决定传播异常或取消。
|
| 9 |
try:
|
| 10 |
-
|
| 11 |
-
except
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
-
task = asyncio.create_task(finish())
|
| 15 |
cancelled = None
|
| 16 |
-
while not
|
| 17 |
try:
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
except asyncio.CancelledError as error:
|
| 20 |
cancelled = error
|
| 21 |
-
|
| 22 |
-
|
| 23 |
if cancelled is not None:
|
| 24 |
-
# 消费线程结果/异常,取消仍是本次调用的终态。
|
| 25 |
-
try:
|
| 26 |
-
task.result()
|
| 27 |
-
except Exception:
|
| 28 |
-
pass
|
| 29 |
raise cancelled
|
| 30 |
-
succeeded, result = task.result()
|
| 31 |
if not succeeded:
|
| 32 |
raise result
|
| 33 |
return result
|
|
|
|
| 1 |
+
"""等待所属同步工作真实终态;内部取消也不能遗留仍读写资源的线程。"""
|
| 2 |
import asyncio
|
| 3 |
+
import threading
|
| 4 |
|
| 5 |
|
| 6 |
async def run_owned_thread(function, *args, **kwargs):
|
| 7 |
+
ownership = threading.Lock()
|
| 8 |
+
finished = threading.Event()
|
| 9 |
+
started = False
|
| 10 |
+
outcome = None
|
| 11 |
+
|
| 12 |
+
def abandon_unstarted(result):
|
| 13 |
+
nonlocal outcome
|
| 14 |
+
with ownership:
|
| 15 |
+
if not started and not finished.is_set():
|
| 16 |
+
outcome = result
|
| 17 |
+
finished.set()
|
| 18 |
+
|
| 19 |
+
def execute():
|
| 20 |
+
nonlocal started, outcome
|
| 21 |
+
with ownership:
|
| 22 |
+
if finished.is_set():
|
| 23 |
+
return
|
| 24 |
+
started = True
|
| 25 |
+
try:
|
| 26 |
+
result = (True, function(*args, **kwargs))
|
| 27 |
+
except BaseException as error:
|
| 28 |
+
# 同步异常作为数据交回;取消shield不暴露私有异常正文。
|
| 29 |
+
result = (False, error)
|
| 30 |
+
with ownership:
|
| 31 |
+
outcome = result
|
| 32 |
+
finished.set()
|
| 33 |
+
|
| 34 |
async def finish():
|
|
|
|
|
|
|
| 35 |
try:
|
| 36 |
+
await asyncio.to_thread(execute)
|
| 37 |
+
except asyncio.CancelledError:
|
| 38 |
+
raise
|
| 39 |
+
except BaseException as error:
|
| 40 |
+
abandon_unstarted((False, error))
|
| 41 |
+
|
| 42 |
+
coroutine = finish()
|
| 43 |
+
try:
|
| 44 |
+
task = asyncio.create_task(coroutine)
|
| 45 |
+
except BaseException:
|
| 46 |
+
coroutine.close()
|
| 47 |
+
raise
|
| 48 |
+
|
| 49 |
+
def consume(completed):
|
| 50 |
+
if completed.cancelled():
|
| 51 |
+
# 取消和线程启动竞争时,只有尚未开始的工作能被撤销。
|
| 52 |
+
abandon_unstarted((False, asyncio.CancelledError()))
|
| 53 |
+
else:
|
| 54 |
+
error = completed.exception()
|
| 55 |
+
if error is not None:
|
| 56 |
+
abandon_unstarted((False, error))
|
| 57 |
+
task.add_done_callback(consume)
|
| 58 |
|
|
|
|
| 59 |
cancelled = None
|
| 60 |
+
while not finished.is_set():
|
| 61 |
try:
|
| 62 |
+
if not task.done():
|
| 63 |
+
await asyncio.shield(task)
|
| 64 |
+
else:
|
| 65 |
+
# 内部task已取消但线程仍在运行,不占线程池等待其终态。
|
| 66 |
+
await asyncio.sleep(0.01)
|
| 67 |
except asyncio.CancelledError as error:
|
| 68 |
cancelled = error
|
| 69 |
+
with ownership:
|
| 70 |
+
succeeded, result = outcome
|
| 71 |
if cancelled is not None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
raise cancelled
|
|
|
|
| 73 |
if not succeeded:
|
| 74 |
raise result
|
| 75 |
return result
|
periodic_tasks.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""后台任务的所属启动、停止信号与取消安全等待(保留周期任务导出名)。"""
|
| 2 |
+
import asyncio
|
| 3 |
+
import logging
|
| 4 |
+
|
| 5 |
+
logger = logging.getLogger("ComfyUI-Ranking.PeriodicTasks")
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def start_periodic_task(state, name, factory):
|
| 9 |
+
current = getattr(state, name + "_worker", None)
|
| 10 |
+
if current is not None and not current.done():
|
| 11 |
+
return current
|
| 12 |
+
stop = asyncio.Event()
|
| 13 |
+
coroutine = factory(stop)
|
| 14 |
+
try:
|
| 15 |
+
task = asyncio.create_task(coroutine)
|
| 16 |
+
except BaseException:
|
| 17 |
+
coroutine.close()
|
| 18 |
+
raise
|
| 19 |
+
setattr(state, name + "_stop", stop)
|
| 20 |
+
setattr(state, name + "_worker", task)
|
| 21 |
+
|
| 22 |
+
def consume(completed):
|
| 23 |
+
if not completed.cancelled():
|
| 24 |
+
error = completed.exception()
|
| 25 |
+
if error is not None:
|
| 26 |
+
logger.warning("后台任务结束异常: task=%s | error_type=%s", name, type(error).__name__)
|
| 27 |
+
task.add_done_callback(consume)
|
| 28 |
+
return task
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def request_periodic_stop(state, names):
|
| 32 |
+
tasks = []
|
| 33 |
+
for name in names:
|
| 34 |
+
stop = getattr(state, name + "_stop", None)
|
| 35 |
+
if stop is not None:
|
| 36 |
+
stop.set()
|
| 37 |
+
task = getattr(state, name + "_worker", None)
|
| 38 |
+
if task is not None:
|
| 39 |
+
tasks.append(task)
|
| 40 |
+
return tasks
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
async def wait_periodic_tasks(tasks):
|
| 44 |
+
cancelled = None
|
| 45 |
+
for task in tasks:
|
| 46 |
+
while not task.done():
|
| 47 |
+
try:
|
| 48 |
+
await asyncio.shield(task)
|
| 49 |
+
except asyncio.CancelledError as error:
|
| 50 |
+
# worker自身取消与关闭等待者取消分别处理。
|
| 51 |
+
if asyncio.current_task().cancelling() or not task.cancelled():
|
| 52 |
+
cancelled = error
|
| 53 |
+
except Exception:
|
| 54 |
+
break
|
| 55 |
+
if not task.cancelled():
|
| 56 |
+
# 业务异常已有所属回调诊断,不阻止其它任务收尾和最终同步。
|
| 57 |
+
task.exception()
|
| 58 |
+
if cancelled is not None:
|
| 59 |
+
raise cancelled
|
profile_io.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""资料文件I/O在后台线程执行;准入在排队前,已开始工作取消不提前释放容量。"""
|
| 2 |
+
import asyncio
|
| 3 |
+
import threading
|
| 4 |
+
from fastapi import HTTPException
|
| 5 |
+
|
| 6 |
+
MAX_PROFILE_IO = 8
|
| 7 |
+
|
| 8 |
+
class ProfileIO:
|
| 9 |
+
def __init__(self, capacity=MAX_PROFILE_IO):
|
| 10 |
+
self.capacity = threading.BoundedSemaphore(capacity)
|
| 11 |
+
self._gate = threading.Lock()
|
| 12 |
+
self._accepting = True
|
| 13 |
+
self._pending = 0
|
| 14 |
+
self._closers = 0
|
| 15 |
+
self._drained = threading.Event()
|
| 16 |
+
self._drained.set()
|
| 17 |
+
|
| 18 |
+
def start(self):
|
| 19 |
+
"""同进程重新启动仅在上一代所属工作全部终态后开放准入。"""
|
| 20 |
+
with self._gate:
|
| 21 |
+
if self._pending or self._closers:
|
| 22 |
+
raise RuntimeError("profile work is still pending")
|
| 23 |
+
self._accepting = True
|
| 24 |
+
|
| 25 |
+
async def close(self):
|
| 26 |
+
"""关闭准入并等真实工作终态;取消关闭等待也不提前宣称排空。"""
|
| 27 |
+
with self._gate:
|
| 28 |
+
self._accepting = False
|
| 29 |
+
self._closers += 1
|
| 30 |
+
try:
|
| 31 |
+
cancelled = False
|
| 32 |
+
while not self._drained.is_set():
|
| 33 |
+
try:
|
| 34 |
+
# 不向共用线程池提交等待任务,避免池饱和时相互等待。
|
| 35 |
+
await asyncio.sleep(0.01)
|
| 36 |
+
except asyncio.CancelledError:
|
| 37 |
+
cancelled = True
|
| 38 |
+
if cancelled:
|
| 39 |
+
raise asyncio.CancelledError
|
| 40 |
+
finally:
|
| 41 |
+
with self._gate:
|
| 42 |
+
self._closers -= 1
|
| 43 |
+
|
| 44 |
+
def _finish(self):
|
| 45 |
+
with self._gate:
|
| 46 |
+
self.capacity.release()
|
| 47 |
+
self._pending -= 1
|
| 48 |
+
if not self._pending:
|
| 49 |
+
self._drained.set()
|
| 50 |
+
|
| 51 |
+
async def run(self, function, *args, **kwargs):
|
| 52 |
+
with self._gate:
|
| 53 |
+
if not self._accepting or not self.capacity.acquire(blocking=False):
|
| 54 |
+
raise HTTPException(status_code=503, detail="用户数据暂不可用")
|
| 55 |
+
self._pending += 1
|
| 56 |
+
self._drained.clear()
|
| 57 |
+
|
| 58 |
+
# 内部任务也可能在宿主关闭时被取消。锁区分未开始与已开始的工作,
|
| 59 |
+
# 保证取消和线程启动竞争时只释放一次,且取消后不再开始业务写入。
|
| 60 |
+
ownership = threading.Lock()
|
| 61 |
+
started = False
|
| 62 |
+
released = False
|
| 63 |
+
|
| 64 |
+
def release_unstarted():
|
| 65 |
+
nonlocal released
|
| 66 |
+
with ownership:
|
| 67 |
+
if not started and not released:
|
| 68 |
+
released = True
|
| 69 |
+
self._finish()
|
| 70 |
+
|
| 71 |
+
def execute():
|
| 72 |
+
nonlocal started, released
|
| 73 |
+
with ownership:
|
| 74 |
+
if released:
|
| 75 |
+
return None
|
| 76 |
+
started = True
|
| 77 |
+
try:
|
| 78 |
+
return function(*args, **kwargs)
|
| 79 |
+
finally:
|
| 80 |
+
with ownership:
|
| 81 |
+
released = True
|
| 82 |
+
self._finish()
|
| 83 |
+
|
| 84 |
+
coroutine = asyncio.to_thread(execute)
|
| 85 |
+
try:
|
| 86 |
+
task = asyncio.create_task(coroutine)
|
| 87 |
+
except BaseException:
|
| 88 |
+
coroutine.close(); release_unstarted()
|
| 89 |
+
raise
|
| 90 |
+
|
| 91 |
+
def consume(completed):
|
| 92 |
+
if completed.cancelled():
|
| 93 |
+
release_unstarted()
|
| 94 |
+
else:
|
| 95 |
+
# 提交默认线程池本身也可能失败,尚未开始的准入必须收回。
|
| 96 |
+
release_unstarted()
|
| 97 |
+
completed.exception()
|
| 98 |
+
task.add_done_callback(consume)
|
| 99 |
+
# 外层请求取消不取消所属任务;已开始的工作只在线程终态释放容量。
|
| 100 |
+
return await asyncio.shield(task)
|
| 101 |
+
|
| 102 |
+
_profile_io = ProfileIO()
|
| 103 |
+
run_profile_io = _profile_io.run
|
| 104 |
+
|
| 105 |
+
start_profile_io = _profile_io.start
|
| 106 |
+
close_profile_io = _profile_io.close
|
proxy_resources.py
CHANGED
|
@@ -17,6 +17,21 @@ async def _await_close(task):
|
|
| 17 |
raise cancelled
|
| 18 |
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
class ProxyResources:
|
| 21 |
def __init__(self, client):
|
| 22 |
self.client = client
|
|
|
|
| 17 |
raise cancelled
|
| 18 |
|
| 19 |
|
| 20 |
+
async def close_redirect_response(response):
|
| 21 |
+
"""中间跳转响应关闭一次;完整等待后取消优先,普通关闭失败仍向外传播。"""
|
| 22 |
+
async def finish():
|
| 23 |
+
try:
|
| 24 |
+
await response.aclose()
|
| 25 |
+
except Exception as error:
|
| 26 |
+
logger.warning("PROXY_REDIRECT_CLOSE_FAIL | error_type=%s", type(error).__name__)
|
| 27 |
+
return error
|
| 28 |
+
task = asyncio.create_task(finish())
|
| 29 |
+
await _await_close(task)
|
| 30 |
+
error = task.result()
|
| 31 |
+
if error is not None:
|
| 32 |
+
raise error
|
| 33 |
+
|
| 34 |
+
|
| 35 |
class ProxyResources:
|
| 36 |
def __init__(self, client):
|
| 37 |
self.client = client
|
rate_limit.py
CHANGED
|
@@ -48,8 +48,9 @@ def _consume_shared_limit(identity: str, scope: str, limit: int) -> None:
|
|
| 48 |
now = int(time.time())
|
| 49 |
window_start = now - (now % 60)
|
| 50 |
bucket_key = hashlib.sha256(f"{identity}\0{scope}".encode("utf-8")).hexdigest()
|
| 51 |
-
session =
|
| 52 |
try:
|
|
|
|
| 53 |
dialect = engine.dialect.name
|
| 54 |
values = {
|
| 55 |
"bucket_key": bucket_key,
|
|
@@ -117,14 +118,22 @@ def _consume_shared_limit(identity: str, scope: str, limit: int) -> None:
|
|
| 117 |
raise HTTPException(status_code=429, detail=f"请求过于频繁,限流 {limit}/分钟,请稍后再试")
|
| 118 |
_cleanup_old_buckets_if_due()
|
| 119 |
except HTTPException:
|
| 120 |
-
session
|
|
|
|
|
|
|
| 121 |
raise
|
| 122 |
except Exception as exc:
|
| 123 |
-
session
|
|
|
|
|
|
|
| 124 |
logger.error("SHARED_RATE_LIMIT_DB_FAIL | scope=%s | error_type=%s", scope, type(exc).__name__)
|
| 125 |
raise HTTPException(status_code=503, detail="限流服务暂不可用") from exc
|
| 126 |
finally:
|
| 127 |
-
session
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
|
| 129 |
|
| 130 |
def _cleanup_old_buckets_if_due() -> None:
|
|
|
|
| 48 |
now = int(time.time())
|
| 49 |
window_start = now - (now % 60)
|
| 50 |
bucket_key = hashlib.sha256(f"{identity}\0{scope}".encode("utf-8")).hexdigest()
|
| 51 |
+
session = None
|
| 52 |
try:
|
| 53 |
+
session = SessionLocal()
|
| 54 |
dialect = engine.dialect.name
|
| 55 |
values = {
|
| 56 |
"bucket_key": bucket_key,
|
|
|
|
| 118 |
raise HTTPException(status_code=429, detail=f"请求过于频繁,限流 {limit}/分钟,请稍后再试")
|
| 119 |
_cleanup_old_buckets_if_due()
|
| 120 |
except HTTPException:
|
| 121 |
+
if session is not None:
|
| 122 |
+
try: session.rollback()
|
| 123 |
+
except Exception: logger.error("SHARED_RATE_LIMIT_ROLLBACK_FAIL")
|
| 124 |
raise
|
| 125 |
except Exception as exc:
|
| 126 |
+
if session is not None:
|
| 127 |
+
try: session.rollback()
|
| 128 |
+
except Exception: logger.error("SHARED_RATE_LIMIT_ROLLBACK_FAIL")
|
| 129 |
logger.error("SHARED_RATE_LIMIT_DB_FAIL | scope=%s | error_type=%s", scope, type(exc).__name__)
|
| 130 |
raise HTTPException(status_code=503, detail="限流服务暂不可用") from exc
|
| 131 |
finally:
|
| 132 |
+
if session is not None:
|
| 133 |
+
try: session.close()
|
| 134 |
+
except Exception as error:
|
| 135 |
+
logger.error("SHARED_RATE_LIMIT_CLOSE_FAIL | error_type=%s", type(error).__name__)
|
| 136 |
+
raise HTTPException(status_code=503, detail="限流服务暂不可用") from None
|
| 137 |
|
| 138 |
|
| 139 |
def _cleanup_old_buckets_if_due() -> None:
|
resource_deletion.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
"""资源删除的本地恢复记录与关联清理;不访问或修改资金数据。"""
|
| 2 |
import asyncio
|
|
|
|
| 3 |
import copy
|
| 4 |
import datetime
|
| 5 |
import logging
|
|
@@ -162,7 +163,7 @@ def delete_resource(store, item_id, account, can_moderate=False):
|
|
| 162 |
raise ResourceStoreError(503, '资源删除暂时无法核实,请稍后核对') from None
|
| 163 |
|
| 164 |
|
| 165 |
-
def repair_resource_deletions(store, cursor=0, limit=50, should_stop=lambda: False):
|
| 166 |
queue = _read_queue(store)
|
| 167 |
ids = list(queue)
|
| 168 |
if not ids:
|
|
@@ -179,6 +180,11 @@ def repair_resource_deletions(store, cursor=0, limit=50, should_stop=lambda: Fal
|
|
| 179 |
_, updated = _cleanup(store, queue[item_id])
|
| 180 |
changed += bool(updated)
|
| 181 |
except Exception as error:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
logger.warning('资源删除恢复失败: error_type=%s', type(error).__name__)
|
| 183 |
processed += 1
|
| 184 |
return (start + processed) % len(ids), changed
|
|
@@ -188,18 +194,41 @@ async def resource_deletion_worker(store, stop, invalidate=None, interval=30):
|
|
| 188 |
initialized = False
|
| 189 |
cursor = 0
|
| 190 |
while not stop.is_set():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
try:
|
| 192 |
if not initialized:
|
| 193 |
-
await
|
| 194 |
initialized = True
|
| 195 |
-
|
| 196 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 197 |
try:
|
| 198 |
invalidate('items:')
|
| 199 |
except Exception:
|
| 200 |
logger.warning('资源删除恢复后的排序失效失败')
|
| 201 |
-
except Exception as error:
|
| 202 |
-
logger.warning('资源删除恢复轮次失败: error_type=%s', type(error).__name__)
|
| 203 |
if stop.is_set():
|
| 204 |
break
|
| 205 |
try:
|
|
|
|
| 1 |
"""资源删除的本地恢复记录与关联清理;不访问或修改资金数据。"""
|
| 2 |
import asyncio
|
| 3 |
+
from owned_thread import run_owned_thread
|
| 4 |
import copy
|
| 5 |
import datetime
|
| 6 |
import logging
|
|
|
|
| 163 |
raise ResourceStoreError(503, '资源删除暂时无法核实,请稍后核对') from None
|
| 164 |
|
| 165 |
|
| 166 |
+
def repair_resource_deletions(store, cursor=0, limit=50, should_stop=lambda: False, *, on_uncertain=None):
|
| 167 |
queue = _read_queue(store)
|
| 168 |
ids = list(queue)
|
| 169 |
if not ids:
|
|
|
|
| 180 |
_, updated = _cleanup(store, queue[item_id])
|
| 181 |
changed += bool(updated)
|
| 182 |
except Exception as error:
|
| 183 |
+
if on_uncertain is not None:
|
| 184 |
+
try:
|
| 185 |
+
on_uncertain()
|
| 186 |
+
except Exception as notification_error:
|
| 187 |
+
logger.warning('资源删除恢复缓存通知失败: error_type=%s', type(notification_error).__name__)
|
| 188 |
logger.warning('资源删除恢复失败: error_type=%s', type(error).__name__)
|
| 189 |
processed += 1
|
| 190 |
return (start + processed) % len(ids), changed
|
|
|
|
| 194 |
initialized = False
|
| 195 |
cursor = 0
|
| 196 |
while not stop.is_set():
|
| 197 |
+
attempted = threading.Event()
|
| 198 |
+
changed_data = threading.Event()
|
| 199 |
+
uncertain = threading.Event()
|
| 200 |
+
cancelled = False
|
| 201 |
+
|
| 202 |
+
def repair():
|
| 203 |
+
attempted.set()
|
| 204 |
+
try:
|
| 205 |
+
result = repair_resource_deletions(store, cursor, 50, stop.is_set, on_uncertain=uncertain.set)
|
| 206 |
+
except Exception:
|
| 207 |
+
uncertain.set()
|
| 208 |
+
raise
|
| 209 |
+
if result[1]:
|
| 210 |
+
changed_data.set()
|
| 211 |
+
return result
|
| 212 |
+
|
| 213 |
try:
|
| 214 |
if not initialized:
|
| 215 |
+
await run_owned_thread(store.atomic_update, DELETION_FILE, _queue, default_data={})
|
| 216 |
initialized = True
|
| 217 |
+
if stop.is_set():
|
| 218 |
+
break
|
| 219 |
+
cursor, changed = await run_owned_thread(repair)
|
| 220 |
+
except asyncio.CancelledError:
|
| 221 |
+
cancelled = True
|
| 222 |
+
raise
|
| 223 |
+
except Exception as error:
|
| 224 |
+
logger.warning('资源删除恢复轮次失败: error_type=%s', type(error).__name__)
|
| 225 |
+
finally:
|
| 226 |
+
# 所属等待已确认真实线程终态,取消也不能跳过已发布数据的失效。
|
| 227 |
+
if invalidate and (changed_data.is_set() or uncertain.is_set() or (cancelled and attempted.is_set())):
|
| 228 |
try:
|
| 229 |
invalidate('items:')
|
| 230 |
except Exception:
|
| 231 |
logger.warning('资源删除恢复后的排序失效失败')
|
|
|
|
|
|
|
| 232 |
if stop.is_set():
|
| 233 |
break
|
| 234 |
try:
|
resource_publication.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
"""持久发布身份先入SQL;JSON失败可恢复,同次重试不制造新资源。"""
|
| 2 |
import asyncio
|
|
|
|
| 3 |
import copy
|
| 4 |
import hashlib
|
| 5 |
import json
|
|
@@ -179,7 +180,7 @@ async def publication_worker(bind, store, invalidate, stop, interval=30):
|
|
| 179 |
cursor = ''
|
| 180 |
while not stop.is_set():
|
| 181 |
try:
|
| 182 |
-
cursor = await
|
| 183 |
except Exception:
|
| 184 |
logging.getLogger(__name__).warning('资源发布恢复轮次暂未完成')
|
| 185 |
try:
|
|
|
|
| 1 |
"""持久发布身份先入SQL;JSON失败可恢复,同次重试不制造新资源。"""
|
| 2 |
import asyncio
|
| 3 |
+
from owned_thread import run_owned_thread
|
| 4 |
import copy
|
| 5 |
import hashlib
|
| 6 |
import json
|
|
|
|
| 180 |
cursor = ''
|
| 181 |
while not stop.is_set():
|
| 182 |
try:
|
| 183 |
+
cursor = await run_owned_thread(repair_publications, bind, store, invalidate, cursor)
|
| 184 |
except Exception:
|
| 185 |
logging.getLogger(__name__).warning('资源发布恢复轮次暂未完成')
|
| 186 |
try:
|
router_comments.py
CHANGED
|
@@ -4,7 +4,7 @@ import time
|
|
| 4 |
import uuid
|
| 5 |
from html import escape
|
| 6 |
import 数据库连接 as db
|
| 7 |
-
from comment_store import append_comment, delete_comment
|
| 8 |
from notifications import add_notification
|
| 9 |
from models import InteractionToggle, CommentCreate
|
| 10 |
from 安全认证 import require_auth, is_admin
|
|
@@ -70,7 +70,7 @@ async def toggle_interaction(request: Request, interaction: InteractionToggle, c
|
|
| 70 |
async def post_comment(request: Request, comment: CommentCreate, current_user: str = Depends(require_auth)):
|
| 71 |
comment.author = current_user # 强制使用认证用户身份,防止伪造
|
| 72 |
# 评论内容长度校验
|
| 73 |
-
if not comment.content or len(comment.content) >
|
| 74 |
raise HTTPException(status_code=400, detail="评论长度必须在1-5000字符之间")
|
| 75 |
# HTML 转义,防止 XSS
|
| 76 |
comment.content = escape(comment.content)
|
|
|
|
| 4 |
import uuid
|
| 5 |
from html import escape
|
| 6 |
import 数据库连接 as db
|
| 7 |
+
from comment_store import COMMENT_MAX_LENGTH, append_comment, delete_comment
|
| 8 |
from notifications import add_notification
|
| 9 |
from models import InteractionToggle, CommentCreate
|
| 10 |
from 安全认证 import require_auth, is_admin
|
|
|
|
| 70 |
async def post_comment(request: Request, comment: CommentCreate, current_user: str = Depends(require_auth)):
|
| 71 |
comment.author = current_user # 强制使用认证用户身份,防止伪造
|
| 72 |
# 评论内容长度校验
|
| 73 |
+
if not comment.content or len(comment.content) > COMMENT_MAX_LENGTH:
|
| 74 |
raise HTTPException(status_code=400, detail="评论长度必须在1-5000字符之间")
|
| 75 |
# HTML 转义,防止 XSS
|
| 76 |
comment.content = escape(comment.content)
|
router_items.py
CHANGED
|
@@ -14,7 +14,7 @@ import 数据库连接 as db
|
|
| 14 |
from models import ItemCreate, ItemUpdate, RatingRequest
|
| 15 |
from 安全认证 import require_auth, check_ownership, is_admin
|
| 16 |
from 数据库连接 import invalidate_cache
|
| 17 |
-
from db_utils import record_view, sort_cache
|
| 18 |
from github_source import repository_parts, select_github_token
|
| 19 |
from github_versions import refresh_resource_version, invalidate_source
|
| 20 |
from resource_store import publish_resource, published_response, edit_resource, ResourceStoreError
|
|
@@ -70,9 +70,10 @@ data_file = "items.json"
|
|
| 70 |
|
| 71 |
@router.get("/api/items")
|
| 72 |
async def get_items(type: str = "tool", sort: str = "time", limit: int = 50): # 优化:默认限制调大至 50,提升前端列表体验
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
|
|
|
| 76 |
|
| 77 |
# 如果是推荐榜,匹配所有 recommend 开头的子类型
|
| 78 |
if type == "recommend":
|
|
@@ -81,28 +82,32 @@ async def get_items(type: str = "tool", sort: str = "time", limit: int = 50): #
|
|
| 81 |
filtered_items = [item for item in items_db if item.get("type") == type]
|
| 82 |
|
| 83 |
# 🗂️ 使用排序缓存优化排序性能
|
| 84 |
-
|
|
|
|
| 85 |
|
| 86 |
-
|
|
|
|
| 87 |
if sort == "likes":
|
| 88 |
-
|
| 89 |
elif sort == "favorites":
|
| 90 |
-
|
| 91 |
elif sort == "downloads":
|
| 92 |
-
|
| 93 |
elif sort == "tips": # 🚀 按近期打赏排序
|
| 94 |
-
|
| 95 |
-
data.sort(key=lambda x: x.get("tip_history", {}).get(current_month, 0), reverse=True)
|
| 96 |
elif sort == "views": # 👁️ 按总访问量排序
|
| 97 |
-
|
| 98 |
elif sort == "daily_views": # 👁️ 按日访问量排序
|
| 99 |
-
|
| 100 |
elif sort == "rating": # ⭐ 按评分排序
|
| 101 |
-
|
| 102 |
else: # time 或其他默认
|
| 103 |
-
|
|
|
|
|
|
|
|
|
|
| 104 |
|
| 105 |
-
filtered_items = sort_cache.get_sorted(cache_key, filtered_items, sort_fn)
|
| 106 |
|
| 107 |
result_items = []
|
| 108 |
for item in filtered_items:
|
|
@@ -162,8 +167,8 @@ async def search_creators(keyword: str, sort: str = "downloads", limit: int = 50
|
|
| 162 |
搜索创作者
|
| 163 |
根据关键词搜索创作者(name、account、shortDesc 不区分大小写子串匹配)
|
| 164 |
"""
|
| 165 |
-
users_db =
|
| 166 |
-
items_db =
|
| 167 |
|
| 168 |
# 🚀 P1性能优化:预构建 author->items 索引
|
| 169 |
author_items_index = {}
|
|
@@ -235,8 +240,8 @@ async def get_creators(sort: str = "downloads", limit: int = 100):
|
|
| 235 |
🚀 P1性能优化:预构建 author->items 索引,避免 N+1 查询
|
| 236 |
⚡ P2性能优化:移除大字段(trendData、commentsData、tip_board),移至详情接口按需加载
|
| 237 |
"""
|
| 238 |
-
users_db =
|
| 239 |
-
items_db =
|
| 240 |
|
| 241 |
# 🚀 P1性能优化:预构建 author->items 索引,复杂��从 O(n*m) 降到 O(n+m)
|
| 242 |
author_items_index = {}
|
|
@@ -297,9 +302,9 @@ async def get_creator_details(account: str):
|
|
| 297 |
供前端展开卡片时按需加载
|
| 298 |
包含:trendData、commentsData、tip_board
|
| 299 |
"""
|
| 300 |
-
users_db =
|
| 301 |
-
items_db =
|
| 302 |
-
comments_db =
|
| 303 |
|
| 304 |
# 检查用户是否存在
|
| 305 |
if account not in users_db:
|
|
@@ -464,9 +469,9 @@ async def check_github_updates(request: Request, current_user: str = Depends(req
|
|
| 464 |
@router.get("/api/items/{item_id}")
|
| 465 |
async def get_item_by_id(item_id: str):
|
| 466 |
"""根据ID获取单个资源的详细信息"""
|
| 467 |
-
items_db =
|
| 468 |
-
comments_db =
|
| 469 |
-
versions_db =
|
| 470 |
|
| 471 |
for item in items_db:
|
| 472 |
if item["id"] == item_id:
|
|
@@ -490,7 +495,7 @@ async def get_item_by_id(item_id: str):
|
|
| 490 |
@router.get("/api/item/{item_id}/version")
|
| 491 |
async def get_item_version(item_id: str):
|
| 492 |
"""获取单个资源的最新版本号"""
|
| 493 |
-
versions_db =
|
| 494 |
return {"status": "success", "version": _get_version_str(versions_db, item_id)}
|
| 495 |
|
| 496 |
@router.delete("/api/items/{item_id}")
|
|
|
|
| 14 |
from models import ItemCreate, ItemUpdate, RatingRequest
|
| 15 |
from 安全认证 import require_auth, check_ownership, is_admin
|
| 16 |
from 数据库连接 import invalidate_cache
|
| 17 |
+
from db_utils import read_public_source, record_view, sort_cache
|
| 18 |
from github_source import repository_parts, select_github_token
|
| 19 |
from github_versions import refresh_resource_version, invalidate_source
|
| 20 |
from resource_store import publish_resource, published_response, edit_resource, ResourceStoreError
|
|
|
|
| 70 |
|
| 71 |
@router.get("/api/items")
|
| 72 |
async def get_items(type: str = "tool", sort: str = "time", limit: int = 50): # 优化:默认限制调大至 50,提升前端列表体验
|
| 73 |
+
read_epoch = sort_cache.capture_read_epoch()
|
| 74 |
+
items_db = read_public_source(db, "items.json", default_data=[])
|
| 75 |
+
comments_db = read_public_source(db, "comments.json", default_data={})
|
| 76 |
+
versions_db = read_public_source(db, "versions.json", default_data={}) # 读取版本库
|
| 77 |
|
| 78 |
# 如果是推荐榜,匹配所有 recommend 开头的子类型
|
| 79 |
if type == "recommend":
|
|
|
|
| 82 |
filtered_items = [item for item in items_db if item.get("type") == type]
|
| 83 |
|
| 84 |
# 🗂️ 使用排序缓存优化排序性能
|
| 85 |
+
cache_sort = sort if sort in ("time", "likes", "favorites", "downloads", "tips", "views", "daily_views", "rating") else "time"
|
| 86 |
+
cache_key = f"items:{type}:{cache_sort}"
|
| 87 |
|
| 88 |
+
current_month = datetime.date.today().strftime("%Y-%m") if sort == "tips" else None
|
| 89 |
+
def order_key(item):
|
| 90 |
if sort == "likes":
|
| 91 |
+
return item.get("likes", 0)
|
| 92 |
elif sort == "favorites":
|
| 93 |
+
return item.get("favorites", 0)
|
| 94 |
elif sort == "downloads":
|
| 95 |
+
return item.get("uses", 0)
|
| 96 |
elif sort == "tips": # 🚀 按近期打赏排序
|
| 97 |
+
return item.get("tip_history", {}).get(current_month, 0)
|
|
|
|
| 98 |
elif sort == "views": # 👁️ 按总访问量排序
|
| 99 |
+
return item.get("views", 0)
|
| 100 |
elif sort == "daily_views": # 👁️ 按日访问量排序
|
| 101 |
+
return item.get("daily_views", 0)
|
| 102 |
elif sort == "rating": # ⭐ 按评分排序
|
| 103 |
+
return (item.get("rating_avg", 0), item.get("rating_count", 0))
|
| 104 |
else: # time 或其他默认
|
| 105 |
+
return item.get("created_at", 0)
|
| 106 |
+
|
| 107 |
+
def sort_fn(data):
|
| 108 |
+
data.sort(key=order_key, reverse=True)
|
| 109 |
|
| 110 |
+
filtered_items = sort_cache.get_sorted(cache_key, filtered_items, sort_fn, read_epoch=read_epoch, fresh=True)
|
| 111 |
|
| 112 |
result_items = []
|
| 113 |
for item in filtered_items:
|
|
|
|
| 167 |
搜索创作者
|
| 168 |
根据关键词搜索创作者(name、account、shortDesc 不区分大小写子串匹配)
|
| 169 |
"""
|
| 170 |
+
users_db = read_public_source(db, "users.json", default_data={})
|
| 171 |
+
items_db = read_public_source(db, "items.json", default_data=[])
|
| 172 |
|
| 173 |
# 🚀 P1性能优化:预构建 author->items 索引
|
| 174 |
author_items_index = {}
|
|
|
|
| 240 |
🚀 P1性能优化:预构建 author->items 索引,避免 N+1 查询
|
| 241 |
⚡ P2性能优化:移除大字段(trendData、commentsData、tip_board),移至详情接口按需加载
|
| 242 |
"""
|
| 243 |
+
users_db = read_public_source(db, "users.json", default_data={})
|
| 244 |
+
items_db = read_public_source(db, "items.json", default_data=[])
|
| 245 |
|
| 246 |
# 🚀 P1性能优化:预构建 author->items 索引,复杂��从 O(n*m) 降到 O(n+m)
|
| 247 |
author_items_index = {}
|
|
|
|
| 302 |
供前端展开卡片时按需加载
|
| 303 |
包含:trendData、commentsData、tip_board
|
| 304 |
"""
|
| 305 |
+
users_db = read_public_source(db, "users.json", default_data={})
|
| 306 |
+
items_db = read_public_source(db, "items.json", default_data=[])
|
| 307 |
+
comments_db = read_public_source(db, "comments.json", default_data={})
|
| 308 |
|
| 309 |
# 检查用户是否存在
|
| 310 |
if account not in users_db:
|
|
|
|
| 469 |
@router.get("/api/items/{item_id}")
|
| 470 |
async def get_item_by_id(item_id: str):
|
| 471 |
"""根据ID获取单个资源的详细信息"""
|
| 472 |
+
items_db = read_public_source(db, "items.json", default_data=[])
|
| 473 |
+
comments_db = read_public_source(db, "comments.json", default_data={})
|
| 474 |
+
versions_db = read_public_source(db, "versions.json", default_data={})
|
| 475 |
|
| 476 |
for item in items_db:
|
| 477 |
if item["id"] == item_id:
|
|
|
|
| 495 |
@router.get("/api/item/{item_id}/version")
|
| 496 |
async def get_item_version(item_id: str):
|
| 497 |
"""获取单个资源的最新版本号"""
|
| 498 |
+
versions_db = read_public_source(db, "versions.json", default_data={})
|
| 499 |
return {"status": "success", "version": _get_version_str(versions_db, item_id)}
|
| 500 |
|
| 501 |
@router.delete("/api/items/{item_id}")
|
router_messages.py
CHANGED
|
@@ -6,6 +6,8 @@ import asyncio
|
|
| 6 |
import time
|
| 7 |
import uuid
|
| 8 |
import subprocess
|
|
|
|
|
|
|
| 9 |
import os
|
| 10 |
import 数据库连接 as db
|
| 11 |
import chat_store
|
|
@@ -15,6 +17,7 @@ from 安全认证 import require_auth, is_admin
|
|
| 15 |
|
| 16 |
# 🔒 频次限制(与 router_users_auth.py / router_wallet.py / router_prompts.py 同一写法)
|
| 17 |
|
|
|
|
| 18 |
router = APIRouter()
|
| 19 |
|
| 20 |
# ==========================================
|
|
@@ -96,20 +99,7 @@ async def run_admin_script(request: Request, req: AdminScriptRequest, current_us
|
|
| 96 |
sensitive_prefixes = ('GITHUB', 'JWT', 'HF_', 'DATABASE', 'ALIPAY', 'PASSWORD')
|
| 97 |
clean_env = {k: v for k, v in os.environ.items()
|
| 98 |
if not any(k.startswith(prefix) for prefix in sensitive_prefixes)}
|
| 99 |
-
|
| 100 |
-
# ⚡ 性能修复:subprocess.run 会同步等待子进程结束(最长 30 秒),
|
| 101 |
-
# 在 async 处理器里直接调用会占死唯一的事件循环,脚本跑完前整个服务无法响应任何请求;
|
| 102 |
-
# 交给线程池执行,TimeoutExpired 等异常仍由下方原有 except 分支按原样处理。
|
| 103 |
-
result = await asyncio.to_thread(
|
| 104 |
-
subprocess.run,
|
| 105 |
-
["python", script_path],
|
| 106 |
-
capture_output=True,
|
| 107 |
-
text=True,
|
| 108 |
-
timeout=30,
|
| 109 |
-
cwd=current_dir,
|
| 110 |
-
encoding="utf-8",
|
| 111 |
-
env=clean_env
|
| 112 |
-
)
|
| 113 |
|
| 114 |
output = ""
|
| 115 |
if result.stdout:
|
|
@@ -129,15 +119,18 @@ async def run_admin_script(request: Request, req: AdminScriptRequest, current_us
|
|
| 129 |
"output": output
|
| 130 |
}
|
| 131 |
|
|
|
|
|
|
|
| 132 |
except subprocess.TimeoutExpired:
|
| 133 |
return {
|
| 134 |
"status": "error",
|
| 135 |
"output": "❌ 脚本执行超时 (30秒)"
|
| 136 |
}
|
| 137 |
-
except Exception as
|
|
|
|
| 138 |
return {
|
| 139 |
"status": "error",
|
| 140 |
-
"output":
|
| 141 |
}
|
| 142 |
|
| 143 |
# ==========================================
|
|
|
|
| 6 |
import time
|
| 7 |
import uuid
|
| 8 |
import subprocess
|
| 9 |
+
import logging
|
| 10 |
+
from script_execution import run_admin_script_process, ScriptExecutionBusy
|
| 11 |
import os
|
| 12 |
import 数据库连接 as db
|
| 13 |
import chat_store
|
|
|
|
| 17 |
|
| 18 |
# 🔒 频次限制(与 router_users_auth.py / router_wallet.py / router_prompts.py 同一写法)
|
| 19 |
|
| 20 |
+
logger = logging.getLogger("ComfyUI-Ranking.AdminScripts")
|
| 21 |
router = APIRouter()
|
| 22 |
|
| 23 |
# ==========================================
|
|
|
|
| 99 |
sensitive_prefixes = ('GITHUB', 'JWT', 'HF_', 'DATABASE', 'ALIPAY', 'PASSWORD')
|
| 100 |
clean_env = {k: v for k, v in os.environ.items()
|
| 101 |
if not any(k.startswith(prefix) for prefix in sensitive_prefixes)}
|
| 102 |
+
result = await run_admin_script_process(script_path, cwd=current_dir, env=clean_env)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
|
| 104 |
output = ""
|
| 105 |
if result.stdout:
|
|
|
|
| 119 |
"output": output
|
| 120 |
}
|
| 121 |
|
| 122 |
+
except ScriptExecutionBusy:
|
| 123 |
+
raise HTTPException(status_code=503, detail="脚本执行繁忙,请先确认当前执行状态") from None
|
| 124 |
except subprocess.TimeoutExpired:
|
| 125 |
return {
|
| 126 |
"status": "error",
|
| 127 |
"output": "❌ 脚本执行超时 (30秒)"
|
| 128 |
}
|
| 129 |
+
except Exception as error:
|
| 130 |
+
logger.error("ADMIN_SCRIPT_ERROR | error_type=%s", type(error).__name__)
|
| 131 |
return {
|
| 132 |
"status": "error",
|
| 133 |
+
"output": "脚本执行暂不可用,请先确认当前状态"
|
| 134 |
}
|
| 135 |
|
| 136 |
# ==========================================
|
router_posts.py
CHANGED
|
@@ -10,9 +10,9 @@ from fastapi import APIRouter, HTTPException, Depends, Request
|
|
| 10 |
from sqlalchemy.orm import Session
|
| 11 |
from models import PostCreate, PostUpdate, RatingRequest
|
| 12 |
import 数据库连接 as db
|
| 13 |
-
from comment_store import append_comment, increment_comment_count, attach_comment_counts
|
| 14 |
from 安全认证 import require_auth
|
| 15 |
-
from db_utils import record_view, sort_cache
|
| 16 |
from database_sql import get_db
|
| 17 |
from models_sql import Wallet, Transaction
|
| 18 |
from notifications import add_notification
|
|
@@ -60,32 +60,37 @@ async def get_posts(page: int = 1, limit: int = 20, sort: str = "latest"):
|
|
| 60 |
- sort=daily_views: 按日访问量降序
|
| 61 |
- sort=tips: 按打赏总额降序
|
| 62 |
"""
|
| 63 |
-
|
| 64 |
-
|
|
|
|
| 65 |
|
| 66 |
# users_db 已经是 {account: user_info} 格式,直接使用
|
| 67 |
user_map = users_db
|
| 68 |
|
| 69 |
# 🗂️ 使用排序缓存优化排序性能
|
| 70 |
-
|
|
|
|
| 71 |
|
| 72 |
-
def
|
| 73 |
if sort == "likes":
|
| 74 |
-
|
| 75 |
elif sort == "favorites":
|
| 76 |
-
|
| 77 |
elif sort == "views":
|
| 78 |
-
|
| 79 |
elif sort == "daily_views":
|
| 80 |
-
|
| 81 |
elif sort == "tips":
|
| 82 |
-
|
| 83 |
elif sort == "rating": # ⭐ 按评分排序
|
| 84 |
-
|
| 85 |
else: # latest 或其他默认
|
| 86 |
-
|
|
|
|
|
|
|
|
|
|
| 87 |
|
| 88 |
-
sorted_posts = sort_cache.get_sorted(cache_key, posts_db, sort_fn)
|
| 89 |
|
| 90 |
# 分页
|
| 91 |
start = (page - 1) * limit
|
|
@@ -439,6 +444,8 @@ async def tip_post(request: Request, post_id: str, amount: int, is_anon: bool =
|
|
| 439 |
Transaction.tx_type == "TIP_OUT",
|
| 440 |
Transaction.amount == -amount,
|
| 441 |
Transaction.related_account == author,
|
|
|
|
|
|
|
| 442 |
Transaction.created_at >= recent_cutoff
|
| 443 |
).with_for_update().first()
|
| 444 |
if duplicate_tx:
|
|
@@ -567,29 +574,8 @@ async def tip_post(request: Request, post_id: str, amount: int, is_anon: bool =
|
|
| 567 |
|
| 568 |
@router.get("/api/posts/{post_id}/comments")
|
| 569 |
async def get_post_comments(post_id: str):
|
| 570 |
-
"""
|
| 571 |
-
|
| 572 |
-
"""
|
| 573 |
-
comments_db = db.load_data("comments.json", default_data={})
|
| 574 |
-
users_db = db.load_data("users.json", default_data={})
|
| 575 |
-
|
| 576 |
-
# users_db 已经是 {account: user_info} 格式,直接使用
|
| 577 |
-
user_map = users_db
|
| 578 |
-
|
| 579 |
-
# comments_db 是 {item_id: [comments]} 格式
|
| 580 |
-
post_comments = comments_db.get(post_id, [])
|
| 581 |
-
|
| 582 |
-
# 附加用户信息
|
| 583 |
-
result = []
|
| 584 |
-
for c in post_comments:
|
| 585 |
-
author_info = user_map.get(c.get("author"), {})
|
| 586 |
-
result.append({
|
| 587 |
-
**c,
|
| 588 |
-
"author_name": author_info.get("name", c.get("author")),
|
| 589 |
-
"author_avatar": author_info.get("avatarDataUrl", "")
|
| 590 |
-
})
|
| 591 |
-
|
| 592 |
-
return {"status": "success", "data": result}
|
| 593 |
|
| 594 |
@router.post("/api/posts/{post_id}/comments")
|
| 595 |
@shared_limit(10)
|
|
@@ -599,6 +585,8 @@ async def add_post_comment(request: Request, post_id: str, content: str, current
|
|
| 599 |
"""
|
| 600 |
if not content or not content.strip():
|
| 601 |
raise HTTPException(status_code=400, detail="评论内容不能为空")
|
|
|
|
|
|
|
| 602 |
|
| 603 |
posts_db = db.load_data("posts.json", default_data=[])
|
| 604 |
|
|
|
|
| 10 |
from sqlalchemy.orm import Session
|
| 11 |
from models import PostCreate, PostUpdate, RatingRequest
|
| 12 |
import 数据库连接 as db
|
| 13 |
+
from comment_store import COMMENT_MAX_LENGTH, append_comment, increment_comment_count, attach_comment_counts, read_comment_view
|
| 14 |
from 安全认证 import require_auth
|
| 15 |
+
from db_utils import read_public_source, record_view, sort_cache
|
| 16 |
from database_sql import get_db
|
| 17 |
from models_sql import Wallet, Transaction
|
| 18 |
from notifications import add_notification
|
|
|
|
| 60 |
- sort=daily_views: 按日访问量降序
|
| 61 |
- sort=tips: 按打赏总额降序
|
| 62 |
"""
|
| 63 |
+
read_epoch = sort_cache.capture_read_epoch()
|
| 64 |
+
posts_db = read_public_source(db, "posts.json", default_data=[])
|
| 65 |
+
users_db = read_public_source(db, "users.json", default_data={})
|
| 66 |
|
| 67 |
# users_db 已经是 {account: user_info} 格式,直接使用
|
| 68 |
user_map = users_db
|
| 69 |
|
| 70 |
# 🗂️ 使用排序缓存优化排序性能
|
| 71 |
+
cache_sort = sort if sort in ("latest", "likes", "favorites", "views", "daily_views", "tips", "rating") else "latest"
|
| 72 |
+
cache_key = f"posts:{cache_sort}"
|
| 73 |
|
| 74 |
+
def order_key(item):
|
| 75 |
if sort == "likes":
|
| 76 |
+
return item.get("likes", 0)
|
| 77 |
elif sort == "favorites":
|
| 78 |
+
return item.get("favorites", 0)
|
| 79 |
elif sort == "views":
|
| 80 |
+
return item.get("views", 0)
|
| 81 |
elif sort == "daily_views":
|
| 82 |
+
return item.get("daily_views", 0)
|
| 83 |
elif sort == "tips":
|
| 84 |
+
return sum(t.get("amount", 0) for t in item.get("tip_board", []))
|
| 85 |
elif sort == "rating": # ⭐ 按评分排序
|
| 86 |
+
return (item.get("rating_avg", 0), item.get("rating_count", 0))
|
| 87 |
else: # latest 或其他默认
|
| 88 |
+
return item.get("created_at", 0)
|
| 89 |
+
|
| 90 |
+
def sort_fn(data):
|
| 91 |
+
data.sort(key=order_key, reverse=True)
|
| 92 |
|
| 93 |
+
sorted_posts = sort_cache.get_sorted(cache_key, posts_db, sort_fn, read_epoch=read_epoch, fresh=True)
|
| 94 |
|
| 95 |
# 分页
|
| 96 |
start = (page - 1) * limit
|
|
|
|
| 444 |
Transaction.tx_type == "TIP_OUT",
|
| 445 |
Transaction.amount == -amount,
|
| 446 |
Transaction.related_account == author,
|
| 447 |
+
Transaction.item_id == post_id,
|
| 448 |
+
Transaction.item_type == "post",
|
| 449 |
Transaction.created_at >= recent_cutoff
|
| 450 |
).with_for_update().first()
|
| 451 |
if duplicate_tx:
|
|
|
|
| 574 |
|
| 575 |
@router.get("/api/posts/{post_id}/comments")
|
| 576 |
async def get_post_comments(post_id: str):
|
| 577 |
+
"""获取评论;有界后台读取并保留原作者附加字段。"""
|
| 578 |
+
return {"status": "success", "data": await read_comment_view(post_id)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 579 |
|
| 580 |
@router.post("/api/posts/{post_id}/comments")
|
| 581 |
@shared_limit(10)
|
|
|
|
| 585 |
"""
|
| 586 |
if not content or not content.strip():
|
| 587 |
raise HTTPException(status_code=400, detail="评论内容不能为空")
|
| 588 |
+
if len(content) > COMMENT_MAX_LENGTH:
|
| 589 |
+
raise HTTPException(status_code=400, detail="评论长度必须在1-5000字符之间")
|
| 590 |
|
| 591 |
posts_db = db.load_data("posts.json", default_data=[])
|
| 592 |
|
router_prompts.py
CHANGED
|
@@ -18,9 +18,9 @@ from sqlalchemy.orm import Session
|
|
| 18 |
from models import PromptCreate, PromptUpdate, PromptCategoryAdd, PromptPurchaseRequest
|
| 19 |
from typing import Optional
|
| 20 |
import 数据库连接 as db
|
| 21 |
-
from comment_store import append_comment, increment_comment_count, attach_comment_counts
|
| 22 |
from 安全认证 import require_auth, optional_auth, is_admin
|
| 23 |
-
from db_utils import record_view, sort_cache
|
| 24 |
from database_sql import get_db
|
| 25 |
from models_sql import Wallet, Transaction, Ownership
|
| 26 |
from notifications import add_notification
|
|
@@ -232,28 +232,33 @@ async def get_prompts(prompt_type: str = None, category: str = None,
|
|
| 232 |
- sort: latest / likes / favorites / views / daily_views / tips
|
| 233 |
⚠️ 列表接口永不下发 prompt_text(付费内容脱敏)
|
| 234 |
"""
|
|
|
|
| 235 |
prompts_db = await _prompt_operation(read_prompts, db)
|
| 236 |
-
users_db =
|
| 237 |
user_map = users_db
|
| 238 |
|
| 239 |
# 🗂️ 使用排序缓存优化排序性能
|
| 240 |
-
|
|
|
|
| 241 |
|
| 242 |
-
def
|
| 243 |
if sort == "likes":
|
| 244 |
-
|
| 245 |
elif sort == "favorites":
|
| 246 |
-
|
| 247 |
elif sort == "views":
|
| 248 |
-
|
| 249 |
elif sort == "daily_views":
|
| 250 |
-
|
| 251 |
elif sort == "tips":
|
| 252 |
-
|
| 253 |
else: # latest 或其他默认
|
| 254 |
-
|
|
|
|
|
|
|
|
|
|
| 255 |
|
| 256 |
-
sorted_prompts = sort_cache.get_sorted(cache_key, prompts_db, sort_fn)
|
| 257 |
|
| 258 |
# 类型与分类过滤
|
| 259 |
if prompt_type in VALID_PROMPT_TYPES:
|
|
@@ -669,6 +674,8 @@ async def tip_prompt(request: Request, prompt_id: str, amount: int, is_anon: boo
|
|
| 669 |
Transaction.tx_type == "TIP_OUT",
|
| 670 |
Transaction.amount == -amount,
|
| 671 |
Transaction.related_account == author,
|
|
|
|
|
|
|
| 672 |
Transaction.created_at >= recent_cutoff
|
| 673 |
).with_for_update().first()
|
| 674 |
if duplicate_tx:
|
|
@@ -905,28 +912,8 @@ async def purchase_prompt(request: Request, prompt_id: str,
|
|
| 905 |
|
| 906 |
@router.get("/api/prompts/{prompt_id}/comments")
|
| 907 |
async def get_prompt_comments(prompt_id: str):
|
| 908 |
-
"""
|
| 909 |
-
|
| 910 |
-
"""
|
| 911 |
-
comments_db = db.load_data("comments.json", default_data={})
|
| 912 |
-
users_db = db.load_data("users.json", default_data={})
|
| 913 |
-
user_map = users_db
|
| 914 |
-
|
| 915 |
-
# comments_db 是 {item_id: [comments]} 格式
|
| 916 |
-
prompt_comments = comments_db.get(prompt_id, [])
|
| 917 |
-
|
| 918 |
-
# 附加用户信息
|
| 919 |
-
result = []
|
| 920 |
-
for c in prompt_comments:
|
| 921 |
-
author_info = user_map.get(c.get("author"), {})
|
| 922 |
-
result.append({
|
| 923 |
-
**c,
|
| 924 |
-
"author_name": author_info.get("name", c.get("author")),
|
| 925 |
-
"author_avatar": author_info.get("avatarDataUrl", "")
|
| 926 |
-
})
|
| 927 |
-
|
| 928 |
-
return {"status": "success", "data": result}
|
| 929 |
-
|
| 930 |
|
| 931 |
@router.post("/api/prompts/{prompt_id}/comments")
|
| 932 |
@shared_limit(10)
|
|
@@ -936,6 +923,8 @@ async def add_prompt_comment(request: Request, prompt_id: str, content: str, cur
|
|
| 936 |
"""
|
| 937 |
if not content or not content.strip():
|
| 938 |
raise HTTPException(status_code=400, detail="评论内容不能为空")
|
|
|
|
|
|
|
| 939 |
|
| 940 |
prompts_db = db.load_data("prompts.json", default_data=[])
|
| 941 |
|
|
|
|
| 18 |
from models import PromptCreate, PromptUpdate, PromptCategoryAdd, PromptPurchaseRequest
|
| 19 |
from typing import Optional
|
| 20 |
import 数据库连接 as db
|
| 21 |
+
from comment_store import COMMENT_MAX_LENGTH, append_comment, increment_comment_count, attach_comment_counts, read_comment_view
|
| 22 |
from 安全认证 import require_auth, optional_auth, is_admin
|
| 23 |
+
from db_utils import read_public_source, record_view, sort_cache
|
| 24 |
from database_sql import get_db
|
| 25 |
from models_sql import Wallet, Transaction, Ownership
|
| 26 |
from notifications import add_notification
|
|
|
|
| 232 |
- sort: latest / likes / favorites / views / daily_views / tips
|
| 233 |
⚠️ 列表接口永不下发 prompt_text(付费内容脱敏)
|
| 234 |
"""
|
| 235 |
+
read_epoch = sort_cache.capture_read_epoch()
|
| 236 |
prompts_db = await _prompt_operation(read_prompts, db)
|
| 237 |
+
users_db = read_public_source(db, "users.json", default_data={})
|
| 238 |
user_map = users_db
|
| 239 |
|
| 240 |
# 🗂️ 使用排序缓存优化排序性能
|
| 241 |
+
cache_sort = sort if sort in ("latest", "likes", "favorites", "views", "daily_views", "tips") else "latest"
|
| 242 |
+
cache_key = f"prompts:{cache_sort}"
|
| 243 |
|
| 244 |
+
def order_key(item):
|
| 245 |
if sort == "likes":
|
| 246 |
+
return item.get("likes", 0)
|
| 247 |
elif sort == "favorites":
|
| 248 |
+
return item.get("favorites", 0)
|
| 249 |
elif sort == "views":
|
| 250 |
+
return item.get("views", 0)
|
| 251 |
elif sort == "daily_views":
|
| 252 |
+
return item.get("daily_views", 0)
|
| 253 |
elif sort == "tips":
|
| 254 |
+
return sum(t.get("amount", 0) for t in item.get("tip_board", []))
|
| 255 |
else: # latest 或其他默认
|
| 256 |
+
return item.get("created_at", 0)
|
| 257 |
+
|
| 258 |
+
def sort_fn(data):
|
| 259 |
+
data.sort(key=order_key, reverse=True)
|
| 260 |
|
| 261 |
+
sorted_prompts = sort_cache.get_sorted(cache_key, prompts_db, sort_fn, read_epoch=read_epoch, fresh=True)
|
| 262 |
|
| 263 |
# 类型与分类过滤
|
| 264 |
if prompt_type in VALID_PROMPT_TYPES:
|
|
|
|
| 674 |
Transaction.tx_type == "TIP_OUT",
|
| 675 |
Transaction.amount == -amount,
|
| 676 |
Transaction.related_account == author,
|
| 677 |
+
Transaction.item_id == prompt_id,
|
| 678 |
+
Transaction.item_type == "prompt",
|
| 679 |
Transaction.created_at >= recent_cutoff
|
| 680 |
).with_for_update().first()
|
| 681 |
if duplicate_tx:
|
|
|
|
| 912 |
|
| 913 |
@router.get("/api/prompts/{prompt_id}/comments")
|
| 914 |
async def get_prompt_comments(prompt_id: str):
|
| 915 |
+
"""获取评论;有界后台读取并保留原作者附加字段。"""
|
| 916 |
+
return {"status": "success", "data": await read_comment_view(prompt_id)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 917 |
|
| 918 |
@router.post("/api/prompts/{prompt_id}/comments")
|
| 919 |
@shared_limit(10)
|
|
|
|
| 923 |
"""
|
| 924 |
if not content or not content.strip():
|
| 925 |
raise HTTPException(status_code=400, detail="评论内容不能为空")
|
| 926 |
+
if len(content) > COMMENT_MAX_LENGTH:
|
| 927 |
+
raise HTTPException(status_code=400, detail="评论长度必须在1-5000字符之间")
|
| 928 |
|
| 929 |
prompts_db = db.load_data("prompts.json", default_data=[])
|
| 930 |
|
router_proxy.py
CHANGED
|
@@ -26,6 +26,7 @@ from safe_http_transport import ValidatedAsyncHTTPTransport
|
|
| 26 |
from workflow_download_service import prepare_workflow_download
|
| 27 |
from zip_stream import ZIP_DOWNLOAD_MAX_BYTES
|
| 28 |
from github_zip_service import prepare_github_zip
|
|
|
|
| 29 |
|
| 30 |
router = APIRouter()
|
| 31 |
MAX_PROXY_DOWNLOAD_BYTES = int(os.environ.get("MAX_PROXY_DOWNLOAD_BYTES", str(256 * 1024 * 1024)))
|
|
@@ -55,7 +56,9 @@ async def _validate_public_http_target(url: str) -> str:
|
|
| 55 |
except OSError:
|
| 56 |
raise HTTPException(status_code=400, detail="下载地址无法解析")
|
| 57 |
addresses = [ipaddress.ip_address(info[4][0].split("%", 1)[0]) for info in infos]
|
| 58 |
-
if not addresses or any(not address.is_global
|
|
|
|
|
|
|
| 59 |
raise HTTPException(status_code=400, detail="下载地址解析到非公网地址")
|
| 60 |
return host
|
| 61 |
|
|
@@ -92,7 +95,7 @@ async def _send_with_validated_redirects(client, method: str, url: str, headers=
|
|
| 92 |
if response.status_code not in (301, 302, 303, 307, 308):
|
| 93 |
return response
|
| 94 |
location = response.headers.get("location")
|
| 95 |
-
await
|
| 96 |
if not location or redirect_count >= max_redirects:
|
| 97 |
raise HTTPException(status_code=502, detail="下载源重定向无效或次数过多")
|
| 98 |
current_url = urljoin(current_url, location)
|
|
@@ -167,10 +170,11 @@ class ProxyDownloadRequest(BaseModel):
|
|
| 167 |
# 检查是否是私有IP
|
| 168 |
try:
|
| 169 |
ip = ipaddress.ip_address(parsed.hostname)
|
| 170 |
-
if ip.is_private or ip.is_loopback or ip.is_reserved:
|
| 171 |
-
raise ValueError('不允许访问内网地址')
|
| 172 |
except ValueError:
|
| 173 |
-
|
|
|
|
|
|
|
|
|
|
| 174 |
|
| 175 |
return v
|
| 176 |
|
|
|
|
| 26 |
from workflow_download_service import prepare_workflow_download
|
| 27 |
from zip_stream import ZIP_DOWNLOAD_MAX_BYTES
|
| 28 |
from github_zip_service import prepare_github_zip
|
| 29 |
+
from proxy_resources import close_redirect_response
|
| 30 |
|
| 31 |
router = APIRouter()
|
| 32 |
MAX_PROXY_DOWNLOAD_BYTES = int(os.environ.get("MAX_PROXY_DOWNLOAD_BYTES", str(256 * 1024 * 1024)))
|
|
|
|
| 56 |
except OSError:
|
| 57 |
raise HTTPException(status_code=400, detail="下载地址无法解析")
|
| 58 |
addresses = [ipaddress.ip_address(info[4][0].split("%", 1)[0]) for info in infos]
|
| 59 |
+
if not addresses or any(not address.is_global or address.is_multicast
|
| 60 |
+
or (getattr(address, 'ipv4_mapped', None) is not None
|
| 61 |
+
and address.ipv4_mapped.is_multicast) for address in addresses):
|
| 62 |
raise HTTPException(status_code=400, detail="下载地址解析到非公网地址")
|
| 63 |
return host
|
| 64 |
|
|
|
|
| 95 |
if response.status_code not in (301, 302, 303, 307, 308):
|
| 96 |
return response
|
| 97 |
location = response.headers.get("location")
|
| 98 |
+
await close_redirect_response(response)
|
| 99 |
if not location or redirect_count >= max_redirects:
|
| 100 |
raise HTTPException(status_code=502, detail="下载源重定向无效或次数过多")
|
| 101 |
current_url = urljoin(current_url, location)
|
|
|
|
| 170 |
# 检查是否是私有IP
|
| 171 |
try:
|
| 172 |
ip = ipaddress.ip_address(parsed.hostname)
|
|
|
|
|
|
|
| 173 |
except ValueError:
|
| 174 |
+
return v # 域名仍由实际请求的逐跳DNS与传输层校验。
|
| 175 |
+
# 拒绝异常必须在解析异常处理之外,否则会被当作域名吞掉。
|
| 176 |
+
if ip.is_private or ip.is_loopback or ip.is_reserved:
|
| 177 |
+
raise ValueError('不允许访问内网地址')
|
| 178 |
|
| 179 |
return v
|
| 180 |
|
router_tasks.py
CHANGED
|
@@ -16,7 +16,7 @@ from rate_limit import shared_limit
|
|
| 16 |
from sqlalchemy.orm import Session
|
| 17 |
from models import TaskCreate, TaskUpdate, TaskSubmit, TaskDispute, TaskDisputeResponse # 🔧 P2修复:提交/申诉/回应端点改为 body 模型,匹配前端 JSON 提交
|
| 18 |
import 数据库连接 as db
|
| 19 |
-
from comment_store import append_comment, increment_comment_count, attach_comment_counts
|
| 20 |
from 安全认证 import require_auth, optional_auth, is_admin
|
| 21 |
from notifications import add_notification
|
| 22 |
from task_publication import freeze_task, project_publications, InsufficientTaskBalance
|
|
@@ -932,6 +932,8 @@ async def tip_task(request: Request, task_id: str, amount: int, is_anon: bool =
|
|
| 932 |
Transaction.tx_type == "TIP_OUT",
|
| 933 |
Transaction.amount == -amount,
|
| 934 |
Transaction.related_account == publisher,
|
|
|
|
|
|
|
| 935 |
Transaction.created_at >= recent_cutoff
|
| 936 |
).with_for_update().first()
|
| 937 |
if duplicate_tx:
|
|
@@ -1060,29 +1062,8 @@ async def tip_task(request: Request, task_id: str, amount: int, is_anon: bool =
|
|
| 1060 |
|
| 1061 |
@router.get("/api/tasks/{task_id}/comments")
|
| 1062 |
async def get_task_comments(task_id: str):
|
| 1063 |
-
"""
|
| 1064 |
-
|
| 1065 |
-
"""
|
| 1066 |
-
comments_db = db.load_data("comments.json", default_data={})
|
| 1067 |
-
users_db = db.load_data("users.json", default_data={})
|
| 1068 |
-
|
| 1069 |
-
# users_db 已经是 {account: user_info} 格式,直接使用
|
| 1070 |
-
user_map = users_db
|
| 1071 |
-
|
| 1072 |
-
# comments_db 是 {item_id: [comments]} 格式
|
| 1073 |
-
task_comments = comments_db.get(task_id, [])
|
| 1074 |
-
|
| 1075 |
-
# 附加用户信息
|
| 1076 |
-
result = []
|
| 1077 |
-
for c in task_comments:
|
| 1078 |
-
author_info = user_map.get(c.get("author"), {})
|
| 1079 |
-
result.append({
|
| 1080 |
-
**c,
|
| 1081 |
-
"author_name": author_info.get("name", c.get("author")),
|
| 1082 |
-
"author_avatar": author_info.get("avatarDataUrl", "")
|
| 1083 |
-
})
|
| 1084 |
-
|
| 1085 |
-
return {"status": "success", "data": result}
|
| 1086 |
|
| 1087 |
@router.post("/api/tasks/{task_id}/comments")
|
| 1088 |
@shared_limit(10)
|
|
@@ -1092,6 +1073,8 @@ async def add_task_comment(request: Request, task_id: str, content: str, current
|
|
| 1092 |
"""
|
| 1093 |
if not content or not content.strip():
|
| 1094 |
raise HTTPException(status_code=400, detail="评论内容不能为空")
|
|
|
|
|
|
|
| 1095 |
|
| 1096 |
tasks_db = db.load_data("tasks.json", default_data=[])
|
| 1097 |
|
|
|
|
| 16 |
from sqlalchemy.orm import Session
|
| 17 |
from models import TaskCreate, TaskUpdate, TaskSubmit, TaskDispute, TaskDisputeResponse # 🔧 P2修复:提交/申诉/回应端点改为 body 模型,匹配前端 JSON 提交
|
| 18 |
import 数据库连接 as db
|
| 19 |
+
from comment_store import COMMENT_MAX_LENGTH, append_comment, increment_comment_count, attach_comment_counts, read_comment_view
|
| 20 |
from 安全认证 import require_auth, optional_auth, is_admin
|
| 21 |
from notifications import add_notification
|
| 22 |
from task_publication import freeze_task, project_publications, InsufficientTaskBalance
|
|
|
|
| 932 |
Transaction.tx_type == "TIP_OUT",
|
| 933 |
Transaction.amount == -amount,
|
| 934 |
Transaction.related_account == publisher,
|
| 935 |
+
Transaction.item_id == task_id,
|
| 936 |
+
Transaction.item_type == "task",
|
| 937 |
Transaction.created_at >= recent_cutoff
|
| 938 |
).with_for_update().first()
|
| 939 |
if duplicate_tx:
|
|
|
|
| 1062 |
|
| 1063 |
@router.get("/api/tasks/{task_id}/comments")
|
| 1064 |
async def get_task_comments(task_id: str):
|
| 1065 |
+
"""获取评论;有界后台读取并保留原作者附加字段。"""
|
| 1066 |
+
return {"status": "success", "data": await read_comment_view(task_id)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1067 |
|
| 1068 |
@router.post("/api/tasks/{task_id}/comments")
|
| 1069 |
@shared_limit(10)
|
|
|
|
| 1073 |
"""
|
| 1074 |
if not content or not content.strip():
|
| 1075 |
raise HTTPException(status_code=400, detail="评论内容不能为空")
|
| 1076 |
+
if len(content) > COMMENT_MAX_LENGTH:
|
| 1077 |
+
raise HTTPException(status_code=400, detail="评论长度必须在1-5000字符之间")
|
| 1078 |
|
| 1079 |
tasks_db = db.load_data("tasks.json", default_data=[])
|
| 1080 |
|
router_users_profile.py
CHANGED
|
@@ -15,10 +15,11 @@
|
|
| 15 |
from fastapi import APIRouter, HTTPException, Depends, Request
|
| 16 |
from rate_limit import shared_limit
|
| 17 |
import 数据库连接 as db
|
| 18 |
-
from models import UserUpdate
|
| 19 |
from 安全认证 import require_auth, optional_auth
|
| 20 |
-
from user_store import update_profile, project_profile, read_user, attach_profile_counts
|
| 21 |
import logging
|
|
|
|
| 22 |
|
| 23 |
logger = logging.getLogger(__name__)
|
| 24 |
|
|
@@ -49,6 +50,10 @@ async def get_user_profile(account: str, current_user: str = Depends(optional_au
|
|
| 49 |
- receivedFavorites: 收到的总收藏数
|
| 50 |
- receivedUses: 发布内容被使用的总次数
|
| 51 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
user_data = read_user(account)
|
| 53 |
# 作品数据读取失败不当作空列表清零既有计数,也不阻断有效资料。
|
| 54 |
try:
|
|
@@ -101,5 +106,20 @@ async def update_user_profile(request: Request, account: str, update_data: UserU
|
|
| 101 |
raise HTTPException(status_code=400, detail="个人介绍不能超过100个字符")
|
| 102 |
dump = getattr(update_data, "model_dump", None) or update_data.dict
|
| 103 |
# 读取、账号存在性检查、字段更新及保存均在同一文件锁内。
|
| 104 |
-
user =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
return {"status": "success", "data": user}
|
|
|
|
| 15 |
from fastapi import APIRouter, HTTPException, Depends, Request
|
| 16 |
from rate_limit import shared_limit
|
| 17 |
import 数据库连接 as db
|
| 18 |
+
from models import UserUpdate, UserSettings
|
| 19 |
from 安全认证 import require_auth, optional_auth
|
| 20 |
+
from user_store import update_settings, update_profile, project_profile, read_user, attach_profile_counts
|
| 21 |
import logging
|
| 22 |
+
from profile_io import run_profile_io
|
| 23 |
|
| 24 |
logger = logging.getLogger(__name__)
|
| 25 |
|
|
|
|
| 50 |
- receivedFavorites: 收到的总收藏数
|
| 51 |
- receivedUses: 发布内容被使用的总次数
|
| 52 |
"""
|
| 53 |
+
return await run_profile_io(_read_profile_response, account, current_user)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _read_profile_response(account, current_user):
|
| 57 |
user_data = read_user(account)
|
| 58 |
# 作品数据读取失败不当作空列表清零既有计数,也不阻断有效资料。
|
| 59 |
try:
|
|
|
|
| 106 |
raise HTTPException(status_code=400, detail="个人介绍不能超过100个字符")
|
| 107 |
dump = getattr(update_data, "model_dump", None) or update_data.dict
|
| 108 |
# 读取、账号存在性检查、字段更新及保存均在同一文件锁内。
|
| 109 |
+
user = await run_profile_io(update_profile, account, dump(exclude_unset=True))
|
| 110 |
+
return {"status": "success", "data": user}
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
@router.put("/api/users/{account}/settings")
|
| 114 |
+
@shared_limit(10)
|
| 115 |
+
async def update_user_settings(request: Request, account: str, update_data: UserSettings, current_user: str = Depends(require_auth)):
|
| 116 |
+
"""一次用户文件发布共同保存资料和完整隐私设置;旧独立接口保持。"""
|
| 117 |
+
if account != current_user:
|
| 118 |
+
raise HTTPException(status_code=403, detail="只能更新自己的资料")
|
| 119 |
+
if update_data.intro and len(update_data.intro) > 100:
|
| 120 |
+
raise HTTPException(status_code=400, detail="个人介绍不能超过100个字符")
|
| 121 |
+
dump = getattr(update_data, "model_dump", None) or update_data.dict
|
| 122 |
+
payload = dump(exclude_unset=True)
|
| 123 |
+
privacy = payload.pop("privacy")
|
| 124 |
+
user = await run_profile_io(update_settings, account, payload, privacy)
|
| 125 |
return {"status": "success", "data": user}
|
router_wallet.py
CHANGED
|
@@ -354,6 +354,16 @@ async def tip_user(request: Request, req: TipRequest, current_user: str = Depend
|
|
| 354 |
# 🔒 P1并发安全:使用悲观锁+异常处理+事务回滚防止双花问题
|
| 355 |
try:
|
| 356 |
sender_wallet, target_wallet = lock_tip_wallets(db, req.sender_account, req.target_account)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 357 |
# 🔒 P1幂等性防护:检查最近30秒内是否存在相同交易
|
| 358 |
recent_cutoff = datetime.datetime.utcnow() - datetime.timedelta(seconds=30)
|
| 359 |
duplicate_tx = db.query(Transaction).filter(
|
|
@@ -361,6 +371,8 @@ async def tip_user(request: Request, req: TipRequest, current_user: str = Depend
|
|
| 361 |
Transaction.tx_type == "TIP_OUT",
|
| 362 |
Transaction.amount == -req.amount,
|
| 363 |
Transaction.related_account == req.target_account,
|
|
|
|
|
|
|
| 364 |
Transaction.created_at >= recent_cutoff
|
| 365 |
).with_for_update().first()
|
| 366 |
if duplicate_tx:
|
|
@@ -391,16 +403,6 @@ async def tip_user(request: Request, req: TipRequest, current_user: str = Depend
|
|
| 391 |
target_user_name = target_user_info.get("name", req.target_account)
|
| 392 |
sender_user_name = sender_user_info.get("name", req.sender_account)
|
| 393 |
|
| 394 |
-
# 如果有关联资源,获取资源信息
|
| 395 |
-
item_title = None
|
| 396 |
-
item_type = None
|
| 397 |
-
if req.item_id:
|
| 398 |
-
items_db = json_db.load_data("items.json", default_data=[])
|
| 399 |
-
item = next((i for i in items_db if i["id"] == req.item_id), None)
|
| 400 |
-
if item:
|
| 401 |
-
item_title = item.get("title")
|
| 402 |
-
item_type = item.get("type")
|
| 403 |
-
|
| 404 |
# 发送方交易记录 (字段名为 related_account,与 models_sql.py 中 Transaction 模型保持一致)
|
| 405 |
tx_sender = Transaction(
|
| 406 |
tx_id=tx_id_sender, account=req.sender_account, tx_type="TIP_OUT", amount=-req.amount,
|
|
|
|
| 354 |
# 🔒 P1并发安全:使用悲观锁+异常处理+事务回滚防止双花问题
|
| 355 |
try:
|
| 356 |
sender_wallet, target_wallet = lock_tip_wallets(db, req.sender_account, req.target_account)
|
| 357 |
+
# 如果有关联资源,获取资源信息
|
| 358 |
+
item_title = None
|
| 359 |
+
item_type = None
|
| 360 |
+
if req.item_id:
|
| 361 |
+
items_db = json_db.load_data("items.json", default_data=[])
|
| 362 |
+
item = next((i for i in items_db if i["id"] == req.item_id), None)
|
| 363 |
+
if item:
|
| 364 |
+
item_title = item.get("title")
|
| 365 |
+
item_type = item.get("type")
|
| 366 |
+
|
| 367 |
# 🔒 P1幂等性防护:检查最近30秒内是否存在相同交易
|
| 368 |
recent_cutoff = datetime.datetime.utcnow() - datetime.timedelta(seconds=30)
|
| 369 |
duplicate_tx = db.query(Transaction).filter(
|
|
|
|
| 371 |
Transaction.tx_type == "TIP_OUT",
|
| 372 |
Transaction.amount == -req.amount,
|
| 373 |
Transaction.related_account == req.target_account,
|
| 374 |
+
Transaction.item_id == req.item_id,
|
| 375 |
+
Transaction.item_type == item_type,
|
| 376 |
Transaction.created_at >= recent_cutoff
|
| 377 |
).with_for_update().first()
|
| 378 |
if duplicate_tx:
|
|
|
|
| 403 |
target_user_name = target_user_info.get("name", req.target_account)
|
| 404 |
sender_user_name = sender_user_info.get("name", req.sender_account)
|
| 405 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 406 |
# 发送方交易记录 (字段名为 related_account,与 models_sql.py 中 Transaction 模型保持一致)
|
| 407 |
tx_sender = Transaction(
|
| 408 |
tx_id=tx_id_sender, account=req.sender_account, tx_type="TIP_OUT", amount=-req.amount,
|
safe_http_transport.py
CHANGED
|
@@ -28,7 +28,9 @@ async def _resolve_public_ips(host: str, port: int) -> tuple[str, ...]:
|
|
| 28 |
for record in records
|
| 29 |
]
|
| 30 |
|
| 31 |
-
if not addresses or any(not address.is_global
|
|
|
|
|
|
|
| 32 |
raise httpx.ConnectError("目标域名解析到非公网地址")
|
| 33 |
return tuple(dict.fromkeys(str(address) for address in addresses))
|
| 34 |
|
|
|
|
| 28 |
for record in records
|
| 29 |
]
|
| 30 |
|
| 31 |
+
if not addresses or any(not address.is_global or address.is_multicast
|
| 32 |
+
or (getattr(address, 'ipv4_mapped', None) is not None
|
| 33 |
+
and address.ipv4_mapped.is_multicast) for address in addresses):
|
| 34 |
raise httpx.ConnectError("目标域名解析到非公网地址")
|
| 35 |
return tuple(dict.fromkeys(str(address) for address in addresses))
|
| 36 |
|
script_execution.py
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""管理脚本的有界输出、每进程准入及所属线程终态等待。"""
|
| 2 |
+
import asyncio
|
| 3 |
+
import logging
|
| 4 |
+
import os
|
| 5 |
+
import subprocess
|
| 6 |
+
import sys
|
| 7 |
+
import threading
|
| 8 |
+
import time
|
| 9 |
+
|
| 10 |
+
from owned_thread import run_owned_thread
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class ScriptExecutionBusy(Exception):
|
| 14 |
+
pass
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
_SCRIPT_TIMEOUT = 30
|
| 18 |
+
_OUTPUT_LIMIT = 64 * 1024
|
| 19 |
+
_READ_CHUNK = 16 * 1024
|
| 20 |
+
logger = logging.getLogger("ComfyUI-Ranking.AdminScripts")
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _pipe_reader(stream):
|
| 24 |
+
descriptor = stream.fileno()
|
| 25 |
+
if os.name == "nt":
|
| 26 |
+
import ctypes
|
| 27 |
+
from ctypes import wintypes
|
| 28 |
+
import msvcrt
|
| 29 |
+
peek = ctypes.WinDLL("kernel32", use_last_error=True).PeekNamedPipe
|
| 30 |
+
peek.argtypes = [wintypes.HANDLE, wintypes.LPVOID, wintypes.DWORD,
|
| 31 |
+
ctypes.POINTER(wintypes.DWORD), ctypes.POINTER(wintypes.DWORD),
|
| 32 |
+
ctypes.POINTER(wintypes.DWORD)]
|
| 33 |
+
peek.restype = wintypes.BOOL
|
| 34 |
+
handle = msvcrt.get_osfhandle(descriptor)
|
| 35 |
+
|
| 36 |
+
def read():
|
| 37 |
+
available = wintypes.DWORD()
|
| 38 |
+
if not peek(handle, None, 0, None, ctypes.byref(available), None):
|
| 39 |
+
error = ctypes.get_last_error()
|
| 40 |
+
if error in (109, 232):
|
| 41 |
+
return b""
|
| 42 |
+
raise ctypes.WinError(error)
|
| 43 |
+
if not available.value:
|
| 44 |
+
return None
|
| 45 |
+
return os.read(descriptor, min(_READ_CHUNK, available.value))
|
| 46 |
+
else:
|
| 47 |
+
os.set_blocking(descriptor, False)
|
| 48 |
+
|
| 49 |
+
def read():
|
| 50 |
+
try:
|
| 51 |
+
return os.read(descriptor, _READ_CHUNK)
|
| 52 |
+
except BlockingIOError:
|
| 53 |
+
return None
|
| 54 |
+
return read
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def _run_script(script_path, *, cwd, env):
|
| 58 |
+
command = [sys.executable, script_path]
|
| 59 |
+
deadline = time.monotonic() + _SCRIPT_TIMEOUT
|
| 60 |
+
process = subprocess.Popen(
|
| 61 |
+
command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
| 62 |
+
bufsize=0, cwd=cwd, env=env,
|
| 63 |
+
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
|
| 64 |
+
)
|
| 65 |
+
streams = (process.stdout, process.stderr)
|
| 66 |
+
buffers = [bytearray(), bytearray()]
|
| 67 |
+
truncated = [False, False]
|
| 68 |
+
try:
|
| 69 |
+
readers = [_pipe_reader(stream) for stream in streams]
|
| 70 |
+
active = {0, 1}
|
| 71 |
+
while active or process.poll() is None:
|
| 72 |
+
if time.monotonic() >= deadline:
|
| 73 |
+
raise subprocess.TimeoutExpired(command, _SCRIPT_TIMEOUT)
|
| 74 |
+
received = False
|
| 75 |
+
for index in tuple(active):
|
| 76 |
+
chunk = readers[index]()
|
| 77 |
+
if chunk is None:
|
| 78 |
+
continue
|
| 79 |
+
if not chunk:
|
| 80 |
+
active.remove(index)
|
| 81 |
+
continue
|
| 82 |
+
received = True
|
| 83 |
+
remaining = _OUTPUT_LIMIT - len(buffers[index])
|
| 84 |
+
buffers[index].extend(chunk[:remaining])
|
| 85 |
+
truncated[index] |= len(chunk) > remaining
|
| 86 |
+
if not received:
|
| 87 |
+
time.sleep(min(.01, max(0, deadline - time.monotonic())))
|
| 88 |
+
outputs = [buffer.decode("utf-8", errors="replace").replace("\r\n", "\n").replace("\r", "\n") + ("\n...(输出已截断)" if cut else "")
|
| 89 |
+
for buffer, cut in zip(buffers, truncated)]
|
| 90 |
+
return subprocess.CompletedProcess(command, process.returncode, *outputs)
|
| 91 |
+
finally:
|
| 92 |
+
try:
|
| 93 |
+
if process.poll() is None:
|
| 94 |
+
process.kill()
|
| 95 |
+
finally:
|
| 96 |
+
try:
|
| 97 |
+
process.wait()
|
| 98 |
+
finally:
|
| 99 |
+
for stream in streams:
|
| 100 |
+
try:
|
| 101 |
+
stream.close()
|
| 102 |
+
except Exception as error:
|
| 103 |
+
logger.error("ADMIN_SCRIPT_PIPE_CLOSE_ERROR | error_type=%s", type(error).__name__)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
class ScriptExecution:
|
| 107 |
+
def __init__(self):
|
| 108 |
+
self._gate = threading.Lock()
|
| 109 |
+
self._accepting = True
|
| 110 |
+
self._pending = False
|
| 111 |
+
self._closers = 0
|
| 112 |
+
self._drained = threading.Event()
|
| 113 |
+
self._drained.set()
|
| 114 |
+
|
| 115 |
+
def start(self):
|
| 116 |
+
with self._gate:
|
| 117 |
+
if self._pending or self._closers:
|
| 118 |
+
raise RuntimeError("script work is still pending")
|
| 119 |
+
self._accepting = True
|
| 120 |
+
|
| 121 |
+
def stop(self):
|
| 122 |
+
with self._gate:
|
| 123 |
+
self._accepting = False
|
| 124 |
+
|
| 125 |
+
async def close(self):
|
| 126 |
+
with self._gate:
|
| 127 |
+
self._accepting = False
|
| 128 |
+
self._closers += 1
|
| 129 |
+
cancelled = None
|
| 130 |
+
try:
|
| 131 |
+
while not self._drained.is_set():
|
| 132 |
+
try:
|
| 133 |
+
await asyncio.sleep(.01)
|
| 134 |
+
except asyncio.CancelledError as error:
|
| 135 |
+
cancelled = error
|
| 136 |
+
if cancelled is not None:
|
| 137 |
+
raise cancelled
|
| 138 |
+
finally:
|
| 139 |
+
with self._gate:
|
| 140 |
+
self._closers -= 1
|
| 141 |
+
|
| 142 |
+
async def run(self, script_path, *, cwd, env):
|
| 143 |
+
with self._gate:
|
| 144 |
+
if not self._accepting or self._pending:
|
| 145 |
+
raise ScriptExecutionBusy()
|
| 146 |
+
self._pending = True
|
| 147 |
+
self._drained.clear()
|
| 148 |
+
try:
|
| 149 |
+
return await run_owned_thread(_run_script, script_path, cwd=cwd, env=env)
|
| 150 |
+
finally:
|
| 151 |
+
with self._gate:
|
| 152 |
+
self._pending = False
|
| 153 |
+
self._drained.set()
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
_script_execution = ScriptExecution()
|
| 157 |
+
run_admin_script_process = _script_execution.run
|
| 158 |
+
start_admin_scripts = _script_execution.start
|
| 159 |
+
stop_admin_scripts = _script_execution.stop
|
| 160 |
+
close_admin_scripts = _script_execution.close
|
task_projection.py
CHANGED
|
@@ -1,5 +1,7 @@
|
|
| 1 |
"""任务状态恢复逐条隔离,并用轮转游标避免坏记录占满每一批。"""
|
| 2 |
import asyncio
|
|
|
|
|
|
|
| 3 |
import logging
|
| 4 |
from sqlalchemy.orm import Session
|
| 5 |
|
|
@@ -37,7 +39,7 @@ def report_pending(stage, error, task_id=None):
|
|
| 37 |
pass
|
| 38 |
|
| 39 |
|
| 40 |
-
def recover_stage(bind, store, model, project, after=None, limit=50, should_stop=lambda: False):
|
| 41 |
"""只读选取独立任务;单条故障不确认、不删除,也不阻断下一条。"""
|
| 42 |
if type(limit) is not int or not 1 <= limit <= 500:
|
| 43 |
raise ValueError('invalid projection batch limit')
|
|
@@ -54,6 +56,12 @@ def recover_stage(bind, store, model, project, after=None, limit=50, should_stop
|
|
| 54 |
try:
|
| 55 |
count += project(bind, store, limit=1, task_id=task_id)
|
| 56 |
except Exception as error:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
report_pending(model.__tablename__, error, task_id)
|
| 58 |
# 失败也推进游标;下一轮走后续任务,到末尾再回头重试。
|
| 59 |
cursor = task_id
|
|
@@ -67,29 +75,50 @@ async def recovery_worker(bind, store, invalidate, stop, interval=30):
|
|
| 67 |
repair_cursor = None
|
| 68 |
while not stop.is_set():
|
| 69 |
count = 0
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
count += recovered
|
| 78 |
-
except Exception as error:
|
| 79 |
-
# 表/连接/批次读取故障仅影响该阶段;之后的阶段仍可恢复。
|
| 80 |
-
report_pending(model.__tablename__, error)
|
| 81 |
-
if not stop.is_set():
|
| 82 |
-
try:
|
| 83 |
-
recovered, repair_cursor = await asyncio.to_thread(
|
| 84 |
-
recover_snapshots, bind, store, repair_cursor, 50, stop.is_set)
|
| 85 |
-
count += recovered
|
| 86 |
-
except Exception as error:
|
| 87 |
-
report_pending('task_snapshots', error)
|
| 88 |
-
if count:
|
| 89 |
try:
|
| 90 |
-
|
| 91 |
-
except Exception
|
| 92 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
if stop.is_set():
|
| 94 |
break
|
| 95 |
try:
|
|
|
|
| 1 |
"""任务状态恢复逐条隔离,并用轮转游标避免坏记录占满每一批。"""
|
| 2 |
import asyncio
|
| 3 |
+
import threading
|
| 4 |
+
from owned_thread import run_owned_thread
|
| 5 |
import logging
|
| 6 |
from sqlalchemy.orm import Session
|
| 7 |
|
|
|
|
| 39 |
pass
|
| 40 |
|
| 41 |
|
| 42 |
+
def recover_stage(bind, store, model, project, after=None, limit=50, should_stop=lambda: False, *, on_uncertain=None):
|
| 43 |
"""只读选取独立任务;单条故障不确认、不删除,也不阻断下一条。"""
|
| 44 |
if type(limit) is not int or not 1 <= limit <= 500:
|
| 45 |
raise ValueError('invalid projection batch limit')
|
|
|
|
| 56 |
try:
|
| 57 |
count += project(bind, store, limit=1, task_id=task_id)
|
| 58 |
except Exception as error:
|
| 59 |
+
# 投影可先发布JSON再在SQL确认/同步处失败;零计数不代表未写入。
|
| 60 |
+
if on_uncertain is not None:
|
| 61 |
+
try:
|
| 62 |
+
on_uncertain()
|
| 63 |
+
except Exception as notification_error:
|
| 64 |
+
report_pending("sort_cache_notification", notification_error)
|
| 65 |
report_pending(model.__tablename__, error, task_id)
|
| 66 |
# 失败也推进游标;下一轮走后续任务,到末尾再回头重试。
|
| 67 |
cursor = task_id
|
|
|
|
| 75 |
repair_cursor = None
|
| 76 |
while not stop.is_set():
|
| 77 |
count = 0
|
| 78 |
+
attempted = threading.Event()
|
| 79 |
+
changed_data = threading.Event()
|
| 80 |
+
uncertain = threading.Event()
|
| 81 |
+
cancelled = False
|
| 82 |
+
|
| 83 |
+
def recover(function, *args):
|
| 84 |
+
attempted.set()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
try:
|
| 86 |
+
result = function(*args, on_uncertain=uncertain.set)
|
| 87 |
+
except Exception:
|
| 88 |
+
uncertain.set()
|
| 89 |
+
raise
|
| 90 |
+
if result[0]:
|
| 91 |
+
changed_data.set()
|
| 92 |
+
return result
|
| 93 |
+
|
| 94 |
+
try:
|
| 95 |
+
for model, project in stages:
|
| 96 |
+
if stop.is_set():
|
| 97 |
+
break
|
| 98 |
+
try:
|
| 99 |
+
recovered, cursor = await run_owned_thread(
|
| 100 |
+
recover, recover_stage, bind, store, model, project, cursors.get(model), 50, stop.is_set)
|
| 101 |
+
cursors[model] = cursor
|
| 102 |
+
count += recovered
|
| 103 |
+
except Exception as error:
|
| 104 |
+
# 表/连接/批次读取故障仅影响该阶段;之后的阶段仍可恢复。
|
| 105 |
+
report_pending(model.__tablename__, error)
|
| 106 |
+
if not stop.is_set():
|
| 107 |
+
try:
|
| 108 |
+
recovered, repair_cursor = await run_owned_thread(
|
| 109 |
+
recover, recover_snapshots, bind, store, repair_cursor, 50, stop.is_set)
|
| 110 |
+
count += recovered
|
| 111 |
+
except Exception as error:
|
| 112 |
+
report_pending('task_snapshots', error)
|
| 113 |
+
except asyncio.CancelledError:
|
| 114 |
+
cancelled = True
|
| 115 |
+
raise
|
| 116 |
+
finally:
|
| 117 |
+
if count or changed_data.is_set() or uncertain.is_set() or (cancelled and attempted.is_set()):
|
| 118 |
+
try:
|
| 119 |
+
invalidate('tasks:')
|
| 120 |
+
except Exception as error:
|
| 121 |
+
report_pending('sort_cache', error)
|
| 122 |
if stop.is_set():
|
| 123 |
break
|
| 124 |
try:
|
task_repair.py
CHANGED
|
@@ -1,106 +1,112 @@
|
|
| 1 |
-
"""定期核对已确认任务投影:仅修复有SQL事实的字段,不重新执行资金操作。"""
|
| 2 |
-
import copy
|
| 3 |
-
from sqlalchemy.orm import Session
|
| 4 |
-
from models_sql import TaskDisputeRecord, TaskDisputeResponseRecord, TaskResolution
|
| 5 |
-
from task_read import MODELS, merge_task_source
|
| 6 |
-
from task_dispute_read import merged_dispute
|
| 7 |
-
from task_projection import report_pending
|
| 8 |
-
from tip_transaction import lock_wallet_accounts
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
TASK_FIELDS = ('title', 'description', 'reference_images', 'reference_link', 'total_price',
|
| 12 |
-
'frozen_amount', 'deposit_ratio', 'deposit_amount', 'deadline', 'created_at',
|
| 13 |
-
'status', 'assignee', 'assigned_at', 'deposit_paid', 'deliverables', 'submit_note',
|
| 14 |
-
'submitted_at', 'feedback', 'completed_at', 'dispute_id', 'refunded', 'refund_amount')
|
| 15 |
-
DISPUTE_FIELDS = ('task_title', 'initiator', 'initiator_role', 'reason', 'evidence', 'created_at',
|
| 16 |
-
'status', 'response', 'response_evidence', 'responded_at', 'resolution',
|
| 17 |
-
'resolution_ratio', 'resolution_note', 'resolved_by', 'resolved_at')
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
def source_record(store, filename, identifier):
|
| 21 |
-
source = store.load_data(filename, default_data=[], strict=True, skip_cache=True)
|
| 22 |
-
if (not isinstance(source, list) or any(not isinstance(r, dict) or not isinstance(r.get('id'), str)
|
| 23 |
-
or not r['id'] for r in source) or len({r['id'] for r in source}) != len(source)):
|
| 24 |
-
raise ValueError('invalid repair source')
|
| 25 |
-
return next((r for r in source if r['id'] == identifier), None)
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
def repair_record(store, filename, canonical, expected, fields):
|
| 29 |
-
identifier = canonical['id']
|
| 30 |
-
patch = {k: copy.deepcopy(canonical[k]) for k in fields if k in canonical}
|
| 31 |
-
if expected is not None and all(k in expected and expected[k] == v for k, v in patch.items()):
|
| 32 |
-
return 0
|
| 33 |
-
def update(records):
|
| 34 |
-
if (not isinstance(records, list) or any(not isinstance(r, dict) or not isinstance(r.get('id'), str)
|
| 35 |
-
or not r['id'] for r in records) or len({r['id'] for r in records}) != len(records)):
|
| 36 |
-
raise ValueError('invalid atomic repair source')
|
| 37 |
-
current = next((r for r in records if r['id'] == identifier), None)
|
| 38 |
-
if expected is None:
|
| 39 |
-
if current is not None:
|
| 40 |
-
raise ValueError('record appeared during repair')
|
| 41 |
-
records.append(copy.deepcopy(canonical))
|
| 42 |
-
else:
|
| 43 |
-
if (current is None or any(current.get(k) != expected.get(k) or (k in current) != (k in expected)
|
| 44 |
-
for k in ('publisher', 'assignee', 'task_id', *fields))):
|
| 45 |
-
raise ValueError('record changed during repair')
|
| 46 |
-
current.update(patch) # 报名、互动和扩展字段保留回调中最新值。
|
| 47 |
-
store.atomic_update(filename, update, default_data=[])
|
| 48 |
-
return 1
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
def repair_task(bind, store, task_id):
|
| 52 |
-
with Session(bind=bind, autoflush=False) as session:
|
| 53 |
-
try:
|
| 54 |
-
owner = None
|
| 55 |
-
for model in MODELS:
|
| 56 |
-
row = session.query(model).filter(model.task_id == task_id).first()
|
| 57 |
-
if row is not None:
|
| 58 |
-
owner = row.publisher
|
| 59 |
-
break
|
| 60 |
-
if owner is None:
|
| 61 |
-
raise ValueError('durable task missing')
|
| 62 |
-
# 与编辑/交付/资金提交串行,读取与文件写入之间不会发生新SQL版本。
|
| 63 |
-
lock_wallet_accounts(session, (owner,))
|
| 64 |
-
expected = source_record(store, 'tasks.json', task_id)
|
| 65 |
-
tasks = merge_task_source(session, [] if expected is None else [expected], task_id)
|
| 66 |
-
if len(tasks) != 1 or tasks[0].get('publisher') != owner:
|
| 67 |
-
raise ValueError('repair task identity mismatch')
|
| 68 |
-
canonical = tasks[0]
|
| 69 |
-
count = repair_record(store, 'tasks.json', canonical, expected, TASK_FIELDS)
|
| 70 |
-
durable_dispute = None
|
| 71 |
-
for model in (TaskDisputeRecord, TaskDisputeResponseRecord, TaskResolution):
|
| 72 |
-
row = session.query(model).filter(model.task_id == task_id).populate_existing().one_or_none()
|
| 73 |
-
if row is not None:
|
| 74 |
-
if durable_dispute is not None and durable_dispute != row.dispute_id:
|
| 75 |
-
raise ValueError('conflicting repair disputes')
|
| 76 |
-
durable_dispute = row.dispute_id
|
| 77 |
-
if durable_dispute is not None:
|
| 78 |
-
expected = source_record(store, 'disputes.json', durable_dispute)
|
| 79 |
-
private = merged_dispute(session, [] if expected is None else [expected], durable_dispute, None, True)
|
| 80 |
-
if private.get('task_id') != task_id or private.get('publisher') != owner:
|
| 81 |
-
raise ValueError('repair dispute identity mismatch')
|
| 82 |
-
count += repair_record(store, 'disputes.json', private, expected, DISPUTE_FIELDS)
|
| 83 |
-
return count
|
| 84 |
-
finally:
|
| 85 |
-
session.rollback() # 只释放协调锁;不改变余额、流水或projected。
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
def recover_snapshots(bind, store, after=None, limit=50, should_stop=lambda: False):
|
| 89 |
-
if type(limit) is not int or not 1 <= limit <= 500:
|
| 90 |
-
raise ValueError('invalid repair batch limit')
|
| 91 |
-
with Session(bind=bind, autoflush=False) as session:
|
| 92 |
-
query = session.query(MODELS[0].task_id).union(*(session.query(m.task_id) for m in MODELS[1:]))
|
| 93 |
-
selected = query.filter(MODELS[0].task_id > after) if after is not None else query
|
| 94 |
-
pending = selected.order_by(MODELS[0].task_id).limit(limit).all()
|
| 95 |
-
if not pending and after is not None:
|
| 96 |
-
pending = query.order_by(MODELS[0].task_id).limit(limit).all()
|
| 97 |
-
count, cursor = 0, after
|
| 98 |
-
for (task_id,) in pending:
|
| 99 |
-
if should_stop():
|
| 100 |
-
break
|
| 101 |
-
try:
|
| 102 |
-
count += repair_task(bind, store, task_id)
|
| 103 |
-
except Exception as error:
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""定期核对已确认任务投影:仅修复有SQL事实的字段,不重新执行资金操作。"""
|
| 2 |
+
import copy
|
| 3 |
+
from sqlalchemy.orm import Session
|
| 4 |
+
from models_sql import TaskDisputeRecord, TaskDisputeResponseRecord, TaskResolution
|
| 5 |
+
from task_read import MODELS, merge_task_source
|
| 6 |
+
from task_dispute_read import merged_dispute
|
| 7 |
+
from task_projection import report_pending
|
| 8 |
+
from tip_transaction import lock_wallet_accounts
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
TASK_FIELDS = ('title', 'description', 'reference_images', 'reference_link', 'total_price',
|
| 12 |
+
'frozen_amount', 'deposit_ratio', 'deposit_amount', 'deadline', 'created_at',
|
| 13 |
+
'status', 'assignee', 'assigned_at', 'deposit_paid', 'deliverables', 'submit_note',
|
| 14 |
+
'submitted_at', 'feedback', 'completed_at', 'dispute_id', 'refunded', 'refund_amount')
|
| 15 |
+
DISPUTE_FIELDS = ('task_title', 'initiator', 'initiator_role', 'reason', 'evidence', 'created_at',
|
| 16 |
+
'status', 'response', 'response_evidence', 'responded_at', 'resolution',
|
| 17 |
+
'resolution_ratio', 'resolution_note', 'resolved_by', 'resolved_at')
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def source_record(store, filename, identifier):
|
| 21 |
+
source = store.load_data(filename, default_data=[], strict=True, skip_cache=True)
|
| 22 |
+
if (not isinstance(source, list) or any(not isinstance(r, dict) or not isinstance(r.get('id'), str)
|
| 23 |
+
or not r['id'] for r in source) or len({r['id'] for r in source}) != len(source)):
|
| 24 |
+
raise ValueError('invalid repair source')
|
| 25 |
+
return next((r for r in source if r['id'] == identifier), None)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def repair_record(store, filename, canonical, expected, fields):
|
| 29 |
+
identifier = canonical['id']
|
| 30 |
+
patch = {k: copy.deepcopy(canonical[k]) for k in fields if k in canonical}
|
| 31 |
+
if expected is not None and all(k in expected and expected[k] == v for k, v in patch.items()):
|
| 32 |
+
return 0
|
| 33 |
+
def update(records):
|
| 34 |
+
if (not isinstance(records, list) or any(not isinstance(r, dict) or not isinstance(r.get('id'), str)
|
| 35 |
+
or not r['id'] for r in records) or len({r['id'] for r in records}) != len(records)):
|
| 36 |
+
raise ValueError('invalid atomic repair source')
|
| 37 |
+
current = next((r for r in records if r['id'] == identifier), None)
|
| 38 |
+
if expected is None:
|
| 39 |
+
if current is not None:
|
| 40 |
+
raise ValueError('record appeared during repair')
|
| 41 |
+
records.append(copy.deepcopy(canonical))
|
| 42 |
+
else:
|
| 43 |
+
if (current is None or any(current.get(k) != expected.get(k) or (k in current) != (k in expected)
|
| 44 |
+
for k in ('publisher', 'assignee', 'task_id', *fields))):
|
| 45 |
+
raise ValueError('record changed during repair')
|
| 46 |
+
current.update(patch) # 报名、互动和扩展字段保留回调中最新值。
|
| 47 |
+
store.atomic_update(filename, update, default_data=[])
|
| 48 |
+
return 1
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def repair_task(bind, store, task_id):
|
| 52 |
+
with Session(bind=bind, autoflush=False) as session:
|
| 53 |
+
try:
|
| 54 |
+
owner = None
|
| 55 |
+
for model in MODELS:
|
| 56 |
+
row = session.query(model).filter(model.task_id == task_id).first()
|
| 57 |
+
if row is not None:
|
| 58 |
+
owner = row.publisher
|
| 59 |
+
break
|
| 60 |
+
if owner is None:
|
| 61 |
+
raise ValueError('durable task missing')
|
| 62 |
+
# 与编辑/交付/资金提交串行,读取与文件写入之间不会发生新SQL版本。
|
| 63 |
+
lock_wallet_accounts(session, (owner,))
|
| 64 |
+
expected = source_record(store, 'tasks.json', task_id)
|
| 65 |
+
tasks = merge_task_source(session, [] if expected is None else [expected], task_id)
|
| 66 |
+
if len(tasks) != 1 or tasks[0].get('publisher') != owner:
|
| 67 |
+
raise ValueError('repair task identity mismatch')
|
| 68 |
+
canonical = tasks[0]
|
| 69 |
+
count = repair_record(store, 'tasks.json', canonical, expected, TASK_FIELDS)
|
| 70 |
+
durable_dispute = None
|
| 71 |
+
for model in (TaskDisputeRecord, TaskDisputeResponseRecord, TaskResolution):
|
| 72 |
+
row = session.query(model).filter(model.task_id == task_id).populate_existing().one_or_none()
|
| 73 |
+
if row is not None:
|
| 74 |
+
if durable_dispute is not None and durable_dispute != row.dispute_id:
|
| 75 |
+
raise ValueError('conflicting repair disputes')
|
| 76 |
+
durable_dispute = row.dispute_id
|
| 77 |
+
if durable_dispute is not None:
|
| 78 |
+
expected = source_record(store, 'disputes.json', durable_dispute)
|
| 79 |
+
private = merged_dispute(session, [] if expected is None else [expected], durable_dispute, None, True)
|
| 80 |
+
if private.get('task_id') != task_id or private.get('publisher') != owner:
|
| 81 |
+
raise ValueError('repair dispute identity mismatch')
|
| 82 |
+
count += repair_record(store, 'disputes.json', private, expected, DISPUTE_FIELDS)
|
| 83 |
+
return count
|
| 84 |
+
finally:
|
| 85 |
+
session.rollback() # 只释放协调锁;不改变余额、流水或projected。
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def recover_snapshots(bind, store, after=None, limit=50, should_stop=lambda: False, *, on_uncertain=None):
|
| 89 |
+
if type(limit) is not int or not 1 <= limit <= 500:
|
| 90 |
+
raise ValueError('invalid repair batch limit')
|
| 91 |
+
with Session(bind=bind, autoflush=False) as session:
|
| 92 |
+
query = session.query(MODELS[0].task_id).union(*(session.query(m.task_id) for m in MODELS[1:]))
|
| 93 |
+
selected = query.filter(MODELS[0].task_id > after) if after is not None else query
|
| 94 |
+
pending = selected.order_by(MODELS[0].task_id).limit(limit).all()
|
| 95 |
+
if not pending and after is not None:
|
| 96 |
+
pending = query.order_by(MODELS[0].task_id).limit(limit).all()
|
| 97 |
+
count, cursor = 0, after
|
| 98 |
+
for (task_id,) in pending:
|
| 99 |
+
if should_stop():
|
| 100 |
+
break
|
| 101 |
+
try:
|
| 102 |
+
count += repair_task(bind, store, task_id)
|
| 103 |
+
except Exception as error:
|
| 104 |
+
# tasks写入后,disputes修复或释放SQL锁仍可能失败。
|
| 105 |
+
if on_uncertain is not None:
|
| 106 |
+
try:
|
| 107 |
+
on_uncertain()
|
| 108 |
+
except Exception as notification_error:
|
| 109 |
+
report_pending('sort_cache_notification', notification_error)
|
| 110 |
+
report_pending('task_snapshots', error, task_id)
|
| 111 |
+
cursor = task_id
|
| 112 |
+
return count, cursor if pending else None
|
user_store.py
CHANGED
|
@@ -77,6 +77,24 @@ def update_profile(account, updates):
|
|
| 77 |
return _mutate(updater)
|
| 78 |
|
| 79 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
def update_user_privacy(account, privacy):
|
| 81 |
# 保持 PrivacySettings 的完整替换语义,而非隐式合并部分设置。
|
| 82 |
if not isinstance(privacy, dict) or any(key not in PRIVACY_FIELDS or type(value) is not bool
|
|
|
|
| 77 |
return _mutate(updater)
|
| 78 |
|
| 79 |
|
| 80 |
+
def update_settings(account, updates, privacy):
|
| 81 |
+
# 联合设置必须完整提交四个严格布尔值,先验证再进入发布操作。
|
| 82 |
+
if not isinstance(privacy, dict) or set(privacy) != PRIVACY_FIELDS or any(type(value) is not bool for value in privacy.values()):
|
| 83 |
+
raise HTTPException(status_code=400, detail="隐私设置不正确")
|
| 84 |
+
settings = copy.deepcopy(privacy)
|
| 85 |
+
patch = copy.deepcopy({key: value for key, value in updates.items()
|
| 86 |
+
if key in EDITABLE_FIELDS and value is not None})
|
| 87 |
+
if isinstance(patch.get("avatarDataUrl"), str) and patch["avatarDataUrl"].startswith("data:"):
|
| 88 |
+
del patch["avatarDataUrl"]
|
| 89 |
+
|
| 90 |
+
def updater(data):
|
| 91 |
+
record = _user(data, account)
|
| 92 |
+
record.update(patch)
|
| 93 |
+
record["privacy"] = settings
|
| 94 |
+
return project_profile(record, owner=True)
|
| 95 |
+
return _mutate(updater)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
def update_user_privacy(account, privacy):
|
| 99 |
# 保持 PrivacySettings 的完整替换语义,而非隐式合并部分设置。
|
| 100 |
if not isinstance(privacy, dict) or any(key not in PRIVACY_FIELDS or type(value) is not bool
|
zip_validation.py
CHANGED
|
@@ -105,7 +105,8 @@ def validate_zip_archive(path, size, *, max_directory_bytes=MAX_DIRECTORY_BYTES,
|
|
| 105 |
expanded = 0
|
| 106 |
for entry in entries:
|
| 107 |
check()
|
| 108 |
-
|
|
|
|
| 109 |
or not 0 <= entry.file_size <= max_member_bytes or entry.compress_size < 0): _fail()
|
| 110 |
expanded += entry.file_size
|
| 111 |
if expanded > min(max_expanded_bytes, size * 10): _fail()
|
|
|
|
| 105 |
expanded = 0
|
| 106 |
for entry in entries:
|
| 107 |
check()
|
| 108 |
+
# 普通/强加密及压缩补丁均沿用ZipFile.open的拒绝契约。
|
| 109 |
+
if (entry.flag_bits & (1 | 32 | 64) or entry.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED)
|
| 110 |
or not 0 <= entry.file_size <= max_member_bytes or entry.compress_size < 0): _fail()
|
| 111 |
expanded += entry.file_size
|
| 112 |
if expanded > min(max_expanded_bytes, size * 10): _fail()
|
云端_定时版本检测引擎.py
CHANGED
|
@@ -191,16 +191,18 @@ async def _delete_old_zip_cache(item_id: str, old_hash: str, entry=None):
|
|
| 191 |
storage_repo = archive_repository(db.DATASET_REPO_ID)
|
| 192 |
path = _stored_archive_path(item_id, old_hash, entry, repo_id=storage_repo)
|
| 193 |
hf_api = HfApi(token=hf_token)
|
| 194 |
-
await
|
| 195 |
except Exception as error:
|
| 196 |
logger.warning("旧缓存清理失败: error_type=%s", type(error).__name__)
|
| 197 |
|
| 198 |
|
| 199 |
-
async def cleanup_expired_cache(versions_db: dict):
|
| 200 |
"""清理过期归档;仅撤销仍对应本次快照的元数据,不覆盖并发新缓存。"""
|
| 201 |
try:
|
| 202 |
now = datetime.datetime.now(datetime.timezone.utc)
|
| 203 |
for item_id, entry in list(versions_db.items()):
|
|
|
|
|
|
|
| 204 |
if not isinstance(entry, dict) or not entry.get('cached_at'):
|
| 205 |
continue
|
| 206 |
try:
|
|
@@ -222,7 +224,7 @@ async def cleanup_expired_cache(versions_db: dict):
|
|
| 222 |
current[item_id] = {key: value for key, value in expected.items()
|
| 223 |
if key not in ('cached_at', 'zip_size') and not key.startswith('archive_')}
|
| 224 |
return True
|
| 225 |
-
changed = await
|
| 226 |
if changed:
|
| 227 |
versions_db[item_id] = {key: value for key, value in expected.items()
|
| 228 |
if key not in ('cached_at', 'zip_size') and not key.startswith('archive_')}
|
|
@@ -230,11 +232,24 @@ async def cleanup_expired_cache(versions_db: dict):
|
|
| 230 |
logger.warning("缓存过期清理失败: error_type=%s", type(error).__name__)
|
| 231 |
|
| 232 |
|
| 233 |
-
async def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 234 |
"""每日 02:00 定时执行的守护进程"""
|
| 235 |
cache_cursor = 0
|
| 236 |
-
while
|
| 237 |
-
now = datetime.datetime.
|
| 238 |
# 计算距离今天凌晨 02:00 的秒数,如果过了就定在明天 02:00
|
| 239 |
next_run = now.replace(hour=2, minute=0, second=0, microsecond=0)
|
| 240 |
if now >= next_run:
|
|
@@ -242,32 +257,41 @@ async def daily_version_check_task():
|
|
| 242 |
sleep_seconds = (next_run - now).total_seconds()
|
| 243 |
|
| 244 |
print(f"🕒 版本探测引擎已挂载,距离下次 02:00 扫描还有 {sleep_seconds} 秒...")
|
| 245 |
-
await
|
|
|
|
| 246 |
|
| 247 |
print("🚀 [02:00] 开始执行全站 GitHub 资源版本检测任务...")
|
| 248 |
-
items_db = await
|
| 249 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 250 |
|
| 251 |
# 先清理过期缓存(cached_at 超过 30 天)
|
| 252 |
-
await cleanup_expired_cache(versions_db)
|
| 253 |
|
| 254 |
updated_count = 0
|
| 255 |
updated_items = [] # 记录有更新的插件信息
|
| 256 |
|
| 257 |
for item in items_db:
|
|
|
|
|
|
|
| 258 |
link = item.get("link", "")
|
| 259 |
if "github.com" in link:
|
| 260 |
try:
|
| 261 |
refreshed = await refresh_resource_version(db, item['id'])
|
| 262 |
if not refreshed:
|
| 263 |
continue
|
|
|
|
|
|
|
| 264 |
current = refreshed['item']
|
| 265 |
token = refreshed['token']
|
| 266 |
latest_hash = refreshed['commit']
|
| 267 |
if refreshed['changed']:
|
| 268 |
updated_count += 1
|
| 269 |
# 每项刷新后再读最新缓存,不持有全站快照执行保存。
|
| 270 |
-
current_versions = await
|
| 271 |
if cached_archive(current, token, current_versions.get(item['id']), repo_id=db.DATASET_REPO_ID) is None:
|
| 272 |
updated_items.append(dict(id=item['id'], link=current['link'], github_token=token, version_hash=latest_hash))
|
| 273 |
except Exception as error:
|
|
@@ -282,6 +306,8 @@ async def daily_version_check_task():
|
|
| 282 |
batch = ordered[:PRECACHE_BATCH_SIZE]
|
| 283 |
cache_cursor = (start + len(batch)) % len(updated_items)
|
| 284 |
for idx, item in enumerate(batch):
|
|
|
|
|
|
|
| 285 |
success = await precache_github_zip(
|
| 286 |
item["link"],
|
| 287 |
item.get("github_token"),
|
|
@@ -292,24 +318,24 @@ async def daily_version_check_task():
|
|
| 292 |
precache_count += 1
|
| 293 |
# 间隔 10 秒(最后一个不需要等待)
|
| 294 |
if idx < len(batch) - 1:
|
| 295 |
-
await
|
|
|
|
| 296 |
logger.info(
|
| 297 |
f"[预缓存] 本次预缓存完成: {precache_count}/{len(batch)} 成功"
|
| 298 |
)
|
| 299 |
|
| 300 |
|
| 301 |
-
async def reset_daily_views_task():
|
| 302 |
"""
|
| 303 |
每日重置 daily_views 计数器的守护进程
|
| 304 |
-
-
|
| 305 |
- 使用 atomic_update 保证并发安全
|
| 306 |
"""
|
| 307 |
# 需要处理的数据文件
|
| 308 |
data_files = ["items.json", "tasks.json", "posts.json"]
|
| 309 |
|
| 310 |
-
while
|
| 311 |
-
now = datetime.datetime.
|
| 312 |
-
today = now.date().isoformat() # 使用 UTC+8 日期而非系统本地日期
|
| 313 |
|
| 314 |
# 计算距离今天 00:00 (UTC+8) 的秒数,如果过了就定在明天 00:00
|
| 315 |
next_run = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
@@ -318,13 +344,18 @@ async def reset_daily_views_task():
|
|
| 318 |
sleep_seconds = (next_run - now).total_seconds()
|
| 319 |
|
| 320 |
print(f"📊 daily_views 重置引擎已挂载,距离下次 00:00 执行还有 {sleep_seconds} 秒...")
|
| 321 |
-
await
|
|
|
|
| 322 |
|
|
|
|
|
|
|
| 323 |
print(f"🚀 [00:00] 开始执行 daily_views 每日重置任务 (日期: {today})...")
|
| 324 |
|
| 325 |
total_reset_count = 0
|
| 326 |
|
| 327 |
for file_name in data_files:
|
|
|
|
|
|
|
| 328 |
reset_count = 0
|
| 329 |
|
| 330 |
def reset_updater(data):
|
|
@@ -344,7 +375,7 @@ async def reset_daily_views_task():
|
|
| 344 |
reset_count += 1
|
| 345 |
|
| 346 |
try:
|
| 347 |
-
db.atomic_update
|
| 348 |
total_reset_count += reset_count
|
| 349 |
if reset_count > 0:
|
| 350 |
print(f" ✓ {file_name}: 重置了 {reset_count} 条记录的 daily_views")
|
|
|
|
| 191 |
storage_repo = archive_repository(db.DATASET_REPO_ID)
|
| 192 |
path = _stored_archive_path(item_id, old_hash, entry, repo_id=storage_repo)
|
| 193 |
hf_api = HfApi(token=hf_token)
|
| 194 |
+
await run_owned_thread(hf_api.delete_file, path_in_repo=path, repo_id=storage_repo, repo_type="dataset")
|
| 195 |
except Exception as error:
|
| 196 |
logger.warning("旧缓存清理失败: error_type=%s", type(error).__name__)
|
| 197 |
|
| 198 |
|
| 199 |
+
async def cleanup_expired_cache(versions_db: dict, stop=None):
|
| 200 |
"""清理过期归档;仅撤销仍对应本次快照的元数据,不覆盖并发新缓存。"""
|
| 201 |
try:
|
| 202 |
now = datetime.datetime.now(datetime.timezone.utc)
|
| 203 |
for item_id, entry in list(versions_db.items()):
|
| 204 |
+
if stop is not None and stop.is_set():
|
| 205 |
+
return
|
| 206 |
if not isinstance(entry, dict) or not entry.get('cached_at'):
|
| 207 |
continue
|
| 208 |
try:
|
|
|
|
| 224 |
current[item_id] = {key: value for key, value in expected.items()
|
| 225 |
if key not in ('cached_at', 'zip_size') and not key.startswith('archive_')}
|
| 226 |
return True
|
| 227 |
+
changed = await run_owned_thread(db.atomic_update, 'versions.json', update, default_data={})
|
| 228 |
if changed:
|
| 229 |
versions_db[item_id] = {key: value for key, value in expected.items()
|
| 230 |
if key not in ('cached_at', 'zip_size') and not key.startswith('archive_')}
|
|
|
|
| 232 |
logger.warning("缓存过期清理失败: error_type=%s", type(error).__name__)
|
| 233 |
|
| 234 |
|
| 235 |
+
async def _wait_periodic_stop(stop, seconds):
|
| 236 |
+
if stop is None:
|
| 237 |
+
await asyncio.sleep(seconds)
|
| 238 |
+
return False
|
| 239 |
+
if stop.is_set():
|
| 240 |
+
return True
|
| 241 |
+
try:
|
| 242 |
+
await asyncio.wait_for(stop.wait(), timeout=seconds)
|
| 243 |
+
return True
|
| 244 |
+
except asyncio.TimeoutError:
|
| 245 |
+
return stop.is_set()
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
async def daily_version_check_task(stop=None):
|
| 249 |
"""每日 02:00 定时执行的守护进程"""
|
| 250 |
cache_cursor = 0
|
| 251 |
+
while stop is None or not stop.is_set():
|
| 252 |
+
now = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=8)
|
| 253 |
# 计算距离今天凌晨 02:00 的秒数,如果过了就定在明天 02:00
|
| 254 |
next_run = now.replace(hour=2, minute=0, second=0, microsecond=0)
|
| 255 |
if now >= next_run:
|
|
|
|
| 257 |
sleep_seconds = (next_run - now).total_seconds()
|
| 258 |
|
| 259 |
print(f"🕒 版本探测引擎已挂载,距离下次 02:00 扫描还有 {sleep_seconds} 秒...")
|
| 260 |
+
if await _wait_periodic_stop(stop, sleep_seconds):
|
| 261 |
+
return
|
| 262 |
|
| 263 |
print("🚀 [02:00] 开始执行全站 GitHub 资源版本检测任务...")
|
| 264 |
+
items_db = await run_owned_thread(db.load_data, "items.json", default_data=[], strict=True, skip_cache=True)
|
| 265 |
+
if stop is not None and stop.is_set():
|
| 266 |
+
return
|
| 267 |
+
versions_db = await run_owned_thread(db.load_data, "versions.json", default_data={}, strict=True, skip_cache=True)
|
| 268 |
+
if stop is not None and stop.is_set():
|
| 269 |
+
return
|
| 270 |
|
| 271 |
# 先清理过期缓存(cached_at 超过 30 天)
|
| 272 |
+
await cleanup_expired_cache(versions_db, stop=stop)
|
| 273 |
|
| 274 |
updated_count = 0
|
| 275 |
updated_items = [] # 记录有更新的插件信息
|
| 276 |
|
| 277 |
for item in items_db:
|
| 278 |
+
if stop is not None and stop.is_set():
|
| 279 |
+
return
|
| 280 |
link = item.get("link", "")
|
| 281 |
if "github.com" in link:
|
| 282 |
try:
|
| 283 |
refreshed = await refresh_resource_version(db, item['id'])
|
| 284 |
if not refreshed:
|
| 285 |
continue
|
| 286 |
+
if stop is not None and stop.is_set():
|
| 287 |
+
return
|
| 288 |
current = refreshed['item']
|
| 289 |
token = refreshed['token']
|
| 290 |
latest_hash = refreshed['commit']
|
| 291 |
if refreshed['changed']:
|
| 292 |
updated_count += 1
|
| 293 |
# 每项刷新后再读最新缓存,不持有全站快照执行保存。
|
| 294 |
+
current_versions = await run_owned_thread(db.load_data, "versions.json", default_data={}, strict=True, skip_cache=True)
|
| 295 |
if cached_archive(current, token, current_versions.get(item['id']), repo_id=db.DATASET_REPO_ID) is None:
|
| 296 |
updated_items.append(dict(id=item['id'], link=current['link'], github_token=token, version_hash=latest_hash))
|
| 297 |
except Exception as error:
|
|
|
|
| 306 |
batch = ordered[:PRECACHE_BATCH_SIZE]
|
| 307 |
cache_cursor = (start + len(batch)) % len(updated_items)
|
| 308 |
for idx, item in enumerate(batch):
|
| 309 |
+
if stop is not None and stop.is_set():
|
| 310 |
+
return
|
| 311 |
success = await precache_github_zip(
|
| 312 |
item["link"],
|
| 313 |
item.get("github_token"),
|
|
|
|
| 318 |
precache_count += 1
|
| 319 |
# 间隔 10 秒(最后一个不需要等待)
|
| 320 |
if idx < len(batch) - 1:
|
| 321 |
+
if await _wait_periodic_stop(stop, PRECACHE_INTERVAL):
|
| 322 |
+
return
|
| 323 |
logger.info(
|
| 324 |
f"[预缓存] 本次预缓存完成: {precache_count}/{len(batch)} 成功"
|
| 325 |
)
|
| 326 |
|
| 327 |
|
| 328 |
+
async def reset_daily_views_task(stop=None):
|
| 329 |
"""
|
| 330 |
每日重置 daily_views 计数器的守护进程
|
| 331 |
+
- 等待 UTC+8 下一次 00:00,关闭信号可唤醒等待
|
| 332 |
- 使用 atomic_update 保证并发安全
|
| 333 |
"""
|
| 334 |
# 需要处理的数据文件
|
| 335 |
data_files = ["items.json", "tasks.json", "posts.json"]
|
| 336 |
|
| 337 |
+
while stop is None or not stop.is_set():
|
| 338 |
+
now = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=8)
|
|
|
|
| 339 |
|
| 340 |
# 计算距离今天 00:00 (UTC+8) 的秒数,如果过了就定在明天 00:00
|
| 341 |
next_run = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
|
|
| 344 |
sleep_seconds = (next_run - now).total_seconds()
|
| 345 |
|
| 346 |
print(f"📊 daily_views 重置引擎已挂载,距离下次 00:00 执行还有 {sleep_seconds} 秒...")
|
| 347 |
+
if await _wait_periodic_stop(stop, sleep_seconds):
|
| 348 |
+
return
|
| 349 |
|
| 350 |
+
# 等待跨过午夜后重新取日期,不能把入睡前的昨天写回。
|
| 351 |
+
today = (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=8)).date().isoformat()
|
| 352 |
print(f"🚀 [00:00] 开始执行 daily_views 每日重置任务 (日期: {today})...")
|
| 353 |
|
| 354 |
total_reset_count = 0
|
| 355 |
|
| 356 |
for file_name in data_files:
|
| 357 |
+
if stop is not None and stop.is_set():
|
| 358 |
+
return
|
| 359 |
reset_count = 0
|
| 360 |
|
| 361 |
def reset_updater(data):
|
|
|
|
| 375 |
reset_count += 1
|
| 376 |
|
| 377 |
try:
|
| 378 |
+
await run_owned_thread(db.atomic_update, file_name, reset_updater, default_data=[])
|
| 379 |
total_reset_count += reset_count
|
| 380 |
if reset_count > 0:
|
| 381 |
print(f" ✓ {file_name}: 重置了 {reset_count} 条记录的 daily_views")
|
数据库连接.py
CHANGED
|
@@ -121,7 +121,7 @@ CACHE_TTL = {
|
|
| 121 |
}
|
| 122 |
|
| 123 |
# 内存缓存存储
|
| 124 |
-
_memory_cache = {} # {file_name: {"data": ..., "time": ..., "
|
| 125 |
_cache_lock = threading.Lock()
|
| 126 |
|
| 127 |
|
|
@@ -130,6 +130,15 @@ def _get_cache_ttl(file_name: str) -> int:
|
|
| 130 |
return CACHE_TTL.get(file_name, CACHE_TTL["_default"])
|
| 131 |
|
| 132 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
def _get_from_cache(file_name: str, local_path: str) -> Optional[Union[Dict, List]]:
|
| 134 |
"""
|
| 135 |
从内存缓存获取数据
|
|
@@ -149,34 +158,50 @@ def _get_from_cache(file_name: str, local_path: str) -> Optional[Union[Dict, Lis
|
|
| 149 |
del _memory_cache[file_name]
|
| 150 |
return None
|
| 151 |
|
| 152 |
-
#
|
| 153 |
try:
|
| 154 |
-
|
| 155 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
del _memory_cache[file_name]
|
| 157 |
return None
|
| 158 |
except OSError:
|
|
|
|
| 159 |
return None
|
| 160 |
-
|
| 161 |
logger.debug(f"✨ 内存缓存命中: {file_name}")
|
| 162 |
-
|
| 163 |
-
import copy
|
| 164 |
-
return copy.deepcopy(cache_entry["data"])
|
| 165 |
|
| 166 |
|
| 167 |
-
def _set_to_cache(file_name: str, data: Union[Dict, List], local_path: str):
|
| 168 |
-
"""
|
| 169 |
with _cache_lock:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
try:
|
| 171 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
except OSError:
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
import copy
|
| 176 |
_memory_cache[file_name] = {
|
| 177 |
-
"data":
|
| 178 |
"time": time.time(),
|
| 179 |
-
"
|
| 180 |
}
|
| 181 |
logger.debug(f"💾 内存缓存更新: {file_name}")
|
| 182 |
|
|
@@ -289,6 +314,40 @@ else:
|
|
| 289 |
# - 文件不存在时从 HF 下载
|
| 290 |
# - JSON 解析失败时返回默认值
|
| 291 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
def load_data(file_name: str, default_data: Optional[Union[Dict, List]] = None, skip_cache: bool = False, strict: bool = False) -> Union[Dict, List]:
|
| 293 |
"""
|
| 294 |
从 JSON 文件加载数据
|
|
@@ -330,9 +389,8 @@ def load_data(file_name: str, default_data: Optional[Union[Dict, List]] = None,
|
|
| 330 |
# 读取下载的文件
|
| 331 |
with open(downloaded_path, "r", encoding="utf-8") as f:
|
| 332 |
data = json.load(f)
|
| 333 |
-
#
|
| 334 |
-
|
| 335 |
-
json.dump(data, f, ensure_ascii=False, indent=2)
|
| 336 |
logger.info(f"从 HF Dataset 下载 {file_name} 成功")
|
| 337 |
# 🚀 P1优化:存入内存缓存
|
| 338 |
_set_to_cache(file_name, data, local_path)
|
|
@@ -345,7 +403,7 @@ def load_data(file_name: str, default_data: Optional[Union[Dict, List]] = None,
|
|
| 345 |
logger.info(f"HF 中未找到 {file_name}(尚未创建),使用默认值")
|
| 346 |
else:
|
| 347 |
logger.warning(f"从 HF 下载 {file_name} 失败: {_safe_error_details(e)}")
|
| 348 |
-
# 缓存
|
| 349 |
_set_to_cache(file_name, default_data, local_path)
|
| 350 |
return default_data
|
| 351 |
return default_data
|
|
@@ -356,12 +414,20 @@ def load_data(file_name: str, default_data: Optional[Union[Dict, List]] = None,
|
|
| 356 |
# 获取共享锁(读锁)
|
| 357 |
_lock_file(f, exclusive=False)
|
| 358 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
data = json.load(f)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
finally:
|
| 361 |
_unlock_file(f)
|
| 362 |
-
|
| 363 |
-
#
|
| 364 |
-
_set_to_cache(file_name, data, local_path)
|
| 365 |
return data
|
| 366 |
except json.JSONDecodeError as e:
|
| 367 |
logger.error(f"JSON 解析错误 {file_name}: {_safe_error_details(e)}")
|
|
@@ -385,14 +451,13 @@ def _recover_from_backup(file_name: str, default_data: Union[Dict, List]) -> Uni
|
|
| 385 |
try:
|
| 386 |
with open(backup_path, "r", encoding="utf-8") as f:
|
| 387 |
data = json.load(f)
|
| 388 |
-
|
| 389 |
-
# 恢复主文件
|
| 390 |
local_path = os.path.join(LOCAL_DB_DIR, file_name)
|
| 391 |
-
|
| 392 |
-
|
| 393 |
return data
|
| 394 |
except Exception as e:
|
| 395 |
-
|
| 396 |
|
| 397 |
return default_data
|
| 398 |
|
|
@@ -404,7 +469,7 @@ def _recover_from_backup(file_name: str, default_data: Union[Dict, List]) -> Uni
|
|
| 404 |
# 1. 先写临时文件,验证成功后原子重命名
|
| 405 |
# 2. 写入前备份上一版本
|
| 406 |
# 3. 写入后验证数据完整性
|
| 407 |
-
# 4.
|
| 408 |
# 5. 异步同步到 HuggingFace
|
| 409 |
|
| 410 |
def save_data(file_name: str, data: Union[Dict, List], force_sync: bool = False) -> bool:
|
|
@@ -473,7 +538,7 @@ def save_data(file_name: str, data: Union[Dict, List], force_sync: bool = False)
|
|
| 473 |
_mark_dirty(file_name)
|
| 474 |
os.replace(temp_path, local_path)
|
| 475 |
|
| 476 |
-
# ==========
|
| 477 |
_set_to_cache(file_name, data, local_path)
|
| 478 |
|
| 479 |
except Exception as e:
|
|
@@ -902,7 +967,7 @@ def atomic_update(file_name: str, updater, default_data=None, force_sync: bool =
|
|
| 902 |
_mark_dirty(file_name)
|
| 903 |
os.replace(temp_path, local_path)
|
| 904 |
|
| 905 |
-
# ==========
|
| 906 |
_set_to_cache(file_name, data, local_path)
|
| 907 |
|
| 908 |
except Exception as e:
|
|
|
|
| 121 |
}
|
| 122 |
|
| 123 |
# 内存缓存存储
|
| 124 |
+
_memory_cache = {} # {file_name: {"data": ..., "time": ..., "identity": ...}}
|
| 125 |
_cache_lock = threading.Lock()
|
| 126 |
|
| 127 |
|
|
|
|
| 130 |
return CACHE_TTL.get(file_name, CACHE_TTL["_default"])
|
| 131 |
|
| 132 |
|
| 133 |
+
def _file_cache_identity(stat):
|
| 134 |
+
"""本机文件身份与变化提示;不是内容哈希或跨文件事务版本。"""
|
| 135 |
+
if not stat.st_ino:
|
| 136 |
+
return None # 文件系统未提供可靠文件ID,不复用缓存。
|
| 137 |
+
# Windows的stat/fstat ctime语义存在差异,不跨两种接口比较该字段。
|
| 138 |
+
changed_ns = stat.st_ctime_ns if os.name != "nt" else None
|
| 139 |
+
return (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, changed_ns)
|
| 140 |
+
|
| 141 |
+
|
| 142 |
def _get_from_cache(file_name: str, local_path: str) -> Optional[Union[Dict, List]]:
|
| 143 |
"""
|
| 144 |
从内存缓存获取数据
|
|
|
|
| 158 |
del _memory_cache[file_name]
|
| 159 |
return None
|
| 160 |
|
| 161 |
+
# 相同mtime的原子替换仍需识别;复制前后都确认路径仍指向同一来源。
|
| 162 |
try:
|
| 163 |
+
identity = _file_cache_identity(os.stat(local_path))
|
| 164 |
+
if identity is None or identity != cache_entry.get("identity"):
|
| 165 |
+
del _memory_cache[file_name]
|
| 166 |
+
return None
|
| 167 |
+
import copy
|
| 168 |
+
result = copy.deepcopy(cache_entry["data"])
|
| 169 |
+
if _file_cache_identity(os.stat(local_path)) != identity:
|
| 170 |
del _memory_cache[file_name]
|
| 171 |
return None
|
| 172 |
except OSError:
|
| 173 |
+
_memory_cache.pop(file_name, None)
|
| 174 |
return None
|
|
|
|
| 175 |
logger.debug(f"✨ 内存缓存命中: {file_name}")
|
| 176 |
+
return result
|
|
|
|
|
|
|
| 177 |
|
| 178 |
|
| 179 |
+
def _set_to_cache(file_name: str, data: Union[Dict, List], local_path: str, *, source_stats=None):
|
| 180 |
+
"""仅缓存已由真实读取句柄确认的数据;旧三参数写后调用只使缓存失效。"""
|
| 181 |
with _cache_lock:
|
| 182 |
+
if source_stats is None:
|
| 183 |
+
_memory_cache.pop(file_name, None)
|
| 184 |
+
return
|
| 185 |
+
before, after = source_stats
|
| 186 |
+
if before is None or after is None:
|
| 187 |
+
return
|
| 188 |
+
identity = _file_cache_identity(before)
|
| 189 |
+
if (identity is None or identity != _file_cache_identity(after)
|
| 190 |
+
or before.st_ctime_ns != after.st_ctime_ns):
|
| 191 |
+
return
|
| 192 |
try:
|
| 193 |
+
if _file_cache_identity(os.stat(local_path)) != identity:
|
| 194 |
+
return
|
| 195 |
+
import copy
|
| 196 |
+
snapshot = copy.deepcopy(data)
|
| 197 |
+
if _file_cache_identity(os.stat(local_path)) != identity:
|
| 198 |
+
return
|
| 199 |
except OSError:
|
| 200 |
+
return
|
|
|
|
|
|
|
| 201 |
_memory_cache[file_name] = {
|
| 202 |
+
"data": snapshot,
|
| 203 |
"time": time.time(),
|
| 204 |
+
"identity": identity
|
| 205 |
}
|
| 206 |
logger.debug(f"💾 内存缓存更新: {file_name}")
|
| 207 |
|
|
|
|
| 314 |
# - 文件不存在时从 HF 下载
|
| 315 |
# - JSON 解析失败时返回默认值
|
| 316 |
|
| 317 |
+
def _publish_read_snapshot(local_path: str, data):
|
| 318 |
+
"""首次下载/非严格备份恢复的原子发布;调用方持有原稳定协调锁。"""
|
| 319 |
+
descriptor = None
|
| 320 |
+
staging_path = None
|
| 321 |
+
try:
|
| 322 |
+
descriptor, staging_path = tempfile.mkstemp(
|
| 323 |
+
prefix=f".{os.path.basename(local_path)}.read-", suffix=".tmp",
|
| 324 |
+
dir=os.path.dirname(local_path))
|
| 325 |
+
stream = os.fdopen(descriptor, "w", encoding="utf-8")
|
| 326 |
+
descriptor = None # fdopen成功后由stream负责关闭。
|
| 327 |
+
with stream:
|
| 328 |
+
json.dump(data, stream, ensure_ascii=False, indent=2)
|
| 329 |
+
stream.flush()
|
| 330 |
+
os.fsync(stream.fileno())
|
| 331 |
+
with open(staging_path, "r", encoding="utf-8") as source:
|
| 332 |
+
verified = json.load(source)
|
| 333 |
+
if type(verified) is not type(data) or (hasattr(data, "__len__") and len(verified) != len(data)):
|
| 334 |
+
raise ValueError("Read snapshot verification failed")
|
| 335 |
+
os.replace(staging_path, local_path)
|
| 336 |
+
finally:
|
| 337 |
+
if descriptor is not None:
|
| 338 |
+
try:
|
| 339 |
+
os.close(descriptor)
|
| 340 |
+
except OSError:
|
| 341 |
+
logger.warning("JSON_READ_DESCRIPTOR_CLEANUP_FAILED")
|
| 342 |
+
if staging_path is not None:
|
| 343 |
+
try:
|
| 344 |
+
os.unlink(staging_path)
|
| 345 |
+
except FileNotFoundError:
|
| 346 |
+
pass
|
| 347 |
+
except OSError:
|
| 348 |
+
logger.warning("JSON_READ_STAGE_CLEANUP_FAILED")
|
| 349 |
+
|
| 350 |
+
|
| 351 |
def load_data(file_name: str, default_data: Optional[Union[Dict, List]] = None, skip_cache: bool = False, strict: bool = False) -> Union[Dict, List]:
|
| 352 |
"""
|
| 353 |
从 JSON 文件加载数据
|
|
|
|
| 389 |
# 读取下载的文件
|
| 390 |
with open(downloaded_path, "r", encoding="utf-8") as f:
|
| 391 |
data = json.load(f)
|
| 392 |
+
# 验证同目录暂存后原子发布,失败不留下半份主文件。
|
| 393 |
+
_publish_read_snapshot(local_path, data)
|
|
|
|
| 394 |
logger.info(f"从 HF Dataset 下载 {file_name} 成功")
|
| 395 |
# 🚀 P1优化:存入内存缓存
|
| 396 |
_set_to_cache(file_name, data, local_path)
|
|
|
|
| 403 |
logger.info(f"HF 中未找到 {file_name}(尚未创建),使用默认值")
|
| 404 |
else:
|
| 405 |
logger.warning(f"从 HF 下载 {file_name} 失败: {_safe_error_details(e)}")
|
| 406 |
+
# 无确认来源的默认值不入内存缓存,保持后续读取可恢复
|
| 407 |
_set_to_cache(file_name, default_data, local_path)
|
| 408 |
return default_data
|
| 409 |
return default_data
|
|
|
|
| 414 |
# 获取共享锁(读锁)
|
| 415 |
_lock_file(f, exclusive=False)
|
| 416 |
try:
|
| 417 |
+
try:
|
| 418 |
+
source_before = os.fstat(f.fileno())
|
| 419 |
+
except OSError:
|
| 420 |
+
source_before = None
|
| 421 |
data = json.load(f)
|
| 422 |
+
try:
|
| 423 |
+
source_after = os.fstat(f.fileno())
|
| 424 |
+
except OSError:
|
| 425 |
+
source_after = None
|
| 426 |
finally:
|
| 427 |
_unlock_file(f)
|
| 428 |
+
|
| 429 |
+
# 句柄身份绑定实际解析的数据;迟到旧读取不得冒充新路径内容。
|
| 430 |
+
_set_to_cache(file_name, data, local_path, source_stats=(source_before, source_after))
|
| 431 |
return data
|
| 432 |
except json.JSONDecodeError as e:
|
| 433 |
logger.error(f"JSON 解析错误 {file_name}: {_safe_error_details(e)}")
|
|
|
|
| 451 |
try:
|
| 452 |
with open(backup_path, "r", encoding="utf-8") as f:
|
| 453 |
data = json.load(f)
|
| 454 |
+
# 先完整发布,再记录成功;保留原备份,不写回默认值。
|
|
|
|
| 455 |
local_path = os.path.join(LOCAL_DB_DIR, file_name)
|
| 456 |
+
_publish_read_snapshot(local_path, data)
|
| 457 |
+
logger.info("从备份恢复 %s 成功", file_name)
|
| 458 |
return data
|
| 459 |
except Exception as e:
|
| 460 |
+
logger.warning("备份恢复失败 %s: %s", file_name, _safe_error_details(e))
|
| 461 |
|
| 462 |
return default_data
|
| 463 |
|
|
|
|
| 469 |
# 1. 先写临时文件,验证成功后原子重命名
|
| 470 |
# 2. 写入前备份上一版本
|
| 471 |
# 3. 写入后验证数据完整性
|
| 472 |
+
# 4. 保存后使旧内存缓存失效,确认读取后重建
|
| 473 |
# 5. 异步同步到 HuggingFace
|
| 474 |
|
| 475 |
def save_data(file_name: str, data: Union[Dict, List], force_sync: bool = False) -> bool:
|
|
|
|
| 538 |
_mark_dirty(file_name)
|
| 539 |
os.replace(temp_path, local_path)
|
| 540 |
|
| 541 |
+
# ========== 写后使旧内存缓存失效;下一次句柄确认读取才缓存 ==========
|
| 542 |
_set_to_cache(file_name, data, local_path)
|
| 543 |
|
| 544 |
except Exception as e:
|
|
|
|
| 967 |
_mark_dirty(file_name)
|
| 968 |
os.replace(temp_path, local_path)
|
| 969 |
|
| 970 |
+
# ========== 写后使旧内存缓存失效;下一次句柄确认读取才缓存 ==========
|
| 971 |
_set_to_cache(file_name, data, local_path)
|
| 972 |
|
| 973 |
except Exception as e:
|