Spaces:
Running
Running
Upload 14 files
Browse files- app.py +12 -2
- github_archive.py +2 -2
- github_source.py +10 -1
- github_versions.py +14 -2
- github_zip_service.py +23 -6
- resource_probe_service.py +16 -4
- router_proxy.py +24 -1
- skill_archive_source.py +246 -0
- skill_preflight.py +146 -0
- 管理界面/app.js +12 -1
app.py
CHANGED
|
@@ -430,12 +430,22 @@ class ValidateResourceRequest(BaseModel):
|
|
| 430 |
|
| 431 |
from resource_probe_service import probe_resource
|
| 432 |
from github_probe_transport import open_github_probe
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 433 |
|
| 434 |
@app.post("/api/validate_resource")
|
| 435 |
-
|
|
|
|
| 436 |
return probe_resource(req_data.item_id, store=db, download=hf_hub_download,
|
| 437 |
open_url=open_github_probe, token_selector=select_github_token,
|
| 438 |
-
error_response=create_error_response)
|
| 439 |
|
| 440 |
# ==========================================
|
| 441 |
# 🔒 管理员:系统配置 API
|
|
|
|
| 430 |
|
| 431 |
from resource_probe_service import probe_resource
|
| 432 |
from github_probe_transport import open_github_probe
|
| 433 |
+
from skill_preflight import verify_skill_manifest
|
| 434 |
+
from router_proxy import _send_with_validated_redirects
|
| 435 |
+
from safe_http_transport import ValidatedAsyncHTTPTransport
|
| 436 |
+
import httpx
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def _probe_skill_manifest(item, token):
|
| 440 |
+
return verify_skill_manifest(item, token, send=_send_with_validated_redirects,
|
| 441 |
+
client_factory=httpx.AsyncClient, transport_factory=ValidatedAsyncHTTPTransport)
|
| 442 |
|
| 443 |
@app.post("/api/validate_resource")
|
| 444 |
+
@shared_limit(30)
|
| 445 |
+
def validate_resource(req_data: ValidateResourceRequest, request: Request, sql_db: Session = Depends(get_db), current_user: str = Depends(require_auth)):
|
| 446 |
return probe_resource(req_data.item_id, store=db, download=hf_hub_download,
|
| 447 |
open_url=open_github_probe, token_selector=select_github_token,
|
| 448 |
+
error_response=create_error_response, skill_probe=_probe_skill_manifest)
|
| 449 |
|
| 450 |
# ==========================================
|
| 451 |
# 🔒 管理员:系统配置 API
|
github_archive.py
CHANGED
|
@@ -3,7 +3,7 @@ import datetime
|
|
| 3 |
import hashlib
|
| 4 |
import json
|
| 5 |
import re
|
| 6 |
-
from github_source import
|
| 7 |
|
| 8 |
ARCHIVE_SCHEMA = 3
|
| 9 |
ARCHIVE_MAX_BYTES = 500 * 1024 * 1024
|
|
@@ -34,7 +34,7 @@ def source_signature(item, token):
|
|
| 34 |
author = item.get('author')
|
| 35 |
if not isinstance(author, str) or not author.strip():
|
| 36 |
raise ValueError('invalid archive author')
|
| 37 |
-
source = [
|
| 38 |
# 元数据只保存摘要,不保存或输出凭据。
|
| 39 |
return hashlib.sha256(json.dumps(source, ensure_ascii=True, separators=(',', ':')).encode()).hexdigest()
|
| 40 |
|
|
|
|
| 3 |
import hashlib
|
| 4 |
import json
|
| 5 |
import re
|
| 6 |
+
from github_source import canonical_resource_source
|
| 7 |
|
| 8 |
ARCHIVE_SCHEMA = 3
|
| 9 |
ARCHIVE_MAX_BYTES = 500 * 1024 * 1024
|
|
|
|
| 34 |
author = item.get('author')
|
| 35 |
if not isinstance(author, str) or not author.strip():
|
| 36 |
raise ValueError('invalid archive author')
|
| 37 |
+
source = [canonical_resource_source(item), author, token or '']
|
| 38 |
# 元数据只保存摘要,不保存或输出凭据。
|
| 39 |
return hashlib.sha256(json.dumps(source, ensure_ascii=True, separators=(',', ':')).encode()).hexdigest()
|
| 40 |
|
github_source.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
| 2 |
import os
|
| 3 |
import re
|
| 4 |
from urllib.parse import urlsplit
|
|
|
|
| 5 |
|
| 6 |
|
| 7 |
def repository_parts(url):
|
|
@@ -28,6 +29,14 @@ def canonical_repository(url):
|
|
| 28 |
return f'https://github.com/{owner}/{repo}'
|
| 29 |
|
| 30 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
def _token(value):
|
| 32 |
if value is None or value == '':
|
| 33 |
return None
|
|
@@ -37,7 +46,7 @@ def _token(value):
|
|
| 37 |
|
| 38 |
|
| 39 |
def select_github_token(item, *, admin_check=None):
|
| 40 |
-
|
| 41 |
author = item.get('author')
|
| 42 |
if not isinstance(author, str) or not author.strip():
|
| 43 |
raise ValueError('invalid resource author')
|
|
|
|
| 2 |
import os
|
| 3 |
import re
|
| 4 |
from urllib.parse import urlsplit
|
| 5 |
+
from skill_archive_source import item_source, source_key
|
| 6 |
|
| 7 |
|
| 8 |
def repository_parts(url):
|
|
|
|
| 29 |
return f'https://github.com/{owner}/{repo}'
|
| 30 |
|
| 31 |
|
| 32 |
+
def github_item_source(item):
|
| 33 |
+
return item_source(item, repository_parts)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def canonical_resource_source(item):
|
| 37 |
+
return source_key(github_item_source(item))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
def _token(value):
|
| 41 |
if value is None or value == '':
|
| 42 |
return None
|
|
|
|
| 46 |
|
| 47 |
|
| 48 |
def select_github_token(item, *, admin_check=None):
|
| 49 |
+
github_item_source(item)
|
| 50 |
author = item.get('author')
|
| 51 |
if not isinstance(author, str) or not author.strip():
|
| 52 |
raise ValueError('invalid resource author')
|
github_versions.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
import asyncio
|
| 3 |
import uuid
|
| 4 |
from github_archive import commit_sha, source_signature
|
| 5 |
-
from github_source import select_github_token
|
| 6 |
from proxy_access import read_proxy_item
|
| 7 |
|
| 8 |
|
|
@@ -83,7 +83,19 @@ def invalidate_source(store, item_id):
|
|
| 83 |
async def refresh_resource_version(store, item_id):
|
| 84 |
from 云端_定时版本检测引擎 import fetch_github_commit
|
| 85 |
operation = await asyncio.to_thread(begin_probe, store, item_id)
|
| 86 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
if not sha:
|
| 88 |
return None
|
| 89 |
return await asyncio.to_thread(finish_probe, store, operation, sha)
|
|
|
|
| 2 |
import asyncio
|
| 3 |
import uuid
|
| 4 |
from github_archive import commit_sha, source_signature
|
| 5 |
+
from github_source import select_github_token, github_item_source
|
| 6 |
from proxy_access import read_proxy_item
|
| 7 |
|
| 8 |
|
|
|
|
| 83 |
async def refresh_resource_version(store, item_id):
|
| 84 |
from 云端_定时版本检测引擎 import fetch_github_commit
|
| 85 |
operation = await asyncio.to_thread(begin_probe, store, item_id)
|
| 86 |
+
source = github_item_source(operation['item'])
|
| 87 |
+
if source['tree']:
|
| 88 |
+
import httpx
|
| 89 |
+
from skill_archive_source import resolve_tree
|
| 90 |
+
from safe_http_transport import ValidatedAsyncHTTPTransport
|
| 91 |
+
from router_proxy import _send_with_validated_redirects
|
| 92 |
+
try:
|
| 93 |
+
sha, _ = await resolve_tree(source, operation['token'], send=_send_with_validated_redirects,
|
| 94 |
+
client_factory=httpx.AsyncClient, transport_factory=ValidatedAsyncHTTPTransport)
|
| 95 |
+
except Exception:
|
| 96 |
+
return None
|
| 97 |
+
else:
|
| 98 |
+
sha = await fetch_github_commit(operation['item']['link'], operation['token'])
|
| 99 |
if not sha:
|
| 100 |
return None
|
| 101 |
return await asyncio.to_thread(finish_probe, store, operation, sha)
|
github_zip_service.py
CHANGED
|
@@ -2,7 +2,8 @@
|
|
| 2 |
import asyncio
|
| 3 |
import os
|
| 4 |
from github_archive import cached_archive
|
| 5 |
-
from github_source import
|
|
|
|
| 6 |
from proxy_resources import ProxyResources
|
| 7 |
from workflow_stream_response import OwnedProxyResponse
|
| 8 |
from zip_stream import zip_response_length, stream_zip_bytes
|
|
@@ -10,12 +11,14 @@ from zip_stream import zip_response_length, stream_zip_bytes
|
|
| 10 |
|
| 11 |
async def prepare_github_zip(item, item_id, *, store, token_selector, resolve_commit,
|
| 12 |
send, client_factory, transport_factory, error_response,
|
| 13 |
-
cache_max_bytes, direct_max_bytes):
|
| 14 |
# 按服务端资源解析仓库,普通作者只使用本人绑定凭据或匿名GitHub。
|
| 15 |
try:
|
| 16 |
-
|
| 17 |
-
|
|
|
|
| 18 |
active_token = token_selector(item)
|
|
|
|
| 19 |
except ValueError:
|
| 20 |
return error_response(content={"error": "资源仓库或凭据暂时无法核实"}, status_code=503)
|
| 21 |
|
|
@@ -25,7 +28,7 @@ async def prepare_github_zip(item, item_id, *, store, token_selector, resolve_co
|
|
| 25 |
try:
|
| 26 |
versions_db = store.load_data("versions.json", default_data={})
|
| 27 |
item_version = versions_db.get(item_id)
|
| 28 |
-
archive = cached_archive(item, active_token, item_version, repo_id=store.DATASET_REPO_ID)
|
| 29 |
hf_cache_token = os.environ.get("HF_TOKEN")
|
| 30 |
if archive and hf_cache_token:
|
| 31 |
hf_headers = {"Authorization": f"Bearer {hf_cache_token}"}
|
|
@@ -48,6 +51,9 @@ async def prepare_github_zip(item, item_id, *, store, token_selector, resolve_co
|
|
| 48 |
cache_length = zip_response_length(stream_resp.headers, max_bytes=cache_max_bytes,
|
| 49 |
expected_size=expected_size)
|
| 50 |
cache_headers = {}
|
|
|
|
|
|
|
|
|
|
| 51 |
content_length = stream_resp.headers.get('content-length')
|
| 52 |
if content_length:
|
| 53 |
cache_headers["Content-Length"] = content_length
|
|
@@ -82,7 +88,15 @@ async def prepare_github_zip(item, item_id, *, store, token_selector, resolve_co
|
|
| 82 |
await cache_resources.close()
|
| 83 |
|
| 84 |
# 先取得完整commit,再按它下载;不在下载后另取“最新版本”冒充实际内容。
|
| 85 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
if not live_commit:
|
| 87 |
return error_response(content={"error": "资源版本暂时无法核实,请稍后重试"}, status_code=502)
|
| 88 |
github_zip_api = f"https://api.github.com/repos/{owner}/{repo}/zipball/{live_commit}"
|
|
@@ -113,6 +127,9 @@ async def prepare_github_zip(item, item_id, *, store, token_selector, resolve_co
|
|
| 113 |
return error_response(content={"error": f"GITHUB_DOWNLOAD_FAILED: {err_msg}"}, status_code=502)
|
| 114 |
declared_length = zip_response_length(response.headers, max_bytes=direct_max_bytes)
|
| 115 |
resp_headers = {}
|
|
|
|
|
|
|
|
|
|
| 116 |
content_length = response.headers.get('content-length', '')
|
| 117 |
if content_length:
|
| 118 |
resp_headers["Content-Length"] = content_length
|
|
|
|
| 2 |
import asyncio
|
| 3 |
import os
|
| 4 |
from github_archive import cached_archive
|
| 5 |
+
from github_source import github_item_source
|
| 6 |
+
from skill_archive_source import resolve_tree, source_header, installation_revision, source_key
|
| 7 |
from proxy_resources import ProxyResources
|
| 8 |
from workflow_stream_response import OwnedProxyResponse
|
| 9 |
from zip_stream import zip_response_length, stream_zip_bytes
|
|
|
|
| 11 |
|
| 12 |
async def prepare_github_zip(item, item_id, *, store, token_selector, resolve_commit,
|
| 13 |
send, client_factory, transport_factory, error_response,
|
| 14 |
+
cache_max_bytes, direct_max_bytes, signing_key=None):
|
| 15 |
# 按服务端资源解析仓库,普通作者只使用本人绑定凭据或匿名GitHub。
|
| 16 |
try:
|
| 17 |
+
source = github_item_source(item)
|
| 18 |
+
repo_url = source["repository"] if source["tree"] else item.get("link", "")
|
| 19 |
+
owner, repo = source["owner"], source["repo"]
|
| 20 |
active_token = token_selector(item)
|
| 21 |
+
skill_revision = installation_revision(item, source_key(source), active_token, signing_key=signing_key) if source['skill'] else None
|
| 22 |
except ValueError:
|
| 23 |
return error_response(content={"error": "资源仓库或凭据暂时无法核实"}, status_code=503)
|
| 24 |
|
|
|
|
| 28 |
try:
|
| 29 |
versions_db = store.load_data("versions.json", default_data={})
|
| 30 |
item_version = versions_db.get(item_id)
|
| 31 |
+
archive = None if source["tree"] else cached_archive(item, active_token, item_version, repo_id=store.DATASET_REPO_ID)
|
| 32 |
hf_cache_token = os.environ.get("HF_TOKEN")
|
| 33 |
if archive and hf_cache_token:
|
| 34 |
hf_headers = {"Authorization": f"Bearer {hf_cache_token}"}
|
|
|
|
| 51 |
cache_length = zip_response_length(stream_resp.headers, max_bytes=cache_max_bytes,
|
| 52 |
expected_size=expected_size)
|
| 53 |
cache_headers = {}
|
| 54 |
+
if source["skill"]:
|
| 55 |
+
cache_headers["X-Ranking-Skill-Source"] = source_header(source, ())
|
| 56 |
+
cache_headers['X-Ranking-Skill-Revision'] = skill_revision
|
| 57 |
content_length = stream_resp.headers.get('content-length')
|
| 58 |
if content_length:
|
| 59 |
cache_headers["Content-Length"] = content_length
|
|
|
|
| 88 |
await cache_resources.close()
|
| 89 |
|
| 90 |
# 先取得完整commit,再按它下载;不在下载后另取“最新版本”冒充实际内容。
|
| 91 |
+
subdirectory = ()
|
| 92 |
+
if source["tree"]:
|
| 93 |
+
try:
|
| 94 |
+
live_commit, subdirectory = await resolve_tree(source, active_token, send=send,
|
| 95 |
+
client_factory=client_factory, transport_factory=transport_factory)
|
| 96 |
+
except Exception:
|
| 97 |
+
return error_response(content={"error": "资源目录或版本暂时无法核实,请稍后重试"}, status_code=502)
|
| 98 |
+
else:
|
| 99 |
+
live_commit = await resolve_commit(repo_url, active_token)
|
| 100 |
if not live_commit:
|
| 101 |
return error_response(content={"error": "资源版本暂时无法核实,请稍后重试"}, status_code=502)
|
| 102 |
github_zip_api = f"https://api.github.com/repos/{owner}/{repo}/zipball/{live_commit}"
|
|
|
|
| 127 |
return error_response(content={"error": f"GITHUB_DOWNLOAD_FAILED: {err_msg}"}, status_code=502)
|
| 128 |
declared_length = zip_response_length(response.headers, max_bytes=direct_max_bytes)
|
| 129 |
resp_headers = {}
|
| 130 |
+
if source["skill"]:
|
| 131 |
+
resp_headers["X-Ranking-Skill-Source"] = source_header(source, subdirectory)
|
| 132 |
+
resp_headers['X-Ranking-Skill-Revision'] = skill_revision
|
| 133 |
content_length = response.headers.get('content-length', '')
|
| 134 |
if content_length:
|
| 135 |
resp_headers["Content-Length"] = content_length
|
resource_probe_service.py
CHANGED
|
@@ -7,14 +7,15 @@ import urllib.parse
|
|
| 7 |
import urllib.request
|
| 8 |
import uuid
|
| 9 |
from fastapi.responses import JSONResponse
|
| 10 |
-
from github_source import
|
| 11 |
from proxy_access import read_proxy_item, ProxyAccessError
|
| 12 |
from workflow_source import authorized_workflow_path, WorkflowSourceError
|
|
|
|
| 13 |
|
| 14 |
logger = logging.getLogger("ComfyUI-Ranking")
|
| 15 |
|
| 16 |
|
| 17 |
-
def probe_resource(item_id, *, store, download, open_url, token_selector, error_response):
|
| 18 |
# 购买前只探测资源可用性;真实下载仍须通过独立所有权授权。
|
| 19 |
try:
|
| 20 |
item = read_proxy_item(store, item_id)
|
|
@@ -36,9 +37,14 @@ def probe_resource(item_id, *, store, download, open_url, token_selector, error_
|
|
| 36 |
if parsed.hostname not in ("github.com", "huggingface.co"):
|
| 37 |
return JSONResponse(content={"error": "无效的下载链接"}, status_code=400)
|
| 38 |
try:
|
| 39 |
-
if
|
| 40 |
-
|
|
|
|
| 41 |
active_token = token_selector(item)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
headers = {"User-Agent": "ComfyUI-Ranking-SaaS"}
|
| 43 |
if active_token:
|
| 44 |
headers["Authorization"] = f"Bearer {active_token}"
|
|
@@ -74,6 +80,12 @@ def probe_resource(item_id, *, store, download, open_url, token_selector, error_
|
|
| 74 |
|
| 75 |
return {"status": "success", "message": "资源有效"}
|
| 76 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
except WorkflowSourceError as error:
|
| 78 |
return JSONResponse(content={"error": error.message}, status_code=error.status_code)
|
| 79 |
except urllib.error.HTTPError as e:
|
|
|
|
| 7 |
import urllib.request
|
| 8 |
import uuid
|
| 9 |
from fastapi.responses import JSONResponse
|
| 10 |
+
from github_source import github_item_source
|
| 11 |
from proxy_access import read_proxy_item, ProxyAccessError
|
| 12 |
from workflow_source import authorized_workflow_path, WorkflowSourceError
|
| 13 |
+
from skill_archive_source import SkillSourceError
|
| 14 |
|
| 15 |
logger = logging.getLogger("ComfyUI-Ranking")
|
| 16 |
|
| 17 |
|
| 18 |
+
def probe_resource(item_id, *, store, download, open_url, token_selector, error_response, skill_probe):
|
| 19 |
# 购买前只探测资源可用性;真实下载仍须通过独立所有权授权。
|
| 20 |
try:
|
| 21 |
item = read_proxy_item(store, item_id)
|
|
|
|
| 37 |
if parsed.hostname not in ("github.com", "huggingface.co"):
|
| 38 |
return JSONResponse(content={"error": "无效的下载链接"}, status_code=400)
|
| 39 |
try:
|
| 40 |
+
if parsed.hostname == "github.com":
|
| 41 |
+
source = github_item_source(item)
|
| 42 |
+
owner, repo = source['owner'], source['repo']
|
| 43 |
active_token = token_selector(item)
|
| 44 |
+
if source['skill']:
|
| 45 |
+
if skill_probe(item, active_token) is not True:
|
| 46 |
+
raise SkillSourceError('invalid_metadata')
|
| 47 |
+
return {'status': 'success', 'message': '资源有效', 'skill_manifest_verified': True}
|
| 48 |
headers = {"User-Agent": "ComfyUI-Ranking-SaaS"}
|
| 49 |
if active_token:
|
| 50 |
headers["Authorization"] = f"Bearer {active_token}"
|
|
|
|
| 80 |
|
| 81 |
return {"status": "success", "message": "资源有效"}
|
| 82 |
|
| 83 |
+
except SkillSourceError as error:
|
| 84 |
+
if str(error) == 'invalid_skill_package':
|
| 85 |
+
return JSONResponse(content={'error': '所选目录没有有效的Skill清单',
|
| 86 |
+
'error_code': 'invalid_skill_package'}, status_code=400)
|
| 87 |
+
return JSONResponse(content={'error': 'Skill内容暂时无法核实,请稍后重试',
|
| 88 |
+
'error_code': 'source_unavailable'}, status_code=503 if str(error) == 'probe_busy' else 502)
|
| 89 |
except WorkflowSourceError as error:
|
| 90 |
return JSONResponse(content={"error": error.message}, status_code=error.status_code)
|
| 91 |
except urllib.error.HTTPError as e:
|
router_proxy.py
CHANGED
|
@@ -21,12 +21,13 @@ from workflow_source import authorized_workflow_path, workflow_credential_target
|
|
| 21 |
from proxy_access import authorized_proxy_item, ProxyAccessError
|
| 22 |
from github_source import repository_parts, select_github_token
|
| 23 |
from github_archive import ARCHIVE_MAX_BYTES
|
| 24 |
-
from 安全认证 import require_auth
|
| 25 |
from safe_http_transport import ValidatedAsyncHTTPTransport
|
| 26 |
from workflow_download_service import prepare_workflow_download
|
| 27 |
from zip_stream import ZIP_DOWNLOAD_MAX_BYTES
|
| 28 |
from github_zip_service import prepare_github_zip
|
| 29 |
from proxy_resources import close_redirect_response
|
|
|
|
| 30 |
|
| 31 |
router = APIRouter()
|
| 32 |
MAX_PROXY_DOWNLOAD_BYTES = int(os.environ.get("MAX_PROXY_DOWNLOAD_BYTES", str(256 * 1024 * 1024)))
|
|
@@ -117,6 +118,27 @@ class ProxyGithubZipRequest(BaseModel):
|
|
| 117 |
raise ValueError('item_id只能包含字母、数字、下划线和连字符')
|
| 118 |
return v
|
| 119 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
@router.post("/api/proxy_github_zip")
|
| 121 |
@shared_limit(5)
|
| 122 |
async def proxy_github_zip(request: Request, req_data: ProxyGithubZipRequest, account: str = Depends(require_auth), db: Session = Depends(get_db)):
|
|
@@ -132,6 +154,7 @@ async def proxy_github_zip(request: Request, req_data: ProxyGithubZipRequest, ac
|
|
| 132 |
return await fetch_github_commit(repo_url, token)
|
| 133 |
|
| 134 |
return await prepare_github_zip(item, req_data.item_id, store=json_db,
|
|
|
|
| 135 |
token_selector=select_github_token, resolve_commit=resolve_commit,
|
| 136 |
send=_send_with_validated_redirects, client_factory=httpx.AsyncClient,
|
| 137 |
transport_factory=ValidatedAsyncHTTPTransport, error_response=JSONResponse,
|
|
|
|
| 21 |
from proxy_access import authorized_proxy_item, ProxyAccessError
|
| 22 |
from github_source import repository_parts, select_github_token
|
| 23 |
from github_archive import ARCHIVE_MAX_BYTES
|
| 24 |
+
from 安全认证 import require_auth, JWT_SECRET
|
| 25 |
from safe_http_transport import ValidatedAsyncHTTPTransport
|
| 26 |
from workflow_download_service import prepare_workflow_download
|
| 27 |
from zip_stream import ZIP_DOWNLOAD_MAX_BYTES
|
| 28 |
from github_zip_service import prepare_github_zip
|
| 29 |
from proxy_resources import close_redirect_response
|
| 30 |
+
from skill_archive_source import install_source, SkillSourceError
|
| 31 |
|
| 32 |
router = APIRouter()
|
| 33 |
MAX_PROXY_DOWNLOAD_BYTES = int(os.environ.get("MAX_PROXY_DOWNLOAD_BYTES", str(256 * 1024 * 1024)))
|
|
|
|
| 118 |
raise ValueError('item_id只能包含字母、数字、下划线和连字符')
|
| 119 |
return v
|
| 120 |
|
| 121 |
+
|
| 122 |
+
class SkillInstallSourceRequest(BaseModel):
|
| 123 |
+
model_config = {'extra': 'forbid'}
|
| 124 |
+
item_id: str = Field(..., min_length=1, max_length=100, pattern=r'^[A-Za-z0-9_-]+$')
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
@router.post('/api/skill_install_source')
|
| 128 |
+
@shared_limit(30)
|
| 129 |
+
async def skill_install_source(request: Request, req_data: SkillInstallSourceRequest,
|
| 130 |
+
account: str = Depends(require_auth), db: Session = Depends(get_db)):
|
| 131 |
+
try:
|
| 132 |
+
item = await authorized_proxy_item(json_db, db, req_data.item_id, account)
|
| 133 |
+
result = install_source(item, account, token_selector=select_github_token, signing_key=JWT_SECRET)
|
| 134 |
+
except ProxyAccessError as error:
|
| 135 |
+
return JSONResponse(content={'error': error.message}, status_code=error.status_code,
|
| 136 |
+
headers={'Cache-Control': 'no-store'})
|
| 137 |
+
except SkillSourceError:
|
| 138 |
+
return JSONResponse(content={'error': '该资源不是可安装的Skill来源'}, status_code=400,
|
| 139 |
+
headers={'Cache-Control': 'no-store'})
|
| 140 |
+
return JSONResponse(content=result, headers={'Cache-Control': 'no-store', 'Vary': 'Authorization'})
|
| 141 |
+
|
| 142 |
@router.post("/api/proxy_github_zip")
|
| 143 |
@shared_limit(5)
|
| 144 |
async def proxy_github_zip(request: Request, req_data: ProxyGithubZipRequest, account: str = Depends(require_auth), db: Session = Depends(get_db)):
|
|
|
|
| 154 |
return await fetch_github_commit(repo_url, token)
|
| 155 |
|
| 156 |
return await prepare_github_zip(item, req_data.item_id, store=json_db,
|
| 157 |
+
signing_key=JWT_SECRET,
|
| 158 |
token_selector=select_github_token, resolve_commit=resolve_commit,
|
| 159 |
send=_send_with_validated_redirects, client_factory=httpx.AsyncClient,
|
| 160 |
transport_factory=ValidatedAsyncHTTPTransport, error_response=JSONResponse,
|
skill_archive_source.py
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Skill目录来源解析与有界GitHub引用读取。"""
|
| 2 |
+
import asyncio
|
| 3 |
+
import base64
|
| 4 |
+
import hashlib
|
| 5 |
+
import hmac
|
| 6 |
+
import json
|
| 7 |
+
import re
|
| 8 |
+
from urllib.parse import quote, unquote, urlsplit
|
| 9 |
+
|
| 10 |
+
from proxy_resources import ProxyResources
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
SKILL_TYPES = frozenset(("skill", "recommend_skill"))
|
| 14 |
+
MAX_SOURCE_BYTES = 4096
|
| 15 |
+
MAX_METADATA_BYTES = 256 * 1024
|
| 16 |
+
MAX_REFERENCES = 512
|
| 17 |
+
MAX_TAG_DEPTH = 8
|
| 18 |
+
METADATA_SECONDS = 20
|
| 19 |
+
SHA = re.compile(r"[0-9a-fA-F]{40}\Z")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class SkillSourceError(ValueError):
|
| 23 |
+
pass
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def item_source(item, repository_parser):
|
| 27 |
+
if not isinstance(item, dict):
|
| 28 |
+
raise SkillSourceError("invalid_source")
|
| 29 |
+
value = item.get("link")
|
| 30 |
+
if (not isinstance(value, str) or not value or value != value.strip()
|
| 31 |
+
or len(value) > MAX_SOURCE_BYTES
|
| 32 |
+
or "\\" in value or any(ord(c) < 32 or ord(c) == 127 for c in value)):
|
| 33 |
+
raise SkillSourceError("invalid_source")
|
| 34 |
+
try:
|
| 35 |
+
if len(value.encode("utf-8")) > MAX_SOURCE_BYTES:
|
| 36 |
+
raise ValueError()
|
| 37 |
+
parsed = urlsplit(value)
|
| 38 |
+
if parsed.username is not None or parsed.password is not None:
|
| 39 |
+
raise ValueError()
|
| 40 |
+
parts = parsed.path.strip("/").split("/")
|
| 41 |
+
root = parsed._replace(path="/" + "/".join(parts[:2])).geturl()
|
| 42 |
+
owner, repo = repository_parser(root)
|
| 43 |
+
kind = item.get("type")
|
| 44 |
+
if kind is not None and not isinstance(kind, str):
|
| 45 |
+
raise ValueError()
|
| 46 |
+
skill = kind in SKILL_TYPES
|
| 47 |
+
tree = ()
|
| 48 |
+
if len(parts) != 2:
|
| 49 |
+
if not skill or len(parts) < 5 or len(parts) > 132 or parts[2] != "tree":
|
| 50 |
+
raise ValueError()
|
| 51 |
+
tree = tuple(unquote(part, errors="strict") for part in parts[3:])
|
| 52 |
+
if any(not part or part in (".", "..") or part.casefold() == ".git"
|
| 53 |
+
or "/" in part or "\\" in part
|
| 54 |
+
or any(ord(c) < 32 or ord(c) == 127 for c in part) for part in tree):
|
| 55 |
+
raise ValueError()
|
| 56 |
+
if "//" in parsed.path or parsed.netloc.endswith(":"):
|
| 57 |
+
raise ValueError()
|
| 58 |
+
return dict(repository=f"https://github.com/{owner}/{repo}", owner=owner,
|
| 59 |
+
repo=repo, tree=tree, skill=skill)
|
| 60 |
+
except (ValueError, UnicodeError, IndexError):
|
| 61 |
+
raise SkillSourceError("invalid_source") from None
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def source_key(source):
|
| 65 |
+
suffix = "/tree/" + "/".join(quote(part, safe="") for part in source["tree"]) if source["tree"] else ""
|
| 66 |
+
return source["repository"].lower() + suffix
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def source_header(source, subdirectory):
|
| 70 |
+
data = dict(schema=1, repository=source["repository"], name=subdirectory[-1] if subdirectory else source["repo"],
|
| 71 |
+
subdirectory=list(subdirectory))
|
| 72 |
+
return base64.urlsafe_b64encode(json.dumps(data, ensure_ascii=True,
|
| 73 |
+
separators=(",", ":")).encode("ascii")).decode("ascii")
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def installation_revision(item, link, token, *, signing_key):
|
| 77 |
+
if (not isinstance(item, dict) or not isinstance(item.get('id'), str)
|
| 78 |
+
or not re.fullmatch('[A-Za-z0-9_-]{1,100}', item['id'])
|
| 79 |
+
or not isinstance(item.get('type'), str) or item['type'] not in SKILL_TYPES
|
| 80 |
+
or not isinstance(item.get('author'), str) or not item['author'].strip()
|
| 81 |
+
or not isinstance(link, str) or token is not None and not isinstance(token, str)
|
| 82 |
+
or not isinstance(signing_key, str) or len(signing_key.encode('utf-8')) < 32):
|
| 83 |
+
raise SkillSourceError('invalid_source')
|
| 84 |
+
message = b'ComfyUI-Ranking.Skill.InstallSource.v1\0' + json.dumps(
|
| 85 |
+
[item['id'], item['type'], link, item['author'], token or ''],
|
| 86 |
+
ensure_ascii=True, separators=(',', ':')).encode('ascii')
|
| 87 |
+
return hmac.new(signing_key.encode('utf-8'), message, hashlib.sha256).hexdigest()
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def install_source(item, account, *, token_selector, signing_key):
|
| 91 |
+
"""仅描述已授权的Skill来源,不返回凭据或下载任意请求地址。"""
|
| 92 |
+
if (not isinstance(item, dict) or not isinstance(item.get('type'), str)
|
| 93 |
+
or item['type'] not in SKILL_TYPES or item.get('is_netdisk', False) is not False):
|
| 94 |
+
raise SkillSourceError('invalid_source')
|
| 95 |
+
value = item.get('link')
|
| 96 |
+
if (not isinstance(value, str) or not value or value != value.strip() or len(value) > MAX_SOURCE_BYTES
|
| 97 |
+
or '\\' in value or any(ord(c) < 32 or ord(c) == 127 for c in value)):
|
| 98 |
+
raise SkillSourceError('invalid_source')
|
| 99 |
+
try:
|
| 100 |
+
if len(value.encode('utf-8')) > MAX_SOURCE_BYTES:
|
| 101 |
+
raise ValueError()
|
| 102 |
+
parsed = urlsplit(value)
|
| 103 |
+
if (parsed.scheme != 'https' or parsed.username is not None or parsed.password is not None
|
| 104 |
+
or parsed.port not in (None, 443) or parsed.query or parsed.fragment
|
| 105 |
+
or parsed.netloc.endswith(':') or '//' in parsed.path):
|
| 106 |
+
raise ValueError()
|
| 107 |
+
if parsed.hostname == 'github.com':
|
| 108 |
+
from github_source import repository_parts
|
| 109 |
+
source = item_source(item, repository_parts)
|
| 110 |
+
repository, link = source['repository'], source_key(source)
|
| 111 |
+
token = token_selector(item)
|
| 112 |
+
transport = 'github_zip'
|
| 113 |
+
else:
|
| 114 |
+
if (parsed.hostname not in ('gitlab.com', 'gitee.com', 'bitbucket.org', 'kkgithub.com')
|
| 115 |
+
or not re.fullmatch(r'/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/?', parsed.path)
|
| 116 |
+
or item.get('github_token') not in (None, '')):
|
| 117 |
+
raise ValueError()
|
| 118 |
+
parts = parsed.path.strip('/').split('/')
|
| 119 |
+
if any(part in ('.', '..') for part in parts):
|
| 120 |
+
raise ValueError()
|
| 121 |
+
repository = f'https://{parsed.hostname}/' + '/'.join(parts)
|
| 122 |
+
link, token, transport = repository, None, 'public_git'
|
| 123 |
+
except (ValueError, UnicodeError):
|
| 124 |
+
raise SkillSourceError('invalid_source') from None
|
| 125 |
+
author, item_id = item.get('author'), item.get('id')
|
| 126 |
+
if (not isinstance(author, str) or not author.strip() or not isinstance(item_id, str)
|
| 127 |
+
or not re.fullmatch(r'[A-Za-z0-9_-]{1,100}', item_id)
|
| 128 |
+
or not isinstance(account, str) or not account):
|
| 129 |
+
raise SkillSourceError('invalid_source')
|
| 130 |
+
revision = installation_revision(item, link, token, signing_key=signing_key)
|
| 131 |
+
return dict(status='success', schema=1, item_id=item_id, account=account,
|
| 132 |
+
transport=transport, repository=repository, link=link, revision=revision)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _unique_object(pairs):
|
| 136 |
+
result = {}
|
| 137 |
+
for key, value in pairs:
|
| 138 |
+
if key in result:
|
| 139 |
+
raise SkillSourceError("invalid_metadata")
|
| 140 |
+
result[key] = value
|
| 141 |
+
return result
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def _invalid_constant(_):
|
| 145 |
+
raise SkillSourceError("invalid_metadata")
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
async def read_metadata_response(response, resources):
|
| 149 |
+
try:
|
| 150 |
+
if response.status_code == 404:
|
| 151 |
+
return None
|
| 152 |
+
if response.status_code != 200 or (response.headers.get("content-encoding") or "identity").lower() != "identity":
|
| 153 |
+
raise SkillSourceError("metadata_unavailable")
|
| 154 |
+
if response.headers.get("link"):
|
| 155 |
+
raise SkillSourceError("incomplete_metadata")
|
| 156 |
+
declared = response.headers.get("content-length")
|
| 157 |
+
if declared is not None and (not re.fullmatch(r"[0-9]{1,10}", declared)
|
| 158 |
+
or not 0 < int(declared) <= MAX_METADATA_BYTES):
|
| 159 |
+
raise SkillSourceError("invalid_metadata")
|
| 160 |
+
body = bytearray()
|
| 161 |
+
async for chunk in response.aiter_raw(chunk_size=65536):
|
| 162 |
+
if len(body) + len(chunk) > MAX_METADATA_BYTES:
|
| 163 |
+
raise SkillSourceError("metadata_too_large")
|
| 164 |
+
body.extend(chunk)
|
| 165 |
+
await asyncio.sleep(0)
|
| 166 |
+
if declared is not None and len(body) != int(declared):
|
| 167 |
+
raise SkillSourceError("invalid_metadata")
|
| 168 |
+
try:
|
| 169 |
+
return json.loads(body.decode("utf-8"), object_pairs_hook=_unique_object,
|
| 170 |
+
parse_constant=_invalid_constant)
|
| 171 |
+
except (UnicodeError, ValueError, RecursionError):
|
| 172 |
+
raise SkillSourceError("invalid_metadata") from None
|
| 173 |
+
finally:
|
| 174 |
+
await resources.close_response(response)
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
async def resolve_tree(source, token, *, send, client_factory, transport_factory):
|
| 178 |
+
tree = source["tree"]
|
| 179 |
+
if not tree:
|
| 180 |
+
raise SkillSourceError("missing_reference")
|
| 181 |
+
client = client_factory(follow_redirects=False, trust_env=False, transport=transport_factory())
|
| 182 |
+
resources = ProxyResources(client)
|
| 183 |
+
headers = {"Accept": "application/vnd.github+json", "Accept-Encoding": "identity",
|
| 184 |
+
"User-Agent": "ComfyUI-Ranking-SaaS"}
|
| 185 |
+
if token:
|
| 186 |
+
headers["Authorization"] = f"Bearer {token}"
|
| 187 |
+
prefix = f"https://api.github.com/repos/{source['owner']}/{source['repo']}/git/"
|
| 188 |
+
|
| 189 |
+
async def read_json(path):
|
| 190 |
+
response = resources.track(await send(client, "GET", prefix + path, headers=headers,
|
| 191 |
+
timeout=METADATA_SECONDS))
|
| 192 |
+
return await read_metadata_response(response, resources)
|
| 193 |
+
|
| 194 |
+
try:
|
| 195 |
+
async with asyncio.timeout(METADATA_SECONDS):
|
| 196 |
+
if SHA.fullmatch(tree[0]):
|
| 197 |
+
sha = tree[0].lower()
|
| 198 |
+
value = await read_json("commits/" + sha)
|
| 199 |
+
if not isinstance(value, dict) or value.get("sha") != sha:
|
| 200 |
+
raise SkillSourceError("invalid_commit")
|
| 201 |
+
return sha, tree[1:]
|
| 202 |
+
candidates = []
|
| 203 |
+
for namespace in ("heads", "tags"):
|
| 204 |
+
values = await read_json("matching-refs/" + namespace + "/" + quote(tree[0], safe=""))
|
| 205 |
+
if values is None:
|
| 206 |
+
continue
|
| 207 |
+
if not isinstance(values, list) or len(values) > MAX_REFERENCES:
|
| 208 |
+
raise SkillSourceError("invalid_metadata")
|
| 209 |
+
seen = set()
|
| 210 |
+
for row in values:
|
| 211 |
+
if not isinstance(row, dict) or not isinstance(row.get("ref"), str):
|
| 212 |
+
raise SkillSourceError("invalid_metadata")
|
| 213 |
+
reference, obj = row["ref"], row.get("object")
|
| 214 |
+
if (reference in seen or not reference.startswith("refs/" + namespace + "/")
|
| 215 |
+
or not isinstance(obj, dict) or obj.get("type") not in ("commit", "tag")
|
| 216 |
+
or not isinstance(obj.get("sha"), str) or not SHA.fullmatch(obj["sha"])
|
| 217 |
+
or namespace == "heads" and obj["type"] != "commit"):
|
| 218 |
+
raise SkillSourceError("invalid_metadata")
|
| 219 |
+
seen.add(reference)
|
| 220 |
+
parts = tuple(reference[len("refs/" + namespace + "/"):].split("/"))
|
| 221 |
+
if parts and tree[:len(parts)] == parts and len(tree) > len(parts):
|
| 222 |
+
candidates.append((len(parts), obj, tree[len(parts):]))
|
| 223 |
+
if not candidates:
|
| 224 |
+
raise SkillSourceError("missing_reference")
|
| 225 |
+
longest = max(depth for depth, _, _ in candidates)
|
| 226 |
+
choices = set()
|
| 227 |
+
for depth, obj, subdirectory in candidates:
|
| 228 |
+
if depth != longest:
|
| 229 |
+
continue
|
| 230 |
+
visited = set()
|
| 231 |
+
while obj["type"] == "tag":
|
| 232 |
+
sha = obj["sha"].lower()
|
| 233 |
+
if sha in visited or len(visited) >= MAX_TAG_DEPTH:
|
| 234 |
+
raise SkillSourceError("invalid_tag")
|
| 235 |
+
visited.add(sha)
|
| 236 |
+
value = await read_json("tags/" + sha)
|
| 237 |
+
obj = value.get("object") if isinstance(value, dict) else None
|
| 238 |
+
if (not isinstance(obj, dict) or obj.get("type") not in ("commit", "tag")
|
| 239 |
+
or not isinstance(obj.get("sha"), str) or not SHA.fullmatch(obj["sha"])):
|
| 240 |
+
raise SkillSourceError("invalid_tag")
|
| 241 |
+
choices.add((obj["sha"].lower(), subdirectory))
|
| 242 |
+
if len(choices) != 1:
|
| 243 |
+
raise SkillSourceError("ambiguous_reference")
|
| 244 |
+
return next(iter(choices))
|
| 245 |
+
finally:
|
| 246 |
+
await resources.close()
|
skill_preflight.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""购买前只核实Skill清单,不交付私有内容或替代下载授权。"""
|
| 2 |
+
import asyncio
|
| 3 |
+
import threading
|
| 4 |
+
from urllib.parse import quote
|
| 5 |
+
|
| 6 |
+
from proxy_resources import ProxyResources
|
| 7 |
+
from github_source import repository_parts, github_item_source
|
| 8 |
+
from skill_archive_source import resolve_tree, read_metadata_response, SkillSourceError, SHA, METADATA_SECONDS
|
| 9 |
+
|
| 10 |
+
MAX_TREE_ENTRIES = 1000
|
| 11 |
+
MAX_REQUESTS = 256
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _sha(value):
|
| 15 |
+
if not isinstance(value, str) or not SHA.fullmatch(value):
|
| 16 |
+
raise SkillSourceError('invalid_metadata')
|
| 17 |
+
return value.lower()
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _segment(value):
|
| 21 |
+
try:
|
| 22 |
+
size = len(value.encode('utf-8')) if isinstance(value, str) else 0
|
| 23 |
+
except UnicodeError:
|
| 24 |
+
return False
|
| 25 |
+
return (isinstance(value, str) and 0 < size <= 4096
|
| 26 |
+
and value not in ('.', '..') and value.casefold() != '.git'
|
| 27 |
+
and '/' not in value and '\\' not in value
|
| 28 |
+
and not any(ord(c) < 32 or ord(c) == 127 for c in value))
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _tree(value, expected):
|
| 32 |
+
if (not isinstance(value, dict) or _sha(value.get('sha')) != expected
|
| 33 |
+
or value.get('truncated') is not False or not isinstance(value.get('tree'), list)
|
| 34 |
+
or len(value['tree']) > MAX_TREE_ENTRIES):
|
| 35 |
+
raise SkillSourceError('invalid_metadata')
|
| 36 |
+
result = {}
|
| 37 |
+
aliases = set()
|
| 38 |
+
for row in value['tree']:
|
| 39 |
+
if (not isinstance(row, dict) or not _segment(row.get('path'))
|
| 40 |
+
or row['path'].casefold() in aliases):
|
| 41 |
+
raise SkillSourceError('invalid_metadata')
|
| 42 |
+
_sha(row.get('sha'))
|
| 43 |
+
kind, mode = row.get('type'), row.get('mode')
|
| 44 |
+
if ((kind, mode) not in {('blob', '100644'), ('blob', '100755'), ('blob', '120000'),
|
| 45 |
+
('tree', '040000'), ('commit', '160000')}
|
| 46 |
+
or kind == 'blob' and (type(row.get('size')) is not int or row['size'] < 0)):
|
| 47 |
+
raise SkillSourceError('invalid_metadata')
|
| 48 |
+
aliases.add(row['path'].casefold()); result[row['path']] = row
|
| 49 |
+
return result
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _manifest(rows):
|
| 53 |
+
for name in ('SKILL.md', 'SKILL.cn.md'):
|
| 54 |
+
if name in rows:
|
| 55 |
+
row = rows[name]
|
| 56 |
+
if row['type'] != 'blob' or row['mode'] not in ('100644', '100755'):
|
| 57 |
+
raise SkillSourceError('invalid_skill_package')
|
| 58 |
+
return True
|
| 59 |
+
return False
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
async def probe_github_skill(source, token, *, send, client_factory, transport_factory):
|
| 63 |
+
if not isinstance(source, dict) or source.get('skill') is not True:
|
| 64 |
+
raise SkillSourceError('invalid_source')
|
| 65 |
+
try:
|
| 66 |
+
if (repository_parts(source['repository']) != (source['owner'], source['repo'])
|
| 67 |
+
or not isinstance(source['tree'], tuple) or len(source['tree']) > 129
|
| 68 |
+
or not all(_segment(part) for part in source['tree'])):
|
| 69 |
+
raise ValueError()
|
| 70 |
+
except (KeyError, TypeError, ValueError):
|
| 71 |
+
raise SkillSourceError('invalid_source') from None
|
| 72 |
+
prefix = 'https://api.github.com/repos/' + source['owner'] + '/' + source['repo']
|
| 73 |
+
client = client_factory(follow_redirects=False, trust_env=False, transport=transport_factory())
|
| 74 |
+
resources = ProxyResources(client)
|
| 75 |
+
headers = {'Accept': 'application/vnd.github+json', 'Accept-Encoding': 'identity',
|
| 76 |
+
'User-Agent': 'ComfyUI-Ranking-SaaS'}
|
| 77 |
+
if token:
|
| 78 |
+
headers['Authorization'] = 'Bearer ' + token
|
| 79 |
+
requests = 0
|
| 80 |
+
async def bounded_send(*args, **kwargs):
|
| 81 |
+
nonlocal requests
|
| 82 |
+
requests += 1
|
| 83 |
+
if requests > MAX_REQUESTS:
|
| 84 |
+
raise SkillSourceError('metadata_too_large')
|
| 85 |
+
return await send(*args, **kwargs)
|
| 86 |
+
|
| 87 |
+
async def read(path):
|
| 88 |
+
response = resources.track(await bounded_send(client, 'GET', prefix + path,
|
| 89 |
+
headers=headers, timeout=METADATA_SECONDS))
|
| 90 |
+
return await read_metadata_response(response, resources)
|
| 91 |
+
|
| 92 |
+
async def get_tree(sha):
|
| 93 |
+
return _tree(await read('/git/trees/' + sha), sha)
|
| 94 |
+
|
| 95 |
+
try:
|
| 96 |
+
async with asyncio.timeout(METADATA_SECONDS):
|
| 97 |
+
if source['tree']:
|
| 98 |
+
commit, subdirectory = await resolve_tree(source, token, send=bounded_send,
|
| 99 |
+
client_factory=client_factory, transport_factory=transport_factory)
|
| 100 |
+
else:
|
| 101 |
+
repository = await read('')
|
| 102 |
+
branch = repository.get('default_branch') if isinstance(repository, dict) else None
|
| 103 |
+
if (not isinstance(branch, str) or not all(_segment(part) for part in branch.split('/'))
|
| 104 |
+
or len(branch.encode('utf-8')) > 4096):
|
| 105 |
+
raise SkillSourceError('invalid_metadata')
|
| 106 |
+
reference = 'refs/heads/' + branch
|
| 107 |
+
values = await read('/git/matching-refs/heads/' + quote(branch, safe=''))
|
| 108 |
+
if not isinstance(values, list) or len(values) > MAX_TREE_ENTRIES:
|
| 109 |
+
raise SkillSourceError('invalid_metadata')
|
| 110 |
+
matches = [row for row in values if isinstance(row, dict) and row.get('ref') == reference]
|
| 111 |
+
if len(matches) != 1 or not isinstance(matches[0].get('object'), dict) or matches[0]['object'].get('type') != 'commit':
|
| 112 |
+
raise SkillSourceError('invalid_metadata')
|
| 113 |
+
commit, subdirectory = _sha(matches[0]['object'].get('sha')), ()
|
| 114 |
+
metadata = await read('/git/commits/' + commit)
|
| 115 |
+
if (not isinstance(metadata, dict) or _sha(metadata.get('sha')) != commit
|
| 116 |
+
or not isinstance(metadata.get('tree'), dict)):
|
| 117 |
+
raise SkillSourceError('invalid_metadata')
|
| 118 |
+
rows = await get_tree(_sha(metadata['tree'].get('sha')))
|
| 119 |
+
for name in subdirectory:
|
| 120 |
+
row = rows.get(name)
|
| 121 |
+
if not row or row['type'] != 'tree' or row['mode'] != '040000':
|
| 122 |
+
raise SkillSourceError('invalid_skill_package')
|
| 123 |
+
rows = await get_tree(_sha(row['sha']))
|
| 124 |
+
if _manifest(rows):
|
| 125 |
+
return True
|
| 126 |
+
for row in rows.values():
|
| 127 |
+
if row['type'] == 'tree' and row['mode'] == '040000':
|
| 128 |
+
if _manifest(await get_tree(_sha(row['sha']))):
|
| 129 |
+
return True
|
| 130 |
+
raise SkillSourceError('invalid_skill_package')
|
| 131 |
+
finally:
|
| 132 |
+
await resources.close()
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
_PROBE_CAPACITY = threading.BoundedSemaphore(2)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def verify_skill_manifest(item, token, *, send, client_factory, transport_factory):
|
| 139 |
+
if not _PROBE_CAPACITY.acquire(blocking=False):
|
| 140 |
+
raise SkillSourceError('probe_busy')
|
| 141 |
+
try:
|
| 142 |
+
source = github_item_source(item)
|
| 143 |
+
return asyncio.run(probe_github_skill(source, token, send=send,
|
| 144 |
+
client_factory=client_factory, transport_factory=transport_factory))
|
| 145 |
+
finally:
|
| 146 |
+
_PROBE_CAPACITY.release()
|
管理界面/app.js
CHANGED
|
@@ -2085,7 +2085,18 @@
|
|
| 2085 |
const res = await apiRequest('GET', '/api/disputes/' + encodeURIComponent(disputeId));
|
| 2086 |
if (!read.isCurrent()) return;
|
| 2087 |
const dispute = parseAdminDispute(res, disputeId);
|
| 2088 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2089 |
} catch (err) {
|
| 2090 |
if (isStaleSessionError(err) || !read.isCurrent()) return;
|
| 2091 |
body.innerHTML = '<div class="state-block state-error"><div class="empty-mark">!</div><h3>' + escapeHtml(adminMessage('dispute_load_failed')) + '</h3><p>' +
|
|
|
|
| 2085 |
const res = await apiRequest('GET', '/api/disputes/' + encodeURIComponent(disputeId));
|
| 2086 |
if (!read.isCurrent()) return;
|
| 2087 |
const dispute = parseAdminDispute(res, disputeId);
|
| 2088 |
+
let display = dispute;
|
| 2089 |
+
if ([dispute.evidence, dispute.response_evidence].some(images => Array.isArray(images) && images.length)) {
|
| 2090 |
+
const { normalizeDisputeEvidence } = await import('../m/assets/task/申诉数据工具.js').catch(() => {
|
| 2091 |
+
throw new ApiError(adminMessage('admin_data_unavailable'), 0);
|
| 2092 |
+
});
|
| 2093 |
+
if (!read.isCurrent()) return;
|
| 2094 |
+
const serviceBaseUrl = new URL(API_BASE || window.location.origin, window.location.href).href;
|
| 2095 |
+
display = { ...dispute,
|
| 2096 |
+
evidence: normalizeDisputeEvidence(dispute.evidence, serviceBaseUrl),
|
| 2097 |
+
response_evidence: normalizeDisputeEvidence(dispute.response_evidence, serviceBaseUrl) };
|
| 2098 |
+
}
|
| 2099 |
+
renderDisputeDetail(body, display);
|
| 2100 |
} catch (err) {
|
| 2101 |
if (isStaleSessionError(err) || !read.isCurrent()) return;
|
| 2102 |
body.innerHTML = '<div class="state-block state-error"><div class="empty-mark">!</div><h3>' + escapeHtml(adminMessage('dispute_load_failed')) + '</h3><p>' +
|