Spaces:
Running on Zero
Running on Zero
Resolve CivitAI LoRA fetches
Browse filesReuse ltxapifetch model resolution, token handling, mirror fallback, bucket persistence, and safetensors validation.
- lora_loader.py +107 -21
lora_loader.py
CHANGED
|
@@ -18,6 +18,7 @@ import json
|
|
| 18 |
import hashlib
|
| 19 |
import tempfile
|
| 20 |
import urllib.request
|
|
|
|
| 21 |
import safetensors.torch
|
| 22 |
from huggingface_hub import hf_hub_download
|
| 23 |
|
|
@@ -30,6 +31,7 @@ HF_TOKEN = os.environ.get("HF_TOKEN") # authenticated downloads (covers private
|
|
| 30 |
LORA_LIBRARY_DIR = os.environ.get("LORA_LIBRARY_DIR", "/data/loras")
|
| 31 |
LORA_LIBRARY_MANIFEST = os.path.join(LORA_LIBRARY_DIR, "library.json")
|
| 32 |
MAX_CUSTOM_LORA_BYTES = int(os.environ.get("MAX_CUSTOM_LORA_BYTES", 8 * 1024**3))
|
|
|
|
| 33 |
|
| 34 |
# Pinned commit hashes (2026-07-19) — protection from breaking upstream changes
|
| 35 |
PINNED_REVISIONS = {
|
|
@@ -218,6 +220,58 @@ def _stage_for_custom_lora(filename, requested_stage):
|
|
| 218 |
return "LOW" if re.search(r"(?:low|[_-]ln\\b|[_-]l\\b)", filename, re.IGNORECASE) else "HIGH"
|
| 219 |
|
| 220 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
def add_custom_lora(label, url, stage="auto"):
|
| 222 |
"""Fetch one public safetensors URL into the persistent LoRA library.
|
| 223 |
|
|
@@ -227,37 +281,69 @@ def add_custom_lora(label, url, stage="auto"):
|
|
| 227 |
"""
|
| 228 |
label = (label or "").strip()
|
| 229 |
url = (url or "").strip()
|
| 230 |
-
if not label:
|
| 231 |
-
raise ValueError("Give this LoRA a name.")
|
| 232 |
-
if not re.fullmatch(r"[\\w .()'\\-]{1,96}", label):
|
| 233 |
-
raise ValueError("LoRA names may use letters, numbers, spaces, . ( ) ' and -.")
|
| 234 |
parsed = urllib.parse.urlparse(url)
|
| 235 |
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
| 236 |
-
raise ValueError("Use
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 240 |
|
| 241 |
os.makedirs(LORA_LIBRARY_DIR, exist_ok=True)
|
| 242 |
digest = hashlib.sha256(url.encode("utf-8")).hexdigest()[:12]
|
| 243 |
safe_label = re.sub(r"[^A-Za-z0-9._-]+", "-", label).strip(".-") or "lora"
|
| 244 |
destination = os.path.join(LORA_LIBRARY_DIR, f"{safe_label}-{digest}.safetensors")
|
| 245 |
fd, temporary_path = tempfile.mkstemp(prefix="download-", suffix=".part", dir=LORA_LIBRARY_DIR)
|
|
|
|
| 246 |
size = 0
|
| 247 |
try:
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 261 |
if size == 0:
|
| 262 |
raise ValueError("The LoRA endpoint returned an empty file.")
|
| 263 |
# Validate the safetensors header without materializing the weights.
|
|
|
|
| 18 |
import hashlib
|
| 19 |
import tempfile
|
| 20 |
import urllib.request
|
| 21 |
+
import requests
|
| 22 |
import safetensors.torch
|
| 23 |
from huggingface_hub import hf_hub_download
|
| 24 |
|
|
|
|
| 31 |
LORA_LIBRARY_DIR = os.environ.get("LORA_LIBRARY_DIR", "/data/loras")
|
| 32 |
LORA_LIBRARY_MANIFEST = os.path.join(LORA_LIBRARY_DIR, "library.json")
|
| 33 |
MAX_CUSTOM_LORA_BYTES = int(os.environ.get("MAX_CUSTOM_LORA_BYTES", 8 * 1024**3))
|
| 34 |
+
CIVITAI_HOSTS = ("civitai.com", "civitai.red", "civitai.green", "civitai.work")
|
| 35 |
|
| 36 |
# Pinned commit hashes (2026-07-19) — protection from breaking upstream changes
|
| 37 |
PINNED_REVISIONS = {
|
|
|
|
| 220 |
return "LOW" if re.search(r"(?:low|[_-]ln\\b|[_-]l\\b)", filename, re.IGNORECASE) else "HIGH"
|
| 221 |
|
| 222 |
|
| 223 |
+
def _civitai_token():
|
| 224 |
+
return (os.environ.get("CIVITAI_TOKEN") or os.environ.get("CIVITAI_API_KEY") or "").strip()
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def _slug(value, fallback):
|
| 228 |
+
value = re.sub(r"[^A-Za-z0-9._-]+", "-", value or "").strip(".-_")
|
| 229 |
+
return (value or fallback)[:120]
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _resolve_civitai_lora(source):
|
| 233 |
+
"""Resolve a CivitAI model page or download link to one safetensors file."""
|
| 234 |
+
parsed = urllib.parse.urlparse(source)
|
| 235 |
+
host = (parsed.hostname or "").lower()
|
| 236 |
+
if host not in CIVITAI_HOSTS:
|
| 237 |
+
raise ValueError("This is not a CivitAI URL.")
|
| 238 |
+
if parsed.scheme != "https":
|
| 239 |
+
raise ValueError("Use an HTTPS CivitAI page or download link.")
|
| 240 |
+
|
| 241 |
+
page_match = re.search(r"/models/(\d+)", parsed.path, flags=re.IGNORECASE)
|
| 242 |
+
download_match = re.search(r"/api/download/models/(\d+)", parsed.path, flags=re.IGNORECASE)
|
| 243 |
+
query = dict(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True))
|
| 244 |
+
if download_match:
|
| 245 |
+
version_id = download_match.group(1)
|
| 246 |
+
file_id = query.get("fileId") or "file"
|
| 247 |
+
return source, f"CivitAI LoRA {version_id}", f"civitai-lora--{version_id}--{file_id}.safetensors"
|
| 248 |
+
|
| 249 |
+
version_id = query.get("modelVersionId") or ""
|
| 250 |
+
api_base = f"https://{host}/api/v1"
|
| 251 |
+
if not version_id and page_match:
|
| 252 |
+
model = requests.get(f"{api_base}/models/{page_match.group(1)}", timeout=30).json()
|
| 253 |
+
versions = model.get("modelVersions") or []
|
| 254 |
+
if not versions:
|
| 255 |
+
raise ValueError("CivitAI returned no versions for that model page.")
|
| 256 |
+
version_id = str(versions[0].get("id") or "")
|
| 257 |
+
if not version_id:
|
| 258 |
+
raise ValueError("Could not identify a CivitAI model version from that link.")
|
| 259 |
+
|
| 260 |
+
version = requests.get(f"{api_base}/model-versions/{version_id}", timeout=30).json()
|
| 261 |
+
files = [item for item in (version.get("files") or []) if str(item.get("name") or "").lower().endswith(".safetensors")]
|
| 262 |
+
if not files:
|
| 263 |
+
raise ValueError("That CivitAI version has no safetensors file.")
|
| 264 |
+
chosen = next((item for item in files if item.get("primary")), files[0])
|
| 265 |
+
file_id = chosen.get("id")
|
| 266 |
+
if not file_id:
|
| 267 |
+
raise ValueError("CivitAI did not provide a safetensors file id.")
|
| 268 |
+
model_name = str((version.get("model") or {}).get("name") or "").strip()
|
| 269 |
+
version_name = str(version.get("name") or "").strip()
|
| 270 |
+
display_name = " - ".join(part for part in (model_name, version_name) if part) or str(chosen.get("name") or "CivitAI LoRA")
|
| 271 |
+
filename = f"{_slug(model_name, 'civitai-lora')}--{_slug(version_name, str(version_id))}.safetensors"
|
| 272 |
+
return f"https://{host}/api/download/models/{version_id}?fileId={file_id}", display_name[:160], filename
|
| 273 |
+
|
| 274 |
+
|
| 275 |
def add_custom_lora(label, url, stage="auto"):
|
| 276 |
"""Fetch one public safetensors URL into the persistent LoRA library.
|
| 277 |
|
|
|
|
| 281 |
"""
|
| 282 |
label = (label or "").strip()
|
| 283 |
url = (url or "").strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 284 |
parsed = urllib.parse.urlparse(url)
|
| 285 |
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
| 286 |
+
raise ValueError("Use an http(s) safetensors endpoint or a CivitAI model page.")
|
| 287 |
+
|
| 288 |
+
host = (parsed.hostname or "").lower()
|
| 289 |
+
if host in CIVITAI_HOSTS:
|
| 290 |
+
download_url, detected_label, source_name = _resolve_civitai_lora(url)
|
| 291 |
+
label = label or detected_label
|
| 292 |
+
else:
|
| 293 |
+
download_url = url
|
| 294 |
+
source_name = os.path.basename(urllib.parse.unquote(parsed.path))
|
| 295 |
+
if not source_name.lower().endswith(".safetensors"):
|
| 296 |
+
raise ValueError("Direct endpoints must end in .safetensors.")
|
| 297 |
+
label = label or os.path.splitext(source_name)[0]
|
| 298 |
+
if not re.fullmatch(r"[\\w .()'\\-]{1,96}", label):
|
| 299 |
+
raise ValueError("LoRA names may use letters, numbers, spaces, . ( ) ' and -.")
|
| 300 |
|
| 301 |
os.makedirs(LORA_LIBRARY_DIR, exist_ok=True)
|
| 302 |
digest = hashlib.sha256(url.encode("utf-8")).hexdigest()[:12]
|
| 303 |
safe_label = re.sub(r"[^A-Za-z0-9._-]+", "-", label).strip(".-") or "lora"
|
| 304 |
destination = os.path.join(LORA_LIBRARY_DIR, f"{safe_label}-{digest}.safetensors")
|
| 305 |
fd, temporary_path = tempfile.mkstemp(prefix="download-", suffix=".part", dir=LORA_LIBRARY_DIR)
|
| 306 |
+
os.close(fd)
|
| 307 |
size = 0
|
| 308 |
try:
|
| 309 |
+
download_parsed = urllib.parse.urlparse(download_url)
|
| 310 |
+
token = _civitai_token()
|
| 311 |
+
if token and host in CIVITAI_HOSTS and "token=" not in download_parsed.query:
|
| 312 |
+
query = urllib.parse.parse_qsl(download_parsed.query, keep_blank_values=True) + [("token", token)]
|
| 313 |
+
download_url = urllib.parse.urlunparse(download_parsed._replace(query=urllib.parse.urlencode(query)))
|
| 314 |
+
download_parsed = urllib.parse.urlparse(download_url)
|
| 315 |
+
candidates = [download_url]
|
| 316 |
+
if host in CIVITAI_HOSTS and (download_parsed.hostname or "").lower() != "civitai.red":
|
| 317 |
+
for alternate in CIVITAI_HOSTS:
|
| 318 |
+
if download_parsed.hostname and alternate != download_parsed.hostname.lower():
|
| 319 |
+
candidates.append(download_url.replace(download_parsed.hostname, alternate, 1))
|
| 320 |
+
|
| 321 |
+
last_error = "download failed"
|
| 322 |
+
for candidate in candidates:
|
| 323 |
+
size = 0
|
| 324 |
+
try:
|
| 325 |
+
with requests.get(candidate, stream=True, allow_redirects=True, timeout=(30, 600), headers={"User-Agent": "wan2.2-lora-library/1.0"}) as response:
|
| 326 |
+
response.raise_for_status()
|
| 327 |
+
if "text/html" in response.headers.get("Content-Type", "").lower():
|
| 328 |
+
raise ValueError("The endpoint returned a web page instead of LoRA weights.")
|
| 329 |
+
length = response.headers.get("Content-Length")
|
| 330 |
+
if length and int(length) > MAX_CUSTOM_LORA_BYTES:
|
| 331 |
+
raise ValueError("That LoRA is larger than this Space's 8 GB library limit.")
|
| 332 |
+
with open(temporary_path, "wb") as output:
|
| 333 |
+
for chunk in response.iter_content(chunk_size=4 * 1024 * 1024):
|
| 334 |
+
if not chunk:
|
| 335 |
+
continue
|
| 336 |
+
size += len(chunk)
|
| 337 |
+
if size > MAX_CUSTOM_LORA_BYTES:
|
| 338 |
+
raise ValueError("That LoRA is larger than this Space's 8 GB library limit.")
|
| 339 |
+
output.write(chunk)
|
| 340 |
+
break
|
| 341 |
+
except (OSError, ValueError, requests.RequestException) as exc:
|
| 342 |
+
last_error = str(exc)
|
| 343 |
+
if os.path.exists(temporary_path):
|
| 344 |
+
os.unlink(temporary_path)
|
| 345 |
+
else:
|
| 346 |
+
raise ValueError(last_error)
|
| 347 |
if size == 0:
|
| 348 |
raise ValueError("The LoRA endpoint returned an empty file.")
|
| 349 |
# Validate the safetensors header without materializing the weights.
|