Quantumbraid commited on
Commit
e159346
·
verified ·
1 Parent(s): d4b016e

Resolve CivitAI LoRA fetches

Browse files

Reuse ltxapifetch model resolution, token handling, mirror fallback, bucket persistence, and safetensors validation.

Files changed (1) hide show
  1. lora_loader.py +107 -21
lora_loader.py CHANGED
@@ -18,6 +18,7 @@ import json
18
  import hashlib
19
  import tempfile
20
  import urllib.request
 
21
  import safetensors.torch
22
  from huggingface_hub import hf_hub_download
23
 
@@ -30,6 +31,7 @@ HF_TOKEN = os.environ.get("HF_TOKEN") # authenticated downloads (covers private
30
  LORA_LIBRARY_DIR = os.environ.get("LORA_LIBRARY_DIR", "/data/loras")
31
  LORA_LIBRARY_MANIFEST = os.path.join(LORA_LIBRARY_DIR, "library.json")
32
  MAX_CUSTOM_LORA_BYTES = int(os.environ.get("MAX_CUSTOM_LORA_BYTES", 8 * 1024**3))
 
33
 
34
  # Pinned commit hashes (2026-07-19) — protection from breaking upstream changes
35
  PINNED_REVISIONS = {
@@ -218,6 +220,58 @@ def _stage_for_custom_lora(filename, requested_stage):
218
  return "LOW" if re.search(r"(?:low|[_-]ln\\b|[_-]l\\b)", filename, re.IGNORECASE) else "HIGH"
219
 
220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
221
  def add_custom_lora(label, url, stage="auto"):
222
  """Fetch one public safetensors URL into the persistent LoRA library.
223
 
@@ -227,37 +281,69 @@ def add_custom_lora(label, url, stage="auto"):
227
  """
228
  label = (label or "").strip()
229
  url = (url or "").strip()
230
- if not label:
231
- raise ValueError("Give this LoRA a name.")
232
- if not re.fullmatch(r"[\\w .()'\\-]{1,96}", label):
233
- raise ValueError("LoRA names may use letters, numbers, spaces, . ( ) ' and -.")
234
  parsed = urllib.parse.urlparse(url)
235
  if parsed.scheme not in {"http", "https"} or not parsed.netloc:
236
- raise ValueError("Use a direct http(s) URL to a .safetensors file.")
237
- source_name = os.path.basename(urllib.parse.unquote(parsed.path))
238
- if not source_name.lower().endswith(".safetensors"):
239
- raise ValueError("The endpoint path must end in .safetensors.")
 
 
 
 
 
 
 
 
 
 
240
 
241
  os.makedirs(LORA_LIBRARY_DIR, exist_ok=True)
242
  digest = hashlib.sha256(url.encode("utf-8")).hexdigest()[:12]
243
  safe_label = re.sub(r"[^A-Za-z0-9._-]+", "-", label).strip(".-") or "lora"
244
  destination = os.path.join(LORA_LIBRARY_DIR, f"{safe_label}-{digest}.safetensors")
245
  fd, temporary_path = tempfile.mkstemp(prefix="download-", suffix=".part", dir=LORA_LIBRARY_DIR)
 
246
  size = 0
247
  try:
248
- request = urllib.request.Request(url, headers={"User-Agent": "wan2.2-lora-library/1.0"})
249
- with urllib.request.urlopen(request, timeout=60) as response, os.fdopen(fd, "wb") as output:
250
- length = response.headers.get("Content-Length")
251
- if length and int(length) > MAX_CUSTOM_LORA_BYTES:
252
- raise ValueError("That LoRA is larger than this Space's 8 GB library limit.")
253
- while True:
254
- chunk = response.read(1024 * 1024)
255
- if not chunk:
256
- break
257
- size += len(chunk)
258
- if size > MAX_CUSTOM_LORA_BYTES:
259
- raise ValueError("That LoRA is larger than this Space's 8 GB library limit.")
260
- output.write(chunk)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
261
  if size == 0:
262
  raise ValueError("The LoRA endpoint returned an empty file.")
263
  # Validate the safetensors header without materializing the weights.
 
18
  import hashlib
19
  import tempfile
20
  import urllib.request
21
+ import requests
22
  import safetensors.torch
23
  from huggingface_hub import hf_hub_download
24
 
 
31
  LORA_LIBRARY_DIR = os.environ.get("LORA_LIBRARY_DIR", "/data/loras")
32
  LORA_LIBRARY_MANIFEST = os.path.join(LORA_LIBRARY_DIR, "library.json")
33
  MAX_CUSTOM_LORA_BYTES = int(os.environ.get("MAX_CUSTOM_LORA_BYTES", 8 * 1024**3))
34
+ CIVITAI_HOSTS = ("civitai.com", "civitai.red", "civitai.green", "civitai.work")
35
 
36
  # Pinned commit hashes (2026-07-19) — protection from breaking upstream changes
37
  PINNED_REVISIONS = {
 
220
  return "LOW" if re.search(r"(?:low|[_-]ln\\b|[_-]l\\b)", filename, re.IGNORECASE) else "HIGH"
221
 
222
 
223
+ def _civitai_token():
224
+ return (os.environ.get("CIVITAI_TOKEN") or os.environ.get("CIVITAI_API_KEY") or "").strip()
225
+
226
+
227
+ def _slug(value, fallback):
228
+ value = re.sub(r"[^A-Za-z0-9._-]+", "-", value or "").strip(".-_")
229
+ return (value or fallback)[:120]
230
+
231
+
232
+ def _resolve_civitai_lora(source):
233
+ """Resolve a CivitAI model page or download link to one safetensors file."""
234
+ parsed = urllib.parse.urlparse(source)
235
+ host = (parsed.hostname or "").lower()
236
+ if host not in CIVITAI_HOSTS:
237
+ raise ValueError("This is not a CivitAI URL.")
238
+ if parsed.scheme != "https":
239
+ raise ValueError("Use an HTTPS CivitAI page or download link.")
240
+
241
+ page_match = re.search(r"/models/(\d+)", parsed.path, flags=re.IGNORECASE)
242
+ download_match = re.search(r"/api/download/models/(\d+)", parsed.path, flags=re.IGNORECASE)
243
+ query = dict(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True))
244
+ if download_match:
245
+ version_id = download_match.group(1)
246
+ file_id = query.get("fileId") or "file"
247
+ return source, f"CivitAI LoRA {version_id}", f"civitai-lora--{version_id}--{file_id}.safetensors"
248
+
249
+ version_id = query.get("modelVersionId") or ""
250
+ api_base = f"https://{host}/api/v1"
251
+ if not version_id and page_match:
252
+ model = requests.get(f"{api_base}/models/{page_match.group(1)}", timeout=30).json()
253
+ versions = model.get("modelVersions") or []
254
+ if not versions:
255
+ raise ValueError("CivitAI returned no versions for that model page.")
256
+ version_id = str(versions[0].get("id") or "")
257
+ if not version_id:
258
+ raise ValueError("Could not identify a CivitAI model version from that link.")
259
+
260
+ version = requests.get(f"{api_base}/model-versions/{version_id}", timeout=30).json()
261
+ files = [item for item in (version.get("files") or []) if str(item.get("name") or "").lower().endswith(".safetensors")]
262
+ if not files:
263
+ raise ValueError("That CivitAI version has no safetensors file.")
264
+ chosen = next((item for item in files if item.get("primary")), files[0])
265
+ file_id = chosen.get("id")
266
+ if not file_id:
267
+ raise ValueError("CivitAI did not provide a safetensors file id.")
268
+ model_name = str((version.get("model") or {}).get("name") or "").strip()
269
+ version_name = str(version.get("name") or "").strip()
270
+ display_name = " - ".join(part for part in (model_name, version_name) if part) or str(chosen.get("name") or "CivitAI LoRA")
271
+ filename = f"{_slug(model_name, 'civitai-lora')}--{_slug(version_name, str(version_id))}.safetensors"
272
+ return f"https://{host}/api/download/models/{version_id}?fileId={file_id}", display_name[:160], filename
273
+
274
+
275
  def add_custom_lora(label, url, stage="auto"):
276
  """Fetch one public safetensors URL into the persistent LoRA library.
277
 
 
281
  """
282
  label = (label or "").strip()
283
  url = (url or "").strip()
 
 
 
 
284
  parsed = urllib.parse.urlparse(url)
285
  if parsed.scheme not in {"http", "https"} or not parsed.netloc:
286
+ raise ValueError("Use an http(s) safetensors endpoint or a CivitAI model page.")
287
+
288
+ host = (parsed.hostname or "").lower()
289
+ if host in CIVITAI_HOSTS:
290
+ download_url, detected_label, source_name = _resolve_civitai_lora(url)
291
+ label = label or detected_label
292
+ else:
293
+ download_url = url
294
+ source_name = os.path.basename(urllib.parse.unquote(parsed.path))
295
+ if not source_name.lower().endswith(".safetensors"):
296
+ raise ValueError("Direct endpoints must end in .safetensors.")
297
+ label = label or os.path.splitext(source_name)[0]
298
+ if not re.fullmatch(r"[\\w .()'\\-]{1,96}", label):
299
+ raise ValueError("LoRA names may use letters, numbers, spaces, . ( ) ' and -.")
300
 
301
  os.makedirs(LORA_LIBRARY_DIR, exist_ok=True)
302
  digest = hashlib.sha256(url.encode("utf-8")).hexdigest()[:12]
303
  safe_label = re.sub(r"[^A-Za-z0-9._-]+", "-", label).strip(".-") or "lora"
304
  destination = os.path.join(LORA_LIBRARY_DIR, f"{safe_label}-{digest}.safetensors")
305
  fd, temporary_path = tempfile.mkstemp(prefix="download-", suffix=".part", dir=LORA_LIBRARY_DIR)
306
+ os.close(fd)
307
  size = 0
308
  try:
309
+ download_parsed = urllib.parse.urlparse(download_url)
310
+ token = _civitai_token()
311
+ if token and host in CIVITAI_HOSTS and "token=" not in download_parsed.query:
312
+ query = urllib.parse.parse_qsl(download_parsed.query, keep_blank_values=True) + [("token", token)]
313
+ download_url = urllib.parse.urlunparse(download_parsed._replace(query=urllib.parse.urlencode(query)))
314
+ download_parsed = urllib.parse.urlparse(download_url)
315
+ candidates = [download_url]
316
+ if host in CIVITAI_HOSTS and (download_parsed.hostname or "").lower() != "civitai.red":
317
+ for alternate in CIVITAI_HOSTS:
318
+ if download_parsed.hostname and alternate != download_parsed.hostname.lower():
319
+ candidates.append(download_url.replace(download_parsed.hostname, alternate, 1))
320
+
321
+ last_error = "download failed"
322
+ for candidate in candidates:
323
+ size = 0
324
+ try:
325
+ with requests.get(candidate, stream=True, allow_redirects=True, timeout=(30, 600), headers={"User-Agent": "wan2.2-lora-library/1.0"}) as response:
326
+ response.raise_for_status()
327
+ if "text/html" in response.headers.get("Content-Type", "").lower():
328
+ raise ValueError("The endpoint returned a web page instead of LoRA weights.")
329
+ length = response.headers.get("Content-Length")
330
+ if length and int(length) > MAX_CUSTOM_LORA_BYTES:
331
+ raise ValueError("That LoRA is larger than this Space's 8 GB library limit.")
332
+ with open(temporary_path, "wb") as output:
333
+ for chunk in response.iter_content(chunk_size=4 * 1024 * 1024):
334
+ if not chunk:
335
+ continue
336
+ size += len(chunk)
337
+ if size > MAX_CUSTOM_LORA_BYTES:
338
+ raise ValueError("That LoRA is larger than this Space's 8 GB library limit.")
339
+ output.write(chunk)
340
+ break
341
+ except (OSError, ValueError, requests.RequestException) as exc:
342
+ last_error = str(exc)
343
+ if os.path.exists(temporary_path):
344
+ os.unlink(temporary_path)
345
+ else:
346
+ raise ValueError(last_error)
347
  if size == 0:
348
  raise ValueError("The LoRA endpoint returned an empty file.")
349
  # Validate the safetensors header without materializing the weights.