from __future__ import annotations import hashlib import json import os import re import shutil import threading import time from pathlib import Path from urllib.parse import unquote, urlparse import requests from huggingface_hub import snapshot_download from factory_config import FactoryConfig, read_factory DATA_ROOT = Path(os.getenv("VIDEO_FACTORY_DATA", "/data")) if not DATA_ROOT.exists(): DATA_ROOT = Path(__file__).with_name("data") MODELS_ROOT = DATA_ROOT / "models" LORAS_ROOT = DATA_ROOT / "loras" OUTPUTS_ROOT = DATA_ROOT / "outputs" MANIFEST_PATH = DATA_ROOT / "factory-manifest.json" IMPORTS_PATH = DATA_ROOT / "imported-factory.json" _lock = threading.RLock() _status: dict[str, str] = {} _worker: threading.Thread | None = None def _slug(source: str) -> str: name = re.sub(r"[^A-Za-z0-9._-]+", "--", source).strip(".-")[:90] or "artifact" return f"{name}-{hashlib.sha256(source.encode()).hexdigest()[:10]}" def _manifest() -> dict: try: return json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError): return {"artifacts": {}} def _save_manifest(data: dict) -> None: DATA_ROOT.mkdir(parents=True, exist_ok=True) temp = MANIFEST_PATH.with_suffix(".tmp") temp.write_text(json.dumps(data, indent=2, sort_keys=True), encoding="utf-8") temp.replace(MANIFEST_PATH) def imported_factory() -> FactoryConfig: try: data = json.loads(IMPORTS_PATH.read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError): return FactoryConfig(models=(), loras=()) return FactoryConfig( models=tuple(item for item in data.get("models", []) if isinstance(item, str) and item.strip()), loras=tuple(item for item in data.get("loras", []) if isinstance(item, str) and item.strip()), ) def effective_factory() -> FactoryConfig: base = read_factory() imported = imported_factory() return FactoryConfig( models=tuple(dict.fromkeys((*base.models, *imported.models))), loras=tuple(dict.fromkeys((*base.loras, *imported.loras))), ) def register_imports(models_csv: str, loras_csv: str) -> str: models = [item.strip() for item in models_csv.split(",") if item.strip()] loras = [item.strip() for item in loras_csv.split(",") if item.strip()] current = imported_factory() merged = FactoryConfig( models=tuple(dict.fromkeys((*current.models, *models))), loras=tuple(dict.fromkeys((*current.loras, *loras))), ) DATA_ROOT.mkdir(parents=True, exist_ok=True) temp = IMPORTS_PATH.with_suffix(".tmp") temp.write_text(json.dumps({"models": merged.models, "loras": merged.loras}, indent=2), encoding="utf-8") temp.replace(IMPORTS_PATH) requested = FactoryConfig(models=tuple(models), loras=tuple(loras)) threading.Thread(target=prepare_factory, args=(requested,), name="factory-import", daemon=True).start() return f"Queued {len(models)} model(s) and {len(loras)} LoRA source(s) into the bucket." def _download_url(source: str, root: Path) -> Path: parsed = urlparse(source) filename = Path(unquote(parsed.path)).name if not filename or filename in {".", ".."}: raise ValueError(f"URL has no filename: {source}") target_dir = root / _slug(source) target_dir.mkdir(parents=True, exist_ok=True) target = target_dir / filename if target.exists() and target.stat().st_size: return target partial = target.with_suffix(target.suffix + ".partial") with requests.get(source, stream=True, timeout=(20, 120), allow_redirects=True) as response: response.raise_for_status() with partial.open("wb") as handle: for chunk in response.iter_content(8 * 1024 * 1024): if chunk: handle.write(chunk) partial.replace(target) return target def _split_repo_file(source: str) -> tuple[str, str | None]: if "::" not in source: return source, None repo_id, filename = source.split("::", 1) return repo_id.strip(), filename.strip() or None def _repair_diffusers_layout(path: Path) -> None: """Repair a common community-repo typo without mutating upstream repositories.""" if not path.is_dir(): return scheduler = path / "scheduler" expected = scheduler / "scheduler_config.json" mislabeled = scheduler / "train.txt" if not expected.exists() and mislabeled.exists(): try: json.loads(mislabeled.read_text(encoding="utf-8")) except (UnicodeDecodeError, json.JSONDecodeError): return shutil.copyfile(mislabeled, expected) def ensure_artifact(source: str, kind: str) -> Path: root = MODELS_ROOT if kind == "model" else LORAS_ROOT root.mkdir(parents=True, exist_ok=True) with _lock: manifest = _manifest() old = manifest["artifacts"].get(source) if old and Path(old["path"]).exists() and old.get("complete"): _repair_diffusers_layout(Path(old["path"])) _status[source] = "ready (bucket cache)" return Path(old["path"]) _status[source] = "downloading" try: if source.startswith(("http://", "https://")): path = _download_url(source, root) else: repo_id, filename = _split_repo_file(source) target = root / _slug(source) patterns = [filename] if filename else None snapshot_download( repo_id=repo_id, repo_type="model", local_dir=target, allow_patterns=patterns, token=os.getenv("HF_TOKEN"), ) path = target / filename if filename else target _repair_diffusers_layout(path) with _lock: manifest = _manifest() manifest["artifacts"][source] = { "kind": kind, "path": str(path), "complete": True, "updated_at": int(time.time()), } _save_manifest(manifest) _status[source] = "ready" return path except Exception as exc: with _lock: _status[source] = f"error: {type(exc).__name__}: {exc}" raise def prepare_factory(config: FactoryConfig | None = None) -> None: config = config or effective_factory() for source in config.models: try: ensure_artifact(source, "model") except Exception: continue for source in config.loras: try: ensure_artifact(source, "lora") except Exception: continue def start_prepare_background() -> str: global _worker with _lock: if _worker and _worker.is_alive(): return "Factory downloads already running." _worker = threading.Thread(target=prepare_factory, name="factory-downloads", daemon=True) _worker.start() return "Factory downloads started in the background." def artifact_path(source: str, kind: str) -> Path: return ensure_artifact(source, kind) def lora_library() -> list[str]: if not LORAS_ROOT.exists(): return [] extensions = {".safetensors", ".bin", ".pt", ".gguf"} return sorted(str(path) for path in LORAS_ROOT.rglob("*") if path.is_file() and path.suffix.lower() in extensions) def status_text() -> str: config = effective_factory() lines = [f"Bucket root: `{DATA_ROOT}`", "", "Models:"] lines.extend(f"- `{item}` — {_status.get(item, 'waiting')}" for item in config.models) if not config.models: lines.append("- none") lines.append("\nLoRAs:") lines.extend(f"- `{item}` — {_status.get(item, 'waiting')}" for item in config.loras) if not config.loras: lines.append("- none") return "\n".join(lines)