Download storage.py from Quantumbraid/video-model-factory: direct link, hf CLI and curl.
- Browser
- Download file 7.85 kB
-
https://huggingface.co/spaces/Quantumbraid/video-model-factory/resolve/main/storage.py
- Command line
-
hf download hf://spaces/Quantumbraid/video-model-factory/storage.py
-
curl -L -o storage.py https://huggingface.co/spaces/Quantumbraid/video-model-factory/resolve/main/storage.py
7.85 kB
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import os | |
| import re | |
| import shutil | |
| import threading | |
| import time | |
| from pathlib import Path | |
| from urllib.parse import unquote, urlparse | |
| import requests | |
| from huggingface_hub import snapshot_download | |
| from factory_config import FactoryConfig, read_factory | |
| DATA_ROOT = Path(os.getenv("VIDEO_FACTORY_DATA", "/data")) | |
| if not DATA_ROOT.exists(): | |
| DATA_ROOT = Path(__file__).with_name("data") | |
| MODELS_ROOT = DATA_ROOT / "models" | |
| LORAS_ROOT = DATA_ROOT / "loras" | |
| OUTPUTS_ROOT = DATA_ROOT / "outputs" | |
| MANIFEST_PATH = DATA_ROOT / "factory-manifest.json" | |
| IMPORTS_PATH = DATA_ROOT / "imported-factory.json" | |
| _lock = threading.RLock() | |
| _status: dict[str, str] = {} | |
| _worker: threading.Thread | None = None | |
| def _slug(source: str) -> str: | |
| name = re.sub(r"[^A-Za-z0-9._-]+", "--", source).strip(".-")[:90] or "artifact" | |
| return f"{name}-{hashlib.sha256(source.encode()).hexdigest()[:10]}" | |
| def _manifest() -> dict: | |
| try: | |
| return json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) | |
| except (FileNotFoundError, json.JSONDecodeError): | |
| return {"artifacts": {}} | |
| def _save_manifest(data: dict) -> None: | |
| DATA_ROOT.mkdir(parents=True, exist_ok=True) | |
| temp = MANIFEST_PATH.with_suffix(".tmp") | |
| temp.write_text(json.dumps(data, indent=2, sort_keys=True), encoding="utf-8") | |
| temp.replace(MANIFEST_PATH) | |
| def imported_factory() -> FactoryConfig: | |
| try: | |
| data = json.loads(IMPORTS_PATH.read_text(encoding="utf-8")) | |
| except (FileNotFoundError, json.JSONDecodeError): | |
| return FactoryConfig(models=(), loras=()) | |
| return FactoryConfig( | |
| models=tuple(item for item in data.get("models", []) if isinstance(item, str) and item.strip()), | |
| loras=tuple(item for item in data.get("loras", []) if isinstance(item, str) and item.strip()), | |
| ) | |
| def effective_factory() -> FactoryConfig: | |
| base = read_factory() | |
| imported = imported_factory() | |
| return FactoryConfig( | |
| models=tuple(dict.fromkeys((*base.models, *imported.models))), | |
| loras=tuple(dict.fromkeys((*base.loras, *imported.loras))), | |
| ) | |
| def register_imports(models_csv: str, loras_csv: str) -> str: | |
| models = [item.strip() for item in models_csv.split(",") if item.strip()] | |
| loras = [item.strip() for item in loras_csv.split(",") if item.strip()] | |
| current = imported_factory() | |
| merged = FactoryConfig( | |
| models=tuple(dict.fromkeys((*current.models, *models))), | |
| loras=tuple(dict.fromkeys((*current.loras, *loras))), | |
| ) | |
| DATA_ROOT.mkdir(parents=True, exist_ok=True) | |
| temp = IMPORTS_PATH.with_suffix(".tmp") | |
| temp.write_text(json.dumps({"models": merged.models, "loras": merged.loras}, indent=2), encoding="utf-8") | |
| temp.replace(IMPORTS_PATH) | |
| requested = FactoryConfig(models=tuple(models), loras=tuple(loras)) | |
| threading.Thread(target=prepare_factory, args=(requested,), name="factory-import", daemon=True).start() | |
| return f"Queued {len(models)} model(s) and {len(loras)} LoRA source(s) into the bucket." | |
| def _download_url(source: str, root: Path) -> Path: | |
| parsed = urlparse(source) | |
| filename = Path(unquote(parsed.path)).name | |
| if not filename or filename in {".", ".."}: | |
| raise ValueError(f"URL has no filename: {source}") | |
| target_dir = root / _slug(source) | |
| target_dir.mkdir(parents=True, exist_ok=True) | |
| target = target_dir / filename | |
| if target.exists() and target.stat().st_size: | |
| return target | |
| partial = target.with_suffix(target.suffix + ".partial") | |
| with requests.get(source, stream=True, timeout=(20, 120), allow_redirects=True) as response: | |
| response.raise_for_status() | |
| with partial.open("wb") as handle: | |
| for chunk in response.iter_content(8 * 1024 * 1024): | |
| if chunk: | |
| handle.write(chunk) | |
| partial.replace(target) | |
| return target | |
| def _split_repo_file(source: str) -> tuple[str, str | None]: | |
| if "::" not in source: | |
| return source, None | |
| repo_id, filename = source.split("::", 1) | |
| return repo_id.strip(), filename.strip() or None | |
| def _repair_diffusers_layout(path: Path) -> None: | |
| """Repair a common community-repo typo without mutating upstream repositories.""" | |
| if not path.is_dir(): | |
| return | |
| scheduler = path / "scheduler" | |
| expected = scheduler / "scheduler_config.json" | |
| mislabeled = scheduler / "train.txt" | |
| if not expected.exists() and mislabeled.exists(): | |
| try: | |
| json.loads(mislabeled.read_text(encoding="utf-8")) | |
| except (UnicodeDecodeError, json.JSONDecodeError): | |
| return | |
| shutil.copyfile(mislabeled, expected) | |
| def ensure_artifact(source: str, kind: str) -> Path: | |
| root = MODELS_ROOT if kind == "model" else LORAS_ROOT | |
| root.mkdir(parents=True, exist_ok=True) | |
| with _lock: | |
| manifest = _manifest() | |
| old = manifest["artifacts"].get(source) | |
| if old and Path(old["path"]).exists() and old.get("complete"): | |
| _repair_diffusers_layout(Path(old["path"])) | |
| _status[source] = "ready (bucket cache)" | |
| return Path(old["path"]) | |
| _status[source] = "downloading" | |
| try: | |
| if source.startswith(("http://", "https://")): | |
| path = _download_url(source, root) | |
| else: | |
| repo_id, filename = _split_repo_file(source) | |
| target = root / _slug(source) | |
| patterns = [filename] if filename else None | |
| snapshot_download( | |
| repo_id=repo_id, | |
| repo_type="model", | |
| local_dir=target, | |
| allow_patterns=patterns, | |
| token=os.getenv("HF_TOKEN"), | |
| ) | |
| path = target / filename if filename else target | |
| _repair_diffusers_layout(path) | |
| with _lock: | |
| manifest = _manifest() | |
| manifest["artifacts"][source] = { | |
| "kind": kind, | |
| "path": str(path), | |
| "complete": True, | |
| "updated_at": int(time.time()), | |
| } | |
| _save_manifest(manifest) | |
| _status[source] = "ready" | |
| return path | |
| except Exception as exc: | |
| with _lock: | |
| _status[source] = f"error: {type(exc).__name__}: {exc}" | |
| raise | |
| def prepare_factory(config: FactoryConfig | None = None) -> None: | |
| config = config or effective_factory() | |
| for source in config.models: | |
| try: | |
| ensure_artifact(source, "model") | |
| except Exception: | |
| continue | |
| for source in config.loras: | |
| try: | |
| ensure_artifact(source, "lora") | |
| except Exception: | |
| continue | |
| def start_prepare_background() -> str: | |
| global _worker | |
| with _lock: | |
| if _worker and _worker.is_alive(): | |
| return "Factory downloads already running." | |
| _worker = threading.Thread(target=prepare_factory, name="factory-downloads", daemon=True) | |
| _worker.start() | |
| return "Factory downloads started in the background." | |
| def artifact_path(source: str, kind: str) -> Path: | |
| return ensure_artifact(source, kind) | |
| def lora_library() -> list[str]: | |
| if not LORAS_ROOT.exists(): | |
| return [] | |
| extensions = {".safetensors", ".bin", ".pt", ".gguf"} | |
| return sorted(str(path) for path in LORAS_ROOT.rglob("*") if path.is_file() and path.suffix.lower() in extensions) | |
| def status_text() -> str: | |
| config = effective_factory() | |
| lines = [f"Bucket root: `{DATA_ROOT}`", "", "Models:"] | |
| lines.extend(f"- `{item}` — {_status.get(item, 'waiting')}" for item in config.models) | |
| if not config.models: | |
| lines.append("- none") | |
| lines.append("\nLoRAs:") | |
| lines.extend(f"- `{item}` — {_status.get(item, 'waiting')}" for item in config.loras) | |
| if not config.loras: | |
| lines.append("- none") | |
| return "\n".join(lines) | |