video-model-factory / storage.py
Quantumbraid's picture
Repair mislabeled Diffusers scheduler configs
e6840bd verified
Raw History Blame Contribute Delete
7.85 kB
from __future__ import annotations
import hashlib
import json
import os
import re
import shutil
import threading
import time
from pathlib import Path
from urllib.parse import unquote, urlparse
import requests
from huggingface_hub import snapshot_download
from factory_config import FactoryConfig, read_factory
DATA_ROOT = Path(os.getenv("VIDEO_FACTORY_DATA", "/data"))
if not DATA_ROOT.exists():
DATA_ROOT = Path(__file__).with_name("data")
MODELS_ROOT = DATA_ROOT / "models"
LORAS_ROOT = DATA_ROOT / "loras"
OUTPUTS_ROOT = DATA_ROOT / "outputs"
MANIFEST_PATH = DATA_ROOT / "factory-manifest.json"
IMPORTS_PATH = DATA_ROOT / "imported-factory.json"
_lock = threading.RLock()
_status: dict[str, str] = {}
_worker: threading.Thread | None = None
def _slug(source: str) -> str:
name = re.sub(r"[^A-Za-z0-9._-]+", "--", source).strip(".-")[:90] or "artifact"
return f"{name}-{hashlib.sha256(source.encode()).hexdigest()[:10]}"
def _manifest() -> dict:
try:
return json.loads(MANIFEST_PATH.read_text(encoding="utf-8"))
except (FileNotFoundError, json.JSONDecodeError):
return {"artifacts": {}}
def _save_manifest(data: dict) -> None:
DATA_ROOT.mkdir(parents=True, exist_ok=True)
temp = MANIFEST_PATH.with_suffix(".tmp")
temp.write_text(json.dumps(data, indent=2, sort_keys=True), encoding="utf-8")
temp.replace(MANIFEST_PATH)
def imported_factory() -> FactoryConfig:
try:
data = json.loads(IMPORTS_PATH.read_text(encoding="utf-8"))
except (FileNotFoundError, json.JSONDecodeError):
return FactoryConfig(models=(), loras=())
return FactoryConfig(
models=tuple(item for item in data.get("models", []) if isinstance(item, str) and item.strip()),
loras=tuple(item for item in data.get("loras", []) if isinstance(item, str) and item.strip()),
)
def effective_factory() -> FactoryConfig:
base = read_factory()
imported = imported_factory()
return FactoryConfig(
models=tuple(dict.fromkeys((*base.models, *imported.models))),
loras=tuple(dict.fromkeys((*base.loras, *imported.loras))),
)
def register_imports(models_csv: str, loras_csv: str) -> str:
models = [item.strip() for item in models_csv.split(",") if item.strip()]
loras = [item.strip() for item in loras_csv.split(",") if item.strip()]
current = imported_factory()
merged = FactoryConfig(
models=tuple(dict.fromkeys((*current.models, *models))),
loras=tuple(dict.fromkeys((*current.loras, *loras))),
)
DATA_ROOT.mkdir(parents=True, exist_ok=True)
temp = IMPORTS_PATH.with_suffix(".tmp")
temp.write_text(json.dumps({"models": merged.models, "loras": merged.loras}, indent=2), encoding="utf-8")
temp.replace(IMPORTS_PATH)
requested = FactoryConfig(models=tuple(models), loras=tuple(loras))
threading.Thread(target=prepare_factory, args=(requested,), name="factory-import", daemon=True).start()
return f"Queued {len(models)} model(s) and {len(loras)} LoRA source(s) into the bucket."
def _download_url(source: str, root: Path) -> Path:
parsed = urlparse(source)
filename = Path(unquote(parsed.path)).name
if not filename or filename in {".", ".."}:
raise ValueError(f"URL has no filename: {source}")
target_dir = root / _slug(source)
target_dir.mkdir(parents=True, exist_ok=True)
target = target_dir / filename
if target.exists() and target.stat().st_size:
return target
partial = target.with_suffix(target.suffix + ".partial")
with requests.get(source, stream=True, timeout=(20, 120), allow_redirects=True) as response:
response.raise_for_status()
with partial.open("wb") as handle:
for chunk in response.iter_content(8 * 1024 * 1024):
if chunk:
handle.write(chunk)
partial.replace(target)
return target
def _split_repo_file(source: str) -> tuple[str, str | None]:
if "::" not in source:
return source, None
repo_id, filename = source.split("::", 1)
return repo_id.strip(), filename.strip() or None
def _repair_diffusers_layout(path: Path) -> None:
"""Repair a common community-repo typo without mutating upstream repositories."""
if not path.is_dir():
return
scheduler = path / "scheduler"
expected = scheduler / "scheduler_config.json"
mislabeled = scheduler / "train.txt"
if not expected.exists() and mislabeled.exists():
try:
json.loads(mislabeled.read_text(encoding="utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
return
shutil.copyfile(mislabeled, expected)
def ensure_artifact(source: str, kind: str) -> Path:
root = MODELS_ROOT if kind == "model" else LORAS_ROOT
root.mkdir(parents=True, exist_ok=True)
with _lock:
manifest = _manifest()
old = manifest["artifacts"].get(source)
if old and Path(old["path"]).exists() and old.get("complete"):
_repair_diffusers_layout(Path(old["path"]))
_status[source] = "ready (bucket cache)"
return Path(old["path"])
_status[source] = "downloading"
try:
if source.startswith(("http://", "https://")):
path = _download_url(source, root)
else:
repo_id, filename = _split_repo_file(source)
target = root / _slug(source)
patterns = [filename] if filename else None
snapshot_download(
repo_id=repo_id,
repo_type="model",
local_dir=target,
allow_patterns=patterns,
token=os.getenv("HF_TOKEN"),
)
path = target / filename if filename else target
_repair_diffusers_layout(path)
with _lock:
manifest = _manifest()
manifest["artifacts"][source] = {
"kind": kind,
"path": str(path),
"complete": True,
"updated_at": int(time.time()),
}
_save_manifest(manifest)
_status[source] = "ready"
return path
except Exception as exc:
with _lock:
_status[source] = f"error: {type(exc).__name__}: {exc}"
raise
def prepare_factory(config: FactoryConfig | None = None) -> None:
config = config or effective_factory()
for source in config.models:
try:
ensure_artifact(source, "model")
except Exception:
continue
for source in config.loras:
try:
ensure_artifact(source, "lora")
except Exception:
continue
def start_prepare_background() -> str:
global _worker
with _lock:
if _worker and _worker.is_alive():
return "Factory downloads already running."
_worker = threading.Thread(target=prepare_factory, name="factory-downloads", daemon=True)
_worker.start()
return "Factory downloads started in the background."
def artifact_path(source: str, kind: str) -> Path:
return ensure_artifact(source, kind)
def lora_library() -> list[str]:
if not LORAS_ROOT.exists():
return []
extensions = {".safetensors", ".bin", ".pt", ".gguf"}
return sorted(str(path) for path in LORAS_ROOT.rglob("*") if path.is_file() and path.suffix.lower() in extensions)
def status_text() -> str:
config = effective_factory()
lines = [f"Bucket root: `{DATA_ROOT}`", "", "Models:"]
lines.extend(f"- `{item}` — {_status.get(item, 'waiting')}" for item in config.models)
if not config.models:
lines.append("- none")
lines.append("\nLoRAs:")
lines.extend(f"- `{item}` — {_status.get(item, 'waiting')}" for item in config.loras)
if not config.loras:
lines.append("- none")
return "\n".join(lines)