video-model-factory / scanner.py
Quantumbraid's picture
Build bucket-backed video model factory
a0d2a22 verified
Raw History Blame Contribute Delete
4.22 kB
from __future__ import annotations
import re
from dataclasses import dataclass, field
from huggingface_hub import HfApi, hf_hub_download
from factory_config import render_factory
WEIGHT = re.compile(r"[\w./-]+\.(?:safetensors|gguf|ckpt|bin)", re.I)
CALL_PATTERNS = [
re.compile(r"(?:from_pretrained|snapshot_download)\(\s*['\"]([^'\"]+/[^'\"]+)['\"]"),
re.compile(r"hf_hub_download\([^)]*?repo_id\s*=\s*['\"]([^'\"]+/[^'\"]+)['\"]", re.S),
]
NAMED_REPO = re.compile(
r"['\"]?(?P<name>[A-Za-z_]\w*)['\"]?\s*[:=]\s*['\"](?P<repo>[\w.-]+/[\w.-]+)['\"]",
re.I,
)
@dataclass
class ScanResult:
models: list[str] = field(default_factory=list)
loras: list[str] = field(default_factory=list)
evidence: list[str] = field(default_factory=list)
@property
def factory(self) -> str:
return render_factory(self.models, self.loras)
def _classify(source: str, text: str, start: int) -> str:
context = text[max(0, start - 180): start + len(source) + 180].lower()
return "lora" if any(word in context for word in ("lora", "adapter", "lycoris")) else "model"
def _valid_repo_id(source: str) -> bool:
if not re.fullmatch(r"[\w.-]+/[\w.-]+", source):
return False
lowered = source.lower()
return not lowered.startswith((".", "tmp/", "var/", "app/", "github.com/", "huggingface.co/")) and not lowered.endswith(
(".py", ".json", ".safetensors", ".gguf", ".bin", ".md")
)
def scan_text(text: str) -> ScanResult:
result = ScanResult()
seen: set[tuple[str, str]] = set()
for pattern in CALL_PATTERNS:
for match in pattern.finditer(text):
source = match.group(1)
if not _valid_repo_id(source):
continue
kind = _classify(source, text, match.start())
seen.add((kind, source))
result.evidence.append(f"{kind}: {source} (loader call)")
for match in NAMED_REPO.finditer(text):
source = match.group("repo")
if not _valid_repo_id(source):
continue
name = match.group("name").lower()
if not any(word in name for word in ("model", "lora", "adapter", "checkpoint", "ckpt", "repo")):
continue
kind = _classify(source, text, match.start()) if name in {"repo", "repo_id"} else (
"lora" if any(word in name for word in ("lora", "adapter")) else "model"
)
seen.add((kind, source))
result.evidence.append(f"{kind}: {source} ({match.group('name')})")
for kind, source in seen:
(result.loras if kind == "lora" else result.models).append(source)
result.models.sort()
result.loras.sort()
for match in WEIGHT.finditer(text):
kind = _classify(match.group(0), text, match.start())
result.evidence.append(f"{kind} file: {match.group(0)}")
return result
def scan_hub_repo(repo_id: str) -> ScanResult:
repo_id = repo_id.strip().removeprefix("https://huggingface.co/spaces/").removeprefix("https://huggingface.co/")
api = HfApi()
repo_type = "space"
try:
info = api.space_info(repo_id, files_metadata=False)
except Exception:
repo_type = "model"
info = api.model_info(repo_id, files_metadata=False)
files = {item.rfilename for item in info.siblings}
candidates = [name for name in ("app.py", "README.md", "config.json", "model_index.json") if name in files]
if repo_type == "space":
# app.py frequently delegates the real model configuration to src/*.py.
candidates.extend(sorted(name for name in files if name.endswith(".py") and name != "app.py")[:24])
combined = []
for name in dict.fromkeys(candidates):
path = hf_hub_download(repo_id, name, repo_type=repo_type, token=None)
combined.append(f"\n# FILE: {name}\n" + open(path, encoding="utf-8", errors="ignore").read())
result = scan_text("\n".join(combined))
if repo_type == "model" and repo_id not in result.models:
result.models.insert(0, repo_id)
result.evidence.insert(0, f"model: {repo_id} (scanned model repository)")
if not combined:
result.evidence.append("No readable app.py, README, or model config was found.")
return result