"""Instant file scan: inspects a repo's file list and metadata. Downloads no weights. Safe to run on a free CPU Space: only config.json (plain JSON) is ever fetched. """ import json from huggingface_hub import HfApi, hf_hub_download # Formats that can execute code when loaded with pickle-based loaders (torch.load, joblib, ...) PICKLE_EXT = (".bin", ".pt", ".pth", ".ckpt", ".pkl", ".pickle", ".joblib") DATA_EXT = (".safetensors", ".gguf") NON_WEIGHT_BIN = ("training_args.bin",) # pickled, but not weights; still flagged as pickle GRADES = { "A": "Safetensors/GGUF weights only, no custom code required.", "B": "Safe weight format, but the model needs trust_remote_code (runs the author's Python).", "C": "Safe weights present, plus pickle files that can run code if loaded.", "F": "Weights only in pickle format: loading them can execute arbitrary code.", } def _base_models(card): if not card: return [] bm = getattr(card, "base_model", None) or (card.get("base_model") if isinstance(card, dict) else None) if not bm: return [] return [bm] if isinstance(bm, str) else list(bm) def scan_files(repo_id, revision=None, token=None): api = HfApi(token=token) try: info = api.model_info(repo_id, revision=revision, files_metadata=True, securityStatus=True) except TypeError: # older huggingface_hub without securityStatus info = api.model_info(repo_id, revision=revision, files_metadata=True) files = [(s.rfilename, s.size or 0) for s in (info.siblings or [])] names = [f for f, _ in files] pickles = [f for f in names if f.lower().endswith(PICKLE_EXT)] safetensors = [f for f in names if f.lower().endswith(".safetensors")] gguf = [f for f in names if f.lower().endswith(".gguf")] py_files = [f for f in names if f.endswith(".py")] auto_map = None if "config.json" in names: try: with open(hf_hub_download(repo_id, "config.json", revision=revision, token=token)) as fh: auto_map = json.load(fh).get("auto_map") except Exception: auto_map = None needs_remote_code = bool(auto_map) weight_pickles = [f for f in pickles if f.split("/")[-1] not in NON_WEIGHT_BIN] if (safetensors or gguf) and not pickles: grade = "B" if needs_remote_code else "A" elif (safetensors or gguf) and pickles: grade = "C" elif weight_pickles: grade = "F" else: grade = "B" if needs_remote_code else "A" sec = getattr(info, "security_repo_status", None) hf_scan = None if sec: hf_scan = {"scans_done": sec.get("scansDone"), "files_with_issues": [f.get("path") for f in sec.get("filesWithIssues", [])]} card = getattr(info, "card_data", None) return { "repo_id": repo_id, "revision": info.sha, "grade": grade, "grade_meaning": GRADES[grade], "pickle_files": pickles, "safetensors_files": len(safetensors), "gguf_files": len(gguf), "custom_code_files": py_files, "needs_trust_remote_code": needs_remote_code, "declared_base_models": _base_models(card), "license": getattr(card, "license", None) if card else None, "gated": getattr(info, "gated", None), "hf_security_scan": hf_scan, "total_size_gb": round(sum(s for _, s in files) / 1e9, 2), "deep_scan_supported": bool(safetensors) and not needs_remote_code, }