Update models_manager_v8.4.py
Browse files- models_manager_v8.4.py +73 -16
models_manager_v8.4.py
CHANGED
|
@@ -51,15 +51,30 @@ IMG_EXTS = {".jpg", ".jpeg", ".png", ".webp"}
|
|
| 51 |
HTTP_TIMEOUT = 20
|
| 52 |
# ==========================================
|
| 53 |
|
| 54 |
-
|
| 55 |
-
|
|
|
|
| 56 |
IMG_EXTS = {".jpg", ".jpeg", ".png", ".webp"}
|
| 57 |
|
| 58 |
-
def custom_sort_key(
|
|
|
|
| 59 |
parts = file_path.split('/')
|
| 60 |
top_dir = parts[0] if len(parts) > 0 else ""
|
| 61 |
sub_dir = parts[1] if len(parts) > 1 else ""
|
| 62 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
|
| 64 |
def expand_urls(items_dict):
|
| 65 |
"""自动为精简版数据补齐完整的 HF 前缀(仅对不包含 http 的相对路径生效)"""
|
|
@@ -513,12 +528,16 @@ class ModelsManager(tk.Tk):
|
|
| 513 |
self.log_json(f"读取失败: {e}")
|
| 514 |
return
|
| 515 |
|
| 516 |
-
|
| 517 |
-
if not
|
| 518 |
-
|
| 519 |
|
| 520 |
# 🌟 补齐前缀
|
| 521 |
-
expand_urls(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 522 |
|
| 523 |
self.new_paths = set()
|
| 524 |
self.stale_paths = set()
|
|
@@ -534,8 +553,33 @@ class ModelsManager(tk.Tk):
|
|
| 534 |
threading.Thread(target=self._load_remote_worker, args=(token,), daemon=True).start()
|
| 535 |
|
| 536 |
def _load_remote_worker(self, token):
|
|
|
|
| 537 |
try:
|
| 538 |
api = HfApi(token=token) if token else HfApi()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 539 |
t0 = time.time()
|
| 540 |
all_files = list(api.list_repo_files(repo_id=MODEL_REPO_ID, repo_type="model"))
|
| 541 |
remote_files = [f for f in all_files if f.endswith(".safetensors")]
|
|
@@ -570,7 +614,7 @@ class ModelsManager(tk.Tk):
|
|
| 570 |
for p in stale_paths:
|
| 571 |
merged[p] = existing_items[p]
|
| 572 |
|
| 573 |
-
items =
|
| 574 |
self.log_json(f"步骤3: 合并 {len(items)} 条,新增 {len(new_paths)},失效 {len(stale_paths)}")
|
| 575 |
|
| 576 |
self.after(0, lambda: self._apply_remote(items, remote_files, new_paths, stale_paths))
|
|
@@ -580,11 +624,24 @@ class ModelsManager(tk.Tk):
|
|
| 580 |
finally:
|
| 581 |
self.after(0, self._finish_loading)
|
| 582 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 583 |
def _apply_remote(self, items, remote_files, new_paths, stale_paths):
|
| 584 |
# 补齐前缀
|
| 585 |
expand_urls(items)
|
| 586 |
|
| 587 |
-
|
|
|
|
|
|
|
|
|
|
| 588 |
self.remote_files = remote_files
|
| 589 |
self.new_paths = new_paths
|
| 590 |
self.stale_paths = stale_paths
|
|
@@ -763,7 +820,7 @@ class ModelsManager(tk.Tk):
|
|
| 763 |
|
| 764 |
def save_local(self):
|
| 765 |
data_items = {k: v for k, v in self.items.items() if k not in self.stale_paths}
|
| 766 |
-
sorted_items =
|
| 767 |
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
| 768 |
json.dump({"items": sorted_items}, f, indent=2, ensure_ascii=False)
|
| 769 |
self.log_json(f"已保存本地 {OUTPUT_FILE},{len(sorted_items)} 条")
|
|
@@ -775,8 +832,7 @@ class ModelsManager(tk.Tk):
|
|
| 775 |
|
| 776 |
import copy
|
| 777 |
data_items = {k: v for k, v in self.items.items() if k not in self.stale_paths}
|
| 778 |
-
sorted_items =
|
| 779 |
-
|
| 780 |
# 🌟 核心修改:深拷贝一份用于上传
|
| 781 |
upload_items = copy.deepcopy(sorted_items)
|
| 782 |
|
|
@@ -925,7 +981,7 @@ class ModelsManager(tk.Tk):
|
|
| 925 |
"""剥离绝对前缀,生成精简版,绝不污染内存"""
|
| 926 |
import copy
|
| 927 |
data_items = {k: v for k, v in self.items.items() if k not in self.stale_paths}
|
| 928 |
-
sorted_items =
|
| 929 |
|
| 930 |
# 🌟 核心修改:深拷贝一份,只对拷贝体动手
|
| 931 |
upload_items = copy.deepcopy(sorted_items)
|
|
@@ -1272,7 +1328,8 @@ class ModelsManager(tk.Tk):
|
|
| 1272 |
self.log_model(f"已为 {added_count} 个新增模型写入时间戳。")
|
| 1273 |
|
| 1274 |
# 按原有逻辑重新排序
|
| 1275 |
-
sorted_items = {k: latest_remote_items[k] for k in sorted(latest_remote_items.keys(), key=custom_sort_key)}
|
|
|
|
| 1276 |
|
| 1277 |
# ========== 步骤 4:剥离前缀并推送 ==========
|
| 1278 |
if added_count > 0:
|
|
@@ -1340,7 +1397,7 @@ class ModelsManager(tk.Tk):
|
|
| 1340 |
def save_local_silent(self):
|
| 1341 |
"""静默保存本地 JSON,不弹出提示框"""
|
| 1342 |
data_items = {k: v for k, v in self.items.items() if k not in self.stale_paths}
|
| 1343 |
-
sorted_items =
|
| 1344 |
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
| 1345 |
json.dump({"items": sorted_items}, f, indent=2, ensure_ascii=False)
|
| 1346 |
self.log_json(f"已静默保存本地 {OUTPUT_FILE},{len(sorted_items)} 条")
|
|
|
|
| 51 |
HTTP_TIMEOUT = 20
|
| 52 |
# ==========================================
|
| 53 |
|
| 54 |
+
# 🌟 动态加载的排序规则
|
| 55 |
+
TOP_ORDER = {}
|
| 56 |
+
SUB_ORDER = {}
|
| 57 |
IMG_EXTS = {".jpg", ".jpeg", ".png", ".webp"}
|
| 58 |
|
| 59 |
+
def custom_sort_key(item_tuple):
|
| 60 |
+
file_path, meta = item_tuple
|
| 61 |
parts = file_path.split('/')
|
| 62 |
top_dir = parts[0] if len(parts) > 0 else ""
|
| 63 |
sub_dir = parts[1] if len(parts) > 1 else ""
|
| 64 |
+
|
| 65 |
+
ts = 0
|
| 66 |
+
if isinstance(meta, dict):
|
| 67 |
+
try:
|
| 68 |
+
ts = int(meta.get("update_time", 0))
|
| 69 |
+
except (ValueError, TypeError):
|
| 70 |
+
ts = 0
|
| 71 |
+
|
| 72 |
+
return (
|
| 73 |
+
TOP_ORDER.get(top_dir, 999),
|
| 74 |
+
SUB_ORDER.get(sub_dir, 999),
|
| 75 |
+
-ts,
|
| 76 |
+
file_path
|
| 77 |
+
)
|
| 78 |
|
| 79 |
def expand_urls(items_dict):
|
| 80 |
"""自动为精简版数据补齐完整的 HF 前缀(仅对不包含 http 的相对路径生效)"""
|
|
|
|
| 528 |
self.log_json(f"读取失败: {e}")
|
| 529 |
return
|
| 530 |
|
| 531 |
+
items = data.get("items", {})
|
| 532 |
+
if not items and "titles" in data:
|
| 533 |
+
items = {k: {"title": v, "trigger": "", "dataset": ""} for k, v in data["titles"].items()}
|
| 534 |
|
| 535 |
# 🌟 补齐前缀
|
| 536 |
+
expand_urls(items)
|
| 537 |
+
|
| 538 |
+
# 🌟 核心修改:强制按分类 + 时间戳倒序重排
|
| 539 |
+
sorted_items = dict(sorted(items.items(), key=custom_sort_key))
|
| 540 |
+
self.items = sorted_items
|
| 541 |
|
| 542 |
self.new_paths = set()
|
| 543 |
self.stale_paths = set()
|
|
|
|
| 553 |
threading.Thread(target=self._load_remote_worker, args=(token,), daemon=True).start()
|
| 554 |
|
| 555 |
def _load_remote_worker(self, token):
|
| 556 |
+
global TOP_ORDER, SUB_ORDER
|
| 557 |
try:
|
| 558 |
api = HfApi(token=token) if token else HfApi()
|
| 559 |
+
|
| 560 |
+
# 🌟 核心新增:先拉取 site_config.json,动态生成排序规则
|
| 561 |
+
try:
|
| 562 |
+
self.log_json("正在拉取 site_config.json 以同步排序规则...")
|
| 563 |
+
config_path = api.hf_hub_download(repo_id=SPACE_ID, filename=SITE_CONFIG_FILE, repo_type="space")
|
| 564 |
+
with open(config_path, "r", encoding="utf-8") as f:
|
| 565 |
+
site_config = json.load(f)
|
| 566 |
+
|
| 567 |
+
# 根据数组顺序生成 { key: index } 的字典
|
| 568 |
+
top_list = site_config.get("models", [])
|
| 569 |
+
top_dict = {item["key"]: idx + 1 for idx, item in enumerate(top_list) if "key" in item}
|
| 570 |
+
|
| 571 |
+
sub_list = site_config.get("categories", [])
|
| 572 |
+
sub_dict = {item["key"]: idx + 1 for idx, item in enumerate(sub_list) if "key" in item}
|
| 573 |
+
|
| 574 |
+
if top_dict:
|
| 575 |
+
TOP_ORDER = top_dict
|
| 576 |
+
if sub_dict:
|
| 577 |
+
SUB_ORDER = sub_dict
|
| 578 |
+
|
| 579 |
+
self.log_json(f"✅ 排序规则已更新: Models有{len(top_dict)}个, Categories有{len(sub_dict)}个")
|
| 580 |
+
except Exception as e:
|
| 581 |
+
self.log_json(f"⚠️ 拉取 site_config 失败,将使用默认空排序: {e}")
|
| 582 |
+
|
| 583 |
t0 = time.time()
|
| 584 |
all_files = list(api.list_repo_files(repo_id=MODEL_REPO_ID, repo_type="model"))
|
| 585 |
remote_files = [f for f in all_files if f.endswith(".safetensors")]
|
|
|
|
| 614 |
for p in stale_paths:
|
| 615 |
merged[p] = existing_items[p]
|
| 616 |
|
| 617 |
+
items = dict(sorted(merged.items(), key=custom_sort_key))
|
| 618 |
self.log_json(f"步骤3: 合并 {len(items)} 条,新增 {len(new_paths)},失效 {len(stale_paths)}")
|
| 619 |
|
| 620 |
self.after(0, lambda: self._apply_remote(items, remote_files, new_paths, stale_paths))
|
|
|
|
| 624 |
finally:
|
| 625 |
self.after(0, self._finish_loading)
|
| 626 |
|
| 627 |
+
# def _apply_remote(self, items, remote_files, new_paths, stale_paths):
|
| 628 |
+
# # 补齐前缀
|
| 629 |
+
# expand_urls(items)
|
| 630 |
+
|
| 631 |
+
# self.items = items
|
| 632 |
+
# self.remote_files = remote_files
|
| 633 |
+
# self.new_paths = new_paths
|
| 634 |
+
# self.stale_paths = stale_paths
|
| 635 |
+
# self.refresh_tree()
|
| 636 |
+
|
| 637 |
def _apply_remote(self, items, remote_files, new_paths, stale_paths):
|
| 638 |
# 补齐前缀
|
| 639 |
expand_urls(items)
|
| 640 |
|
| 641 |
+
# 🌟 核心修改:强制���分类 + 时间戳倒序重排
|
| 642 |
+
sorted_items = dict(sorted(items.items(), key=custom_sort_key))
|
| 643 |
+
self.items = sorted_items
|
| 644 |
+
|
| 645 |
self.remote_files = remote_files
|
| 646 |
self.new_paths = new_paths
|
| 647 |
self.stale_paths = stale_paths
|
|
|
|
| 820 |
|
| 821 |
def save_local(self):
|
| 822 |
data_items = {k: v for k, v in self.items.items() if k not in self.stale_paths}
|
| 823 |
+
sorted_items = dict(sorted(data_items.items(), key=custom_sort_key))
|
| 824 |
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
| 825 |
json.dump({"items": sorted_items}, f, indent=2, ensure_ascii=False)
|
| 826 |
self.log_json(f"已保存本地 {OUTPUT_FILE},{len(sorted_items)} 条")
|
|
|
|
| 832 |
|
| 833 |
import copy
|
| 834 |
data_items = {k: v for k, v in self.items.items() if k not in self.stale_paths}
|
| 835 |
+
sorted_items = dict(sorted(data_items.items(), key=custom_sort_key))
|
|
|
|
| 836 |
# 🌟 核心修改:深拷贝一份用于上传
|
| 837 |
upload_items = copy.deepcopy(sorted_items)
|
| 838 |
|
|
|
|
| 981 |
"""剥离绝对前缀,生成精简版,绝不污染内存"""
|
| 982 |
import copy
|
| 983 |
data_items = {k: v for k, v in self.items.items() if k not in self.stale_paths}
|
| 984 |
+
sorted_items = dict(sorted(data_items.items(), key=custom_sort_key))
|
| 985 |
|
| 986 |
# 🌟 核心修改:深拷贝一份,只对拷贝体动手
|
| 987 |
upload_items = copy.deepcopy(sorted_items)
|
|
|
|
| 1328 |
self.log_model(f"已为 {added_count} 个新增模型写入时间戳。")
|
| 1329 |
|
| 1330 |
# 按原有逻辑重新排序
|
| 1331 |
+
# sorted_items = {k: latest_remote_items[k] for k in sorted(latest_remote_items.keys(), key=custom_sort_key)}
|
| 1332 |
+
sorted_items = dict(sorted(latest_remote_items.items(), key=custom_sort_key))
|
| 1333 |
|
| 1334 |
# ========== 步骤 4:剥离前缀并推送 ==========
|
| 1335 |
if added_count > 0:
|
|
|
|
| 1397 |
def save_local_silent(self):
|
| 1398 |
"""静默保存本地 JSON,不弹出提示框"""
|
| 1399 |
data_items = {k: v for k, v in self.items.items() if k not in self.stale_paths}
|
| 1400 |
+
sorted_items = dict(sorted(data_items.items(), key=custom_sort_key))
|
| 1401 |
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
| 1402 |
json.dump({"items": sorted_items}, f, indent=2, ensure_ascii=False)
|
| 1403 |
self.log_json(f"已静默保存本地 {OUTPUT_FILE},{len(sorted_items)} 条")
|