import os import sys import threading import time import logging from concurrent.futures import ThreadPoolExecutor, as_completed import tkinter as tk from tkinter import filedialog, messagebox, ttk, simpledialog from huggingface_hub import HfApi # 开启 hf_transfer 加速 os.environ["HF_XET_HIGH_PERFORMANCE"] = "1" # 禁用底层 CLI 的多线程进度条,防止刷屏 os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1" class TextHandler(logging.Handler): def __init__(self, text_widget): super().__init__() self.text_widget = text_widget def emit(self, record): msg = self.format(record) def append(): self.text_widget.configure(state='normal') self.text_widget.insert(tk.END, msg + '\n') self.text_widget.configure(state='disabled') self.text_widget.see(tk.END) self.text_widget.after(0, append) class HFUploaderApp: def __init__(self, root): self.root = root self.root.title("HF 仓库管理 - 大模型批量上传版") self.root.geometry("950x700") self.token_var = tk.StringVar() self.repo_id_var = tk.StringVar(value="waveshaker/comfy_models") self.selected_tree_path = "" self.upload_queue = {} # 控制停止的全局标志位 self.is_uploading = False self.setup_ui() self.load_token() def setup_ui(self): # 1. 顶部 Token 区 top_frame = tk.Frame(self.root, pady=10) top_frame.pack(fill=tk.X, padx=10) tk.Label(top_frame, text="HF Token:").pack(side=tk.LEFT) tk.Entry(top_frame, textvariable=self.token_var, show="*", width=50).pack(side=tk.LEFT, padx=5) tk.Button(top_frame, text="保存 Token", command=self.save_token).pack(side=tk.LEFT) # 2. 仓库配置区 repo_frame = tk.Frame(self.root) repo_frame.pack(fill=tk.X, padx=10, pady=5) tk.Label(repo_frame, text="仓库 ID:").pack(side=tk.LEFT) tk.Entry(repo_frame, textvariable=self.repo_id_var, width=30).pack(side=tk.LEFT, padx=5) # 3. 中间分割(左侧:队列,右侧:远端树) paned = tk.PanedWindow(self.root, orient=tk.HORIZONTAL, sashwidth=5) paned.pack(fill=tk.BOTH, expand=True, padx=10, pady=5) # --- 左侧:上传队列 --- left_frame = tk.Frame(paned) paned.add(left_frame, width=550) tk.Label(left_frame, text="上传队列 (双击目标路径可修改):").pack(anchor=tk.W) # 🌟 修正:列名统一为 source cols = ("source", "remote", "status") self.queue_tree = ttk.Treeview(left_frame, columns=cols, show="headings", height=10) self.queue_tree.heading("source", text="文件源 (本地/远端链接)") self.queue_tree.heading("remote", text="远端目标目录") self.queue_tree.heading("status", text="状态") self.queue_tree.column("source", width=250) self.queue_tree.column("remote", width=150) self.queue_tree.column("status", width=80, anchor="center") self.queue_tree.tag_configure("done", foreground="gray") self.queue_tree.pack(fill=tk.BOTH, expand=True) self.queue_tree.bind("", self.on_edit_remote_path) queue_btns = tk.Frame(left_frame) queue_btns.pack(fill=tk.X, pady=5) tk.Button(queue_btns, text="➕ 添加本地文件", command=self.add_to_queue).pack(side=tk.LEFT) tk.Button(queue_btns, text="❌ 移除选中", command=self.remove_from_queue).pack(side=tk.LEFT, padx=5) # 🌟 极简的链接输入框 self.remote_url_var = tk.StringVar() tk.Entry(queue_btns, textvariable=self.remote_url_var, width=30).pack(side=tk.LEFT, padx=5) tk.Button(queue_btns, text="🔗 加入队列", command=self.add_remote_to_queue, ).pack(side=tk.LEFT) right_frame = tk.Frame(paned) paned.add(right_frame, width=350) tk.Label(right_frame, text="远端仓库目录树:").pack(anchor=tk.W) self.tree_view = ttk.Treeview(right_frame, show="tree", height=10) self.tree_view.pack(fill=tk.BOTH, expand=True) self.tree_view.bind("<>", self.on_tree_select) tree_btns = tk.Frame(right_frame) tree_btns.pack(fill=tk.X, pady=5) tk.Button(tree_btns, text="🔄 拉取远端目录", command=self.fetch_remote_tree).pack(side=tk.LEFT) tk.Button(tree_btns, text="📁 新建文件夹", command=self.create_remote_folder).pack(side=tk.LEFT, padx=5) tk.Button(tree_btns, text="⬇️ 设为目标", command=self.set_target_from_tree).pack(side=tk.LEFT) # 4. 操作区 (包含开始和停止) action_frame = tk.Frame(self.root, pady=10) action_frame.pack(fill=tk.X, padx=10) self.upload_btn = tk.Button(action_frame, text="🚀 开始并发上传", command=self.start_upload, font=("Arial", 10, "bold")) self.upload_btn.pack(side=tk.LEFT, ipadx=20, ipady=5) # 停止按钮 self.stop_btn = tk.Button(action_frame, text="🛑 停止上传", command=self.stop_upload, font=("Arial", 10, "bold"), state=tk.DISABLED) self.stop_btn.pack(side=tk.LEFT, padx=10, ipadx=20, ipady=5) # 🌟 GUI 内部总进度条 self.total_progress_var = tk.DoubleVar() self.progress_bar = ttk.Progressbar(action_frame, orient="horizontal", length=200, mode="determinate", variable=self.total_progress_var) self.progress_bar.pack(side=tk.LEFT, padx=15, fill=tk.X, expand=True) # 进度文字标签 self.progress_text = tk.Label(action_frame, text="等待上传...", fg="#2196F3", font=("Arial", 10, "bold")) self.progress_text.pack(side=tk.LEFT, padx=5) # 5. 日志区 log_frame = tk.Frame(self.root, padx=10, pady=5) log_frame.pack(fill=tk.BOTH, expand=True) tk.Label(log_frame, text="操作日志:").pack(anchor=tk.W) self.log_text = tk.Text(log_frame, height=12, state='disabled', bg="#f0f0f0") self.log_text.pack(fill=tk.BOTH, expand=True) self.logger = logging.getLogger("HF_Uploader") self.logger.setLevel(logging.INFO) handler = TextHandler(self.log_text) handler.setFormatter(logging.Formatter('[%(asctime)s] %(message)s', '%H:%M:%S')) self.logger.addHandler(handler) def load_token(self): if os.path.exists("hf_token.txt"): with open("hf_token.txt", "r") as f: self.token_var.set(f.read().strip()) def save_token(self): with open("hf_token.txt", "w") as f: f.write(self.token_var.get().strip()) messagebox.showinfo("成功", "Token 已保存到 hf_token.txt") def add_to_queue(self): files = filedialog.askopenfilenames( title="选择要上传的模型文件", filetypes=[("SafeTensors Model", "*.safetensors"), ("All Files", "*.*")] ) if files: for file_path in files: if file_path not in self.upload_queue: self.upload_queue[file_path] = "" self.refresh_queue() def remove_from_queue(self): selected = self.queue_tree.selection() for item in selected: local_path = self.queue_tree.item(item, "values")[0] if local_path in self.upload_queue: del self.upload_queue[local_path] self.refresh_queue() def refresh_queue(self): # 🌟 在删除之前,先把旧的“已完成”状态存下来 existing_status = {} for item in self.queue_tree.get_children(): values = self.queue_tree.item(item, "values") if len(values) >= 3 and values[2] == "已完成": existing_status[values[0]] = "已完成" # 清空现有表格 for item in self.queue_tree.get_children(): self.queue_tree.delete(item) # 重新插入所有数据 for local_path, remote_target in self.upload_queue.items(): # 如果之前传完了,状态保持“已完成”,否则是“待上传” status = existing_status.get(local_path, "待上传") if status == "已完成": self.queue_tree.insert("", tk.END, values=(local_path, remote_target, status), tags=("done",)) else: self.queue_tree.insert("", tk.END, values=(local_path, remote_target, status)) # 自动选中最后一个 children = self.queue_tree.get_children() if children: last_item = children[-1] self.queue_tree.selection_set(last_item) self.queue_tree.focus(last_item) self.queue_tree.see(last_item) def on_edit_remote_path(self, event): item = self.queue_tree.selection() if not item: return item = item[0] current_local = self.queue_tree.item(item, "values")[0] current_remote = self.queue_tree.item(item, "values")[1] top = tk.Toplevel(self.root) top.title("修改目标路径") tk.Label(top, text="输入远端目标文件夹 (例如: moss/ 或留空):").pack(padx=10, pady=5) entry = tk.Entry(top, width=40) entry.insert(0, current_remote) entry.pack(padx=10, pady=5) def save(): new_path = entry.get().strip().strip("/") self.upload_queue[current_local] = new_path self.refresh_queue() top.destroy() tk.Button(top, text="确定", command=save).pack(pady=10) def fetch_remote_tree(self): token = self.token_var.get().strip() repo_id = self.repo_id_var.get().strip() if not token or not repo_id: return self.logger.info("正在拉取远端目录结构...") self.tree_view.delete(*self.tree_view.get_children()) def fetch(): try: api = HfApi(token=token) repo_files = api.list_repo_files(repo_id=repo_id, repo_type="model") tree_dict = {} for file_path in repo_files: parts = file_path.split('/') current = tree_dict for part in parts: if part not in current: current[part] = {} current = current[part] def insert_tree(parent, dict_data, current_path=""): for key, children in dict_data.items(): path = f"{current_path}/{key}" if current_path else key node_text = f"📄 {key}" if not children else f"📁 {key}" node = self.tree_view.insert(parent, tk.END, text=node_text, values=(path,)) if children: insert_tree(node, children, path) self.root.after(0, lambda: insert_tree("", tree_dict)) self.logger.info(f"成功加载远端目录结构。") except Exception as e: self.root.after(0, lambda err=e: self.logger.error(f"拉取失败: {err}")) threading.Thread(target=fetch, daemon=True).start() def create_remote_folder(self): token = self.token_var.get().strip() repo_id = self.repo_id_var.get().strip() if not token or not repo_id: return parent_path = self.selected_tree_path if self.selected_tree_path else "" folder_name = simpledialog.askstring("新建文件夹", f"在 '{parent_path or '根目录'}' 下新建文件夹:") if not folder_name: return folder_name = folder_name.strip().strip("/") new_folder_path = f"{parent_path}/{folder_name}" if parent_path else folder_name gitkeep_path = f"{new_folder_path}/.gitkeep" self.logger.info(f"正在新建远端文件夹: {new_folder_path} ...") def create(): try: api = HfApi(token=token) api.upload_file(path_or_fileobj=b"", path_in_repo=gitkeep_path, repo_id=repo_id, repo_type="model") self.root.after(0, lambda: self.logger.info(f"✅ 文件夹新建成功: {new_folder_path}")) self.root.after(0, self.fetch_remote_tree) except Exception as e: self.root.after(0, lambda err=e: self.logger.error(f"❌ 新建失败: {err}")) threading.Thread(target=create, daemon=True).start() def on_tree_select(self, event): selected = self.tree_view.selection() if selected: self.selected_tree_path = self.tree_view.item(selected[0], "values")[0] if "." in os.path.basename(self.selected_tree_path): self.selected_tree_path = os.path.dirname(self.selected_tree_path) def set_target_from_tree(self): if not self.selected_tree_path: messagebox.showwarning("提示", "请先在右侧树中选中一个文件夹") return queue_selected = self.queue_tree.selection() if not queue_selected: messagebox.showwarning("提示", "请先在左侧队列中选中任务") return target_path = self.selected_tree_path.strip("/") count = 0 for item in queue_selected: values = self.queue_tree.item(item, "values") # 🌟 如果已经是“已完成”,就不改它的路径了 if len(values) >= 3 and values[2] == "已完成": continue local_path = values[0] self.upload_queue[local_path] = target_path count += 1 self.refresh_queue() self.logger.info(f"✅ 已将 {count} 个文件的目标路径批量设置为: {target_path}") # === 上传与停止逻辑 === def start_upload(self): token = self.token_var.get().strip() repo_id = self.repo_id_var.get().strip() # 🌟 核心修复:先检查有哪些文件是“待上传”的 pending_tasks = [] for tree_item in self.queue_tree.get_children(): values = self.queue_tree.item(tree_item, "values") status = values[2] if len(values) >= 3 else "待上传" if status != "已完成": pending_tasks.append(tree_item) if not token or not repo_id or not pending_tasks: messagebox.showwarning("警告", "请检查 Token、仓库 ID,或者当前没有待上传的新文件") return self.is_uploading = True self.upload_btn.config(state=tk.DISABLED, text="上传中...") self.stop_btn.config(state=tk.NORMAL) # 计算要上传的真实数量 total_files = len(pending_tasks) self.logger.info(f"🚀 开始并发上传 {total_files} 个待处理文件...") def upload_worker(): api = HfApi(token=token) tasks = [] for tree_item in pending_tasks: values = self.queue_tree.item(tree_item, "values") source = values[0] remote_dir = values[1] # 区分是 HTTP 链接还是本地文件 if source.startswith("http://") or source.startswith("https://"): # 这是一个远端链接,提取文件名 filename = source.split("/")[-1] repo_path = f"{remote_dir}/{filename}" if remote_dir else filename tasks.append((tree_item, source, repo_path, True)) # True 代表是远端 else: # 这是一个本地文件 filename = os.path.basename(source) repo_path = f"{remote_dir}/{filename}" if remote_dir else filename tasks.append((tree_item, source, repo_path, False)) # False 代表是本地 completed_files = 0 progress_lock = threading.Lock() def update_ui_progress(completed, total): percent = (completed / total) * 100 self.root.after(0, lambda: self.total_progress_var.set(percent)) self.root.after(0, lambda: self.progress_text.config(text=f"{completed} / {total} 个文件 ({percent:.1f}%)")) def upload_one(tree_item, source, repo_path, is_remote): nonlocal completed_files if not self.is_uploading: return False, "用户已取消" try: if is_remote: import subprocess import os clean_path = source.replace("https://huggingface.co/", "") parts = clean_path.split("/") if len(parts) >= 5 and parts[2] in ["blob", "resolve"]: src_repo_id = f"{parts[0]}/{parts[1]}" src_file_path = "/".join(parts[4:]) # 这里已经修复过了 # 读取 GUI 里的 token current_token = self.token_var.get().strip() cmd = [ "hf", "cp", f"hf://{src_repo_id}/{src_file_path}", f"hf://{repo_id}/{repo_path}" ] # 🌟 强行把 Token 塞给子进程的环境变量,防止它去读错误的系统缓存 env = os.environ.copy() env["HF_TOKEN"] = current_token result = subprocess.run(cmd, capture_output=True, text=True, env=env) if result.returncode != 0: raise RuntimeError(result.stderr.strip() or result.stdout.strip()) else: # 🌟 本地文件上传逻辑 api.upload_file( path_or_fileobj=source, path_in_repo=repo_path, repo_id=repo_id, repo_type="model" ) with progress_lock: completed_files += 1 update_ui_progress(completed_files, total_files) self.root.after(0, lambda: self.mark_item_done(tree_item)) return True, repo_path except Exception as e: return False, f"{repo_path} ({e})" self.root.after(0, lambda: self.total_progress_var.set(0)) self.root.after(0, lambda: self.progress_text.config(text=f"0 / {total_files} 个文件", fg="#2196F3")) with ThreadPoolExecutor(max_workers=4) as executor: future_to_task = {executor.submit(upload_one, t, s, r, is_r): r for t, s, r, is_r in tasks} for future in as_completed(future_to_task): if not self.is_uploading: break success, msg = future.result() if success: self.root.after(0, lambda m=msg: self.logger.info(f"✅ 成功: {m}")) elif msg != "用户已取消": self.root.after(0, lambda m=msg: self.logger.error(f"❌ 失败: {m}")) self.root.after(0, self.upload_finished) threading.Thread(target=upload_worker, daemon=True).start() def update_ui_progress(self, percent, completed, total): self.total_progress_var.set(percent) self.progress_text.config(text=f"{completed} / {total} 个文件 ({percent:.1f}%)") def mark_item_done(self, tree_item): """将表格中的某一行标记为已完成(灰色)""" values = list(self.queue_tree.item(tree_item, "values")) if len(values) >= 3: values[2] = "已完成" else: values.append("已完成") self.queue_tree.item(tree_item, values=values, tags=("done",)) def stop_upload(self): """强行停止上传""" if messagebox.askyesno("确认停止", "确定要停止上传吗?\n(已传完的部分已保存在服务器,下次点上传会自动断点续传)"): self.is_uploading = False self.logger.warning("🛑 已中断后台请求,界面已重置。等待后台线程安全退出...") self.upload_btn.config(state=tk.NORMAL, text="🚀 开始并发上传") self.stop_btn.config(state=tk.DISABLED) self.progress_text.config(text="已完成 / 已停止", fg="#2E7D32") def upload_finished(self): self.is_uploading = False self.upload_btn.config(state=tk.NORMAL, text="🚀 开始并发上传") self.stop_btn.config(state=tk.DISABLED) self.progress_text.config(text="已完成 / 已停止", fg="#2E7D32") self.logger.info("🎉 任务处理完毕!") messagebox.showinfo("完成", "上传任务已结束(或已停止),请查看日志。") def add_remote_to_queue(self): url = self.remote_url_var.get().strip() if not url: messagebox.showwarning("提示", "请先粘贴 HF 文件链接") return # 自动拼合一个初始的目标路径(比如默认丢到 diffusion_models 下) if url not in self.upload_queue: self.upload_queue[url] = "" # 默认路径留空 self.refresh_queue() self.remote_url_var.set("") # 加入后清空输入框 # 自动把最后一行高亮 children = self.queue_tree.get_children() if children: self.queue_tree.selection_set(children[-1]) if __name__ == "__main__": root = tk.Tk() app = HFUploaderApp(root) root.mainloop()