"""Hugging Face 儲存庫同步:commit 並上傳程式碼與量測結果到 EPM model repo。 用法: # commit + 上傳(需要 HF_TOKEN,或 ~/.cache/huggingface/token) python -m sdq.hf_sync push -m "說明這次改了什麼" # 只看本地狀態(給接手的人 / AI) python -m sdq.hf_sync status # 只下載 HF 上的 code(給 demo.py 用) python -m sdq.hf_sync pull --dest /path/to/dir # 從 patch 重新產生 patches/0001-sdq-expert-pager.patch python -m sdq.hf_sync regen-patch --src /path/to/llama.cpp `push` 會尊重 `.gitignore`:模型權重、建置產物與 `tools/epq_lock.so` (由 `tools/strict8g.py` 在執行時編譯)不會被上傳。 """ from __future__ import annotations import argparse import os import subprocess from pathlib import Path REPO_DIR = Path(__file__).resolve().parent.parent DEFAULT_HF_REPO = os.environ.get( "EPM_HF_REPO", os.environ.get("SDQ_HF_REPO", "HelloSun/minicpm47woek"), ) # demo.py fetch_code() 下載的檔案類型。新增檔案時這裡也要跟著改,否則 # 使用者從 HF 拉下來的 code 會缺東西。 CODE_PATTERNS = [ "*.py", "*.md", "*.c", "cpp/*", "docs/*", "patches/*", "tools/*", "sdq/*.py", "bench/expert_index.json", ] def git(*args: str) -> str: return subprocess.run( ["git", *args], cwd=REPO_DIR, check=True, capture_output=True, text=True ).stdout.strip() def _git_optional(*args: str) -> str: r = subprocess.run(["git", *args], cwd=REPO_DIR, capture_output=True, text=True) return r.stdout.strip() if r.returncode == 0 else "" def _token() -> str | None: token = os.environ.get("HF_TOKEN") if token: return token p = Path.home() / ".cache" / "huggingface" / "token" return p.read_text().strip() if p.exists() else None def push(message: str, hf_repo: str = DEFAULT_HF_REPO) -> None: git("add", "-A") if not git("diff", "--cached", "--name-only"): print("[sync] 沒有變更,略過 commit") else: git("commit", "-m", message) print(f"[sync] 已 commit:{message}") from huggingface_hub import HfApi api = HfApi(token=_token()) try: api.create_repo(hf_repo, repo_type="model", exist_ok=True) except Exception as exc: print(f"[sync] create_repo: {exc}") ignore = REPO_DIR / ".gitignore" patterns = ([ln.strip() for ln in ignore.read_text().splitlines() if ln.strip()] if ignore.exists() else []) api.upload_folder( folder_path=str(REPO_DIR), repo_id=hf_repo, repo_type="model", ignore_patterns=patterns or None, commit_message=message, ) print(f"[sync] 已上傳到 https://huggingface.co/{hf_repo}") def status() -> None: print(f"[sync] repo:{REPO_DIR}") print(f"[sync] 遠端:{_git_optional('remote', '-v').splitlines()[:1]}") log = _git_optional("log", "--oneline", "-10") print(f"\n最近提交:\n{log or '(無)'}") print(f"\n工作區狀態:\n{_git_optional('status', '--short') or '(乾淨)'}") docs = sorted(p.name for p in (REPO_DIR / "docs").glob("*.md")) print(f"\ndocs/:{', '.join(docs) if docs else '(空)'}") def pull(dest: str) -> None: from huggingface_hub import snapshot_download path = snapshot_download( repo_id=DEFAULT_HF_REPO, repo_type="model", local_dir=dest, token=_token(), allow_patterns=CODE_PATTERNS, ) print(f"[sync] 已下載 code 到 {path}") def regen_patch(src_dir: str) -> None: """把 cpp/ 的引擎檔重新產生成 llama.cpp 可套用的 patch。 `patches/` 裡的 patch 是給 `demo.py` 從零重建 llama.cpp 用的。改了 `cpp/*.cpp` 之後一定要重跑這個指令,否則從 HF 下載的使用者會拿到舊引擎。 """ from sdq.config import PATCH_FILE, SDQ_SOURCE_FILES src = Path(src_dir).resolve() if not (src / ".git").exists(): raise SystemExit(f"{src} 不是 git 工作目錄") if not (src / "src").exists(): raise SystemExit(f"{src}/src 不存在") for f in SDQ_SOURCE_FILES: cur = REPO_DIR / "cpp" / f dst = src / "src" / f if cur.exists() and dst.exists() and cur.read_bytes() != dst.read_bytes(): dst.write_bytes(cur.read_bytes()) print(f"[sync] 已同步 cpp/{f} → {src}/src/{f}") out = REPO_DIR / PATCH_FILE # `git diff` does not see untracked files. The engine sources are new files in # the llama.cpp tree, so mark them intent-to-add first -- without this the # patch would carry only the small edits to CMakeLists/llama-graph/llama-model # and a fresh clone would build without any of the pager code. subprocess.run(["git", "add", "-N", *[f"src/{f}" for f in SDQ_SOURCE_FILES]], cwd=src, check=True, capture_output=True) diff = subprocess.run( ["git", "diff", "--binary", "--", "src/CMakeLists.txt", "src/llama-graph.cpp", "src/llama-model.cpp", *[f"src/{f}" for f in SDQ_SOURCE_FILES]], cwd=src, capture_output=True, text=True, check=True, ).stdout if not diff.strip(): raise SystemExit("diff 是空的:llama.cpp 樹裡已經沒有 EPM 的改動?") out.write_text(diff, encoding="utf-8") print(f"[sync] 已重新產生 {out}({len(diff)} bytes)") def main() -> None: ap = argparse.ArgumentParser(description="Hugging Face 同步") sub = ap.add_subparsers(dest="cmd", required=True) p_push = sub.add_parser("push", help="commit 並上傳 HF") p_push.add_argument("-m", "--message", default=None) p_push.add_argument("--repo", default=DEFAULT_HF_REPO) sub.add_parser("status", help="印出目前狀態") p_pull = sub.add_parser("pull", help="下載 HF 上的 code") p_pull.add_argument("--dest", required=True) p_patch = sub.add_parser("regen-patch", help="從 cpp/ 重新產生 llama.cpp patch") p_patch.add_argument("--src", required=True, help="已套用 patch 的 llama.cpp 原始碼目錄") args = ap.parse_args() if args.cmd == "push": push(args.message or "update", args.repo) elif args.cmd == "status": status() elif args.cmd == "pull": pull(args.dest) elif args.cmd == "regen-patch": regen_patch(args.src) if __name__ == "__main__": main()