File size: 2,451 Bytes
dcc24f8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | #!/usr/bin/env python3
"""
Sync model files to HuggingFace.
Run this after pushing code to ensure model files are present.
"""
import os
import sys
from pathlib import Path
try:
from huggingface_hub import HfApi, create_commit, CommitOperationAdd
except ImportError:
print("Installing huggingface_hub...")
os.system("pip install huggingface_hub -q")
from huggingface_hub import HfApi, create_commit, CommitOperationAdd
REPO_ID = "Ranjit0034/finance-entity-extractor"
PROJECT_DIR = Path.home() / "llm-mail-trainer"
# Model files to keep in sync
MODEL_FILES = [
(PROJECT_DIR / "models/base/phi3-mini/config.json", "config.json"),
(PROJECT_DIR / "models/adapters/finance-lora-v2/adapter_config.json", "adapter_config.json"),
(PROJECT_DIR / "models/adapters/finance-lora-v2/adapters.safetensors", "adapters.safetensors"),
]
def check_remote_files(api):
"""Check which model files exist on HuggingFace."""
try:
files = api.list_repo_files(REPO_ID)
return set(files)
except Exception as e:
print(f"Error checking remote: {e}")
return set()
def sync_models():
"""Sync model files to HuggingFace."""
api = HfApi()
print("🔍 Checking HuggingFace repository...")
remote_files = check_remote_files(api)
operations = []
for local_path, repo_path in MODEL_FILES:
if repo_path not in remote_files:
if local_path.exists():
size = local_path.stat().st_size
print(f" 📤 Will upload: {repo_path} ({size/1024:.1f} KB)")
operations.append(CommitOperationAdd(
path_in_repo=repo_path,
path_or_fileobj=str(local_path)
))
else:
print(f" ❌ Local file missing: {local_path}")
else:
print(f" ✅ Already exists: {repo_path}")
if operations:
print(f"\n📤 Uploading {len(operations)} files...")
try:
commit = create_commit(
repo_id=REPO_ID,
operations=operations,
commit_message="sync: Restore model files",
repo_type="model"
)
print(f"✅ Uploaded! {commit.commit_url}")
except Exception as e:
print(f"❌ Upload failed: {e}")
else:
print("\n✅ All model files are present!")
if __name__ == "__main__":
sync_models()
|