MiniSEAL / scripts /upload_to_hf.py
aaronr43210's picture
Upload scripts
f3ab765 verified
Raw History Blame Contribute Delete
3.77 kB
import os
import sys
import time
from huggingface_hub import HfApi
api = HfApi()
repo_id = "aaronr43210/MiniSEAL"
project_root = "/Users/aaronr/Desktop/DUK/GENAI/self_owned_llm"
def upload_folder_with_retry(folder_name, path_in_repo, ignore_patterns=None, max_retries=5):
full_path = os.path.join(project_root, folder_name)
if not os.path.exists(full_path):
print(f"Skipping {folder_name} (not found)")
return
print(f"\n==========================================")
print(f"Uploading {folder_name} -> {path_in_repo}")
print(f"==========================================")
for attempt in range(1, max_retries + 1):
try:
api.upload_folder(
folder_path=full_path,
path_in_repo=path_in_repo,
repo_id=repo_id,
repo_type="model",
ignore_patterns=ignore_patterns or ["__pycache__/**", "*.pyc", ".DS_Store"],
commit_message=f"Upload {folder_name}"
)
print(f"✓ Successfully uploaded {folder_name}")
return
except Exception as e:
print(f"[Attempt {attempt}/{max_retries}] Error: {e}")
if attempt < max_retries:
time.sleep(5 * attempt)
else:
print(f"Failed to upload {folder_name} after {max_retries} attempts.")
raise e
def upload_file_with_retry(rel_file_path, path_in_repo=None, max_retries=5):
full_path = os.path.join(project_root, rel_file_path)
if not os.path.exists(full_path):
print(f"Skipping file {rel_file_path} (not found)")
return
path_in_repo = path_in_repo or rel_file_path
print(f"\nUploading file: {rel_file_path} -> {path_in_repo}")
for attempt in range(1, max_retries + 1):
try:
api.upload_file(
path_or_fileobj=full_path,
path_in_repo=path_in_repo,
repo_id=repo_id,
repo_type="model",
commit_message=f"Upload {rel_file_path}"
)
print(f"✓ Successfully uploaded {rel_file_path}")
return
except Exception as e:
print(f"[Attempt {attempt}/{max_retries}] Error: {e}")
if attempt < max_retries:
time.sleep(5 * attempt)
else:
print(f"Failed to upload {rel_file_path} after {max_retries} attempts.")
raise e
if __name__ == "__main__":
# 1. Source code and configurations (lightweight, committed first)
print("--- Phase 1: Uploading Code & Configs ---")
for f in ["README.md", "requirements.txt", "architecture.html", "docker-compose.yml"]:
upload_file_with_retry(f)
for d in ["src", "configs", "scripts", "services", "tests", "docs", "artifacts"]:
upload_folder_with_retry(d, d)
# 2. Adapters
print("\n--- Phase 2: Uploading LoRA Adapters ---")
for adapter in [
"dgx_sft_rag/final_adapter",
"dgx_dpo_qwen/dpo_adapter_v1791831990",
"dgx_seal/seal_adapter"
]:
upload_folder_with_retry(f"model_weights/{adapter}", f"model_weights/{adapter}")
# 3. Small training runs & logs
print("\n--- Phase 3: Uploading Logs & Metadata ---")
upload_file_with_retry("runs/self_adapt/results.json", "runs/self_adapt/results.json")
upload_file_with_retry("ckpts/dgx_pretrain_124M/train_log.jsonl", "ckpts/dgx_pretrain_124M/train_log.jsonl")
# 4. Large pretrain checkpoint (1.09GB)
print("\n--- Phase 4: Uploading Pretrain Checkpoint ---")
upload_file_with_retry("ckpts/dgx_pretrain_124M/latest.pt", "ckpts/dgx_pretrain_124M/latest.pt")
print("\n🎉 ALL UPLOADS COMPLETED SUCCESSFULLY!")