import os import sys import time from huggingface_hub import HfApi api = HfApi() repo_id = "aaronr43210/MiniSEAL" project_root = "/Users/aaronr/Desktop/DUK/GENAI/self_owned_llm" def upload_folder_with_retry(folder_name, path_in_repo, ignore_patterns=None, max_retries=5): full_path = os.path.join(project_root, folder_name) if not os.path.exists(full_path): print(f"Skipping {folder_name} (not found)") return print(f"\n==========================================") print(f"Uploading {folder_name} -> {path_in_repo}") print(f"==========================================") for attempt in range(1, max_retries + 1): try: api.upload_folder( folder_path=full_path, path_in_repo=path_in_repo, repo_id=repo_id, repo_type="model", ignore_patterns=ignore_patterns or ["__pycache__/**", "*.pyc", ".DS_Store"], commit_message=f"Upload {folder_name}" ) print(f"āœ“ Successfully uploaded {folder_name}") return except Exception as e: print(f"[Attempt {attempt}/{max_retries}] Error: {e}") if attempt < max_retries: time.sleep(5 * attempt) else: print(f"Failed to upload {folder_name} after {max_retries} attempts.") raise e def upload_file_with_retry(rel_file_path, path_in_repo=None, max_retries=5): full_path = os.path.join(project_root, rel_file_path) if not os.path.exists(full_path): print(f"Skipping file {rel_file_path} (not found)") return path_in_repo = path_in_repo or rel_file_path print(f"\nUploading file: {rel_file_path} -> {path_in_repo}") for attempt in range(1, max_retries + 1): try: api.upload_file( path_or_fileobj=full_path, path_in_repo=path_in_repo, repo_id=repo_id, repo_type="model", commit_message=f"Upload {rel_file_path}" ) print(f"āœ“ Successfully uploaded {rel_file_path}") return except Exception as e: print(f"[Attempt {attempt}/{max_retries}] Error: {e}") if attempt < max_retries: time.sleep(5 * attempt) else: print(f"Failed to upload {rel_file_path} after {max_retries} attempts.") raise e if __name__ == "__main__": # 1. Source code and configurations (lightweight, committed first) print("--- Phase 1: Uploading Code & Configs ---") for f in ["README.md", "requirements.txt", "architecture.html", "docker-compose.yml"]: upload_file_with_retry(f) for d in ["src", "configs", "scripts", "services", "tests", "docs", "artifacts"]: upload_folder_with_retry(d, d) # 2. Adapters print("\n--- Phase 2: Uploading LoRA Adapters ---") for adapter in [ "dgx_sft_rag/final_adapter", "dgx_dpo_qwen/dpo_adapter_v1791831990", "dgx_seal/seal_adapter" ]: upload_folder_with_retry(f"model_weights/{adapter}", f"model_weights/{adapter}") # 3. Small training runs & logs print("\n--- Phase 3: Uploading Logs & Metadata ---") upload_file_with_retry("runs/self_adapt/results.json", "runs/self_adapt/results.json") upload_file_with_retry("ckpts/dgx_pretrain_124M/train_log.jsonl", "ckpts/dgx_pretrain_124M/train_log.jsonl") # 4. Large pretrain checkpoint (1.09GB) print("\n--- Phase 4: Uploading Pretrain Checkpoint ---") upload_file_with_retry("ckpts/dgx_pretrain_124M/latest.pt", "ckpts/dgx_pretrain_124M/latest.pt") print("\nšŸŽ‰ ALL UPLOADS COMPLETED SUCCESSFULLY!")