#!/usr/bin/env python """ Upload the trained Gemma 4 E4B Sinhala CPT model to the Hugging Face Hub and VERIFY every file. What it does 1. Checks your HF token and that the local folders really contain a complete model. 2. (optional) --make_merged: builds the merged 16-bit model from the LoRA adapter (needs the GPU free). 3. Counts the parameters of the merged model from the safetensors headers (8B rule for the shared task). 4. Writes a README.md model card if the folder has none. 5. Creates the repo(s) (private by default) and uploads, retrying on network errors. 6. Re-reads the repo from the Hub and compares EVERY local file: presence, size, and SHA-256 for large (LFS) files. Exits with code 1 if anything is missing or different. Usage export HF_TOKEN=hf_xxx # a WRITE token (do not paste it into chat or commit it) python upload_to_hf.py --adapter_repo YOURNAME/gemma4-e4b-sinhala-cpt-lora \ --merged_repo YOURNAME/gemma4-e4b-sinhala-cpt \ --make_merged # adapter only: omit --merged_repo merged only: omit --adapter_repo """ import argparse import fnmatch import hashlib import json import os import shutil import struct import sys import time from pathlib import Path from huggingface_hub import HfApi, create_repo, get_token IGNORE = [".cache/*", ".cache", "__pycache__/*", "*.pyc", ".git/*", "optimizer.pt", "scheduler.pt", "rng_state*.pth", "*.tmp", "*.lock", ".DS_Store"] DEFAULT_OUT = "/workspace/outputs/gemma4_e4b_sinhala_cpt" # ----------------------------------------------------------------------------- helpers def ignored(rel): return any(fnmatch.fnmatch(rel, pat) or fnmatch.fnmatch(os.path.basename(rel), pat) for pat in IGNORE) def local_files(folder): out = {} for p in sorted(Path(folder).rglob("*")): if p.is_file(): rel = p.relative_to(folder).as_posix() if not ignored(rel): out[rel] = p return out def sha256_file(path, chunk=1 << 24): h = hashlib.sha256() with open(path, "rb") as f: while True: b = f.read(chunk) if not b: break h.update(b) return h.hexdigest() def human(n): for u in ["B", "KB", "MB", "GB", "TB"]: if n < 1024: return f"{n:.1f} {u}" n /= 1024 return f"{n:.1f} PB" def safetensors_param_count(folder): """Sum of tensor element counts from the .safetensors headers (no weights are loaded).""" total = 0 for p in Path(folder).glob("*.safetensors"): with open(p, "rb") as f: (n,) = struct.unpack(" {flag}") if not (Path(folder) / "README.md").exists(): (Path(folder) / "README.md").write_text(model_card(kind, repo, args.base_model, args.dataset)) print(f"[card] wrote README.md in {folder}") # ---- upload + verify ---- all_problems = [] for kind, folder, repo in targets: print(f"\n[upload] {kind} -> https://huggingface.co/{repo}") upload(api, token, repo, folder, not args.public, f"Upload {kind} (Gemma 4 E4B Sinhala CPT)", large=(kind == "merged")) probs = verify(api, token, repo, folder, not args.skip_hash) all_problems += [f"{repo}: {p}" for p in probs] print() if all_problems: print("VERIFICATION FAILED:") for p in all_problems: print(" -", p) print("Re-run the same command: already-uploaded files are skipped and missing ones are retried.") sys.exit(1) print("ALL FILES UPLOADED AND VERIFIED") for _, _, repo in targets: print(f" https://huggingface.co/{repo}") if args.smoke_test and args.merged_repo: smoke_test(args.merged_repo, token, args.max_seq_length) if __name__ == "__main__": main()