team-qbits / training /upload_to_hf.py
ovinduG's picture
Upload folder using huggingface_hub
e55c8a4 verified
Raw History Blame Contribute Delete
12.8 kB
#!/usr/bin/env python
"""
Upload the trained Gemma 4 E4B Sinhala CPT model to the Hugging Face Hub and VERIFY every file.
What it does
1. Checks your HF token and that the local folders really contain a complete model.
2. (optional) --make_merged: builds the merged 16-bit model from the LoRA adapter (needs the GPU free).
3. Counts the parameters of the merged model from the safetensors headers (8B rule for the shared task).
4. Writes a README.md model card if the folder has none.
5. Creates the repo(s) (private by default) and uploads, retrying on network errors.
6. Re-reads the repo from the Hub and compares EVERY local file: presence, size, and SHA-256
for large (LFS) files. Exits with code 1 if anything is missing or different.
Usage
export HF_TOKEN=hf_xxx # a WRITE token (do not paste it into chat or commit it)
python upload_to_hf.py --adapter_repo YOURNAME/gemma4-e4b-sinhala-cpt-lora \
--merged_repo YOURNAME/gemma4-e4b-sinhala-cpt \
--make_merged
# adapter only: omit --merged_repo merged only: omit --adapter_repo
"""
import argparse
import fnmatch
import hashlib
import json
import os
import shutil
import struct
import sys
import time
from pathlib import Path
from huggingface_hub import HfApi, create_repo, get_token
IGNORE = [".cache/*", ".cache", "__pycache__/*", "*.pyc", ".git/*", "optimizer.pt", "scheduler.pt",
"rng_state*.pth", "*.tmp", "*.lock", ".DS_Store"]
DEFAULT_OUT = "/workspace/outputs/gemma4_e4b_sinhala_cpt"
# ----------------------------------------------------------------------------- helpers
def ignored(rel):
return any(fnmatch.fnmatch(rel, pat) or fnmatch.fnmatch(os.path.basename(rel), pat) for pat in IGNORE)
def local_files(folder):
out = {}
for p in sorted(Path(folder).rglob("*")):
if p.is_file():
rel = p.relative_to(folder).as_posix()
if not ignored(rel):
out[rel] = p
return out
def sha256_file(path, chunk=1 << 24):
h = hashlib.sha256()
with open(path, "rb") as f:
while True:
b = f.read(chunk)
if not b:
break
h.update(b)
return h.hexdigest()
def human(n):
for u in ["B", "KB", "MB", "GB", "TB"]:
if n < 1024:
return f"{n:.1f} {u}"
n /= 1024
return f"{n:.1f} PB"
def safetensors_param_count(folder):
"""Sum of tensor element counts from the .safetensors headers (no weights are loaded)."""
total = 0
for p in Path(folder).glob("*.safetensors"):
with open(p, "rb") as f:
(n,) = struct.unpack("<Q", f.read(8))
header = json.loads(f.read(n))
for k, v in header.items():
if k == "__metadata__":
continue
c = 1
for d in v["shape"]:
c *= d
total += c
return total
def check_adapter(folder):
miss = [f for f in ("adapter_config.json", "adapter_model.safetensors") if not (Path(folder) / f).exists()]
return [f"missing {m}" for m in miss]
def check_merged(folder):
folder = Path(folder)
problems = []
if not (folder / "config.json").exists():
problems.append("missing config.json")
idx = folder / "model.safetensors.index.json"
if idx.exists():
shards = set(json.load(open(idx))["weight_map"].values())
problems += [f"missing shard {s}" for s in sorted(shards) if not (folder / s).exists()]
elif not list(folder.glob("*.safetensors")):
problems.append("no .safetensors weights found")
return problems
def model_card(kind, repo, base, dataset):
if kind == "adapter":
lib, what = "peft", "LoRA adapter"
use = (f"```python\nfrom unsloth import FastModel\n"
f"model, tok = FastModel.from_pretrained('{repo}', max_seq_length=2048, load_in_4bit=False)\n```")
else:
lib, what = "transformers", "merged 16-bit weights (LoRA merged into the base model)"
use = (f"```python\nfrom unsloth import FastModel\n"
f"model, tok = FastModel.from_pretrained('{repo}', max_seq_length=2048, load_in_4bit=False)\n```")
return f"""---
base_model: {base}
library_name: {lib}
language:
- si
tags:
- gemma
- sinhala
- continued-pretraining
- unsloth
datasets:
- {dataset}
---
# {repo.split('/')[-1]}
Continued pretraining (CPT) of `{base}` (text only) on Sinhala text from `{dataset}`
(`high_confidence_7_7M.csv`), trained with Unsloth + LoRA. This repo contains the {what}.
This is a **base-style** model: it was trained with the next-token loss on raw Sinhala text, not on
instructions or multiple-choice answers. Check the base model's licence before using or redistributing.
## Load
{use}
"""
# ----------------------------------------------------------------------------- steps
def make_merged(adapter_dir, merged_dir, max_seq_length):
print(f"[merge] loading adapter from {adapter_dir} (GPU must be free: stop the training job first)")
from unsloth import FastModel # noqa: E402 (imported late on purpose)
model, tok = FastModel.from_pretrained(model_name=adapter_dir, max_seq_length=max_seq_length,
dtype=None, load_in_4bit=False)
print(f"[merge] saving merged 16-bit model to {merged_dir}")
model.save_pretrained_merged(merged_dir, tok, save_method="merged_16bit")
def upload(api, token, repo, folder, private, msg, large):
create_repo(repo, repo_type="model", private=private, exist_ok=True, token=token)
for attempt in range(1, 4):
try:
if large:
api.upload_large_folder(repo_id=repo, folder_path=str(folder), repo_type="model",
ignore_patterns=IGNORE)
else:
api.upload_folder(repo_id=repo, folder_path=str(folder), repo_type="model",
commit_message=msg, ignore_patterns=IGNORE)
return
except Exception as e: # noqa: BLE001
print(f"[upload] attempt {attempt} failed: {type(e).__name__}: {e}")
if attempt == 3:
raise
time.sleep(15 * attempt)
def verify(api, token, repo, folder, do_hash):
"""Compare every local file with what the Hub actually holds. Returns a list of problems."""
info = api.model_info(repo, files_metadata=True, token=token)
remote = {s.rfilename: s for s in info.siblings}
problems, checked = [], 0
for rel, path in local_files(folder).items():
s = remote.get(rel)
if s is None:
problems.append(f"MISSING on Hub: {rel}")
continue
lsize = path.stat().st_size
if s.size is not None and s.size != lsize:
problems.append(f"SIZE mismatch: {rel} local={lsize} remote={s.size}")
continue
lfs = getattr(s, "lfs", None)
rsha = (lfs.get("sha256") if isinstance(lfs, dict) else getattr(lfs, "sha256", None)) if lfs else None
if rsha and do_hash:
if sha256_file(path) != rsha:
problems.append(f"SHA-256 mismatch: {rel}")
continue
checked += 1
extra = [r for r in remote if r not in local_files(folder) and r not in (".gitattributes",)]
if extra:
print(f"[verify] note: files on the Hub that are not in the local folder: {extra}")
print(f"[verify] {repo}: {checked}/{len(local_files(folder))} local files confirmed "
f"({'size + SHA-256' if do_hash else 'size only'})")
return problems
def smoke_test(repo, token, max_seq_length):
print(f"[smoke] downloading {repo} from the Hub and generating a few tokens")
from unsloth import FastModel # noqa: E402
model, tok = FastModel.from_pretrained(repo, max_seq_length=max_seq_length, dtype=None,
load_in_4bit=False, token=token)
t = getattr(tok, "tokenizer", tok)
ids = t("ශ්‍රී ලංකාව", return_tensors="pt").to("cuda")
out = model.generate(**ids, max_new_tokens=30, do_sample=False)
print("[smoke] output:", t.decode(out[0], skip_special_tokens=True))
# ----------------------------------------------------------------------------- main
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--adapter_dir", default=f"{DEFAULT_OUT}/lora_adapter")
ap.add_argument("--merged_dir", default=f"{DEFAULT_OUT}/merged_16bit")
ap.add_argument("--adapter_repo", default="", help="e.g. yourname/gemma4-e4b-sinhala-cpt-lora")
ap.add_argument("--merged_repo", default="", help="e.g. yourname/gemma4-e4b-sinhala-cpt")
ap.add_argument("--make_merged", action="store_true", help="build merged_16bit from the adapter if missing")
ap.add_argument("--extra", nargs="*", default=[], help="files to include under training/ in the adapter repo "
"(e.g. cpt_run.log cpt_gemma4_sinhala.py)")
ap.add_argument("--public", action="store_true", help="make the repos public (default: private)")
ap.add_argument("--skip_hash", action="store_true", help="verify sizes only (faster)")
ap.add_argument("--smoke_test", action="store_true", help="after upload, load the merged repo from the Hub")
ap.add_argument("--base_model", default="google/gemma-4-E4B")
ap.add_argument("--dataset", default="Minuri/diverse_sinhala_dataset")
ap.add_argument("--max_seq_length", type=int, default=2048)
ap.add_argument("--token", default="")
args = ap.parse_args()
if not (args.adapter_repo or args.merged_repo):
sys.exit("Give --adapter_repo and/or --merged_repo")
token = args.token or os.environ.get("HF_TOKEN") or get_token()
if not token:
sys.exit("No Hugging Face token. Run: export HF_TOKEN=hf_xxx (a token with WRITE access)")
api = HfApi(token=token)
me = api.whoami()
print(f"[auth] logged in as {me['name']}")
targets = [] # (kind, folder, repo)
if args.adapter_repo:
if not Path(args.adapter_dir).exists():
sys.exit(f"adapter_dir not found: {args.adapter_dir}")
for f in args.extra:
if Path(f).is_file():
dst = Path(args.adapter_dir) / "training"
dst.mkdir(exist_ok=True)
shutil.copy2(f, dst / Path(f).name)
else:
print(f"[extra] skipping {f} (not a file)")
targets.append(("adapter", args.adapter_dir, args.adapter_repo))
if args.merged_repo:
if not Path(args.merged_dir).exists() or not list(Path(args.merged_dir).glob("*.safetensors")):
if args.make_merged:
make_merged(args.adapter_dir, args.merged_dir, args.max_seq_length)
else:
sys.exit(f"merged_dir missing or empty: {args.merged_dir} (use --make_merged)")
targets.append(("merged", args.merged_dir, args.merged_repo))
# ---- pre-flight ----
for kind, folder, repo in targets:
problems = check_adapter(folder) if kind == "adapter" else check_merged(folder)
if problems:
sys.exit(f"[check] {kind} folder incomplete: {problems}")
files = local_files(folder)
print(f"[check] {kind}: {len(files)} files, {human(sum(p.stat().st_size for p in files.values()))}")
if kind == "merged":
n = safetensors_param_count(folder)
flag = "OK (<= 8.0B)" if n <= 8_000_000_000 else "OVER the 8B shared-task limit!"
print(f"[check] merged parameter count: {n:,} -> {flag}")
if not (Path(folder) / "README.md").exists():
(Path(folder) / "README.md").write_text(model_card(kind, repo, args.base_model, args.dataset))
print(f"[card] wrote README.md in {folder}")
# ---- upload + verify ----
all_problems = []
for kind, folder, repo in targets:
print(f"\n[upload] {kind} -> https://huggingface.co/{repo}")
upload(api, token, repo, folder, not args.public, f"Upload {kind} (Gemma 4 E4B Sinhala CPT)",
large=(kind == "merged"))
probs = verify(api, token, repo, folder, not args.skip_hash)
all_problems += [f"{repo}: {p}" for p in probs]
print()
if all_problems:
print("VERIFICATION FAILED:")
for p in all_problems:
print(" -", p)
print("Re-run the same command: already-uploaded files are skipped and missing ones are retried.")
sys.exit(1)
print("ALL FILES UPLOADED AND VERIFIED")
for _, _, repo in targets:
print(f" https://huggingface.co/{repo}")
if args.smoke_test and args.merged_repo:
smoke_test(args.merged_repo, token, args.max_seq_length)
if __name__ == "__main__":
main()