Download training/upload_to_hf.py from SinhalaMMLU-SharedTask/team-qbits: direct link, hf CLI and curl.
- Browser
- Download file 12.8 kB
-
https://huggingface.co/SinhalaMMLU-SharedTask/team-qbits/resolve/main/training/upload_to_hf.py
- Command line
-
hf download hf://SinhalaMMLU-SharedTask/team-qbits/training/upload_to_hf.py
-
curl -L -o upload_to_hf.py https://huggingface.co/SinhalaMMLU-SharedTask/team-qbits/resolve/main/training/upload_to_hf.py
12.8 kB
| #!/usr/bin/env python | |
| """ | |
| Upload the trained Gemma 4 E4B Sinhala CPT model to the Hugging Face Hub and VERIFY every file. | |
| What it does | |
| 1. Checks your HF token and that the local folders really contain a complete model. | |
| 2. (optional) --make_merged: builds the merged 16-bit model from the LoRA adapter (needs the GPU free). | |
| 3. Counts the parameters of the merged model from the safetensors headers (8B rule for the shared task). | |
| 4. Writes a README.md model card if the folder has none. | |
| 5. Creates the repo(s) (private by default) and uploads, retrying on network errors. | |
| 6. Re-reads the repo from the Hub and compares EVERY local file: presence, size, and SHA-256 | |
| for large (LFS) files. Exits with code 1 if anything is missing or different. | |
| Usage | |
| export HF_TOKEN=hf_xxx # a WRITE token (do not paste it into chat or commit it) | |
| python upload_to_hf.py --adapter_repo YOURNAME/gemma4-e4b-sinhala-cpt-lora \ | |
| --merged_repo YOURNAME/gemma4-e4b-sinhala-cpt \ | |
| --make_merged | |
| # adapter only: omit --merged_repo merged only: omit --adapter_repo | |
| """ | |
| import argparse | |
| import fnmatch | |
| import hashlib | |
| import json | |
| import os | |
| import shutil | |
| import struct | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from huggingface_hub import HfApi, create_repo, get_token | |
| IGNORE = [".cache/*", ".cache", "__pycache__/*", "*.pyc", ".git/*", "optimizer.pt", "scheduler.pt", | |
| "rng_state*.pth", "*.tmp", "*.lock", ".DS_Store"] | |
| DEFAULT_OUT = "/workspace/outputs/gemma4_e4b_sinhala_cpt" | |
| # ----------------------------------------------------------------------------- helpers | |
| def ignored(rel): | |
| return any(fnmatch.fnmatch(rel, pat) or fnmatch.fnmatch(os.path.basename(rel), pat) for pat in IGNORE) | |
| def local_files(folder): | |
| out = {} | |
| for p in sorted(Path(folder).rglob("*")): | |
| if p.is_file(): | |
| rel = p.relative_to(folder).as_posix() | |
| if not ignored(rel): | |
| out[rel] = p | |
| return out | |
| def sha256_file(path, chunk=1 << 24): | |
| h = hashlib.sha256() | |
| with open(path, "rb") as f: | |
| while True: | |
| b = f.read(chunk) | |
| if not b: | |
| break | |
| h.update(b) | |
| return h.hexdigest() | |
| def human(n): | |
| for u in ["B", "KB", "MB", "GB", "TB"]: | |
| if n < 1024: | |
| return f"{n:.1f} {u}" | |
| n /= 1024 | |
| return f"{n:.1f} PB" | |
| def safetensors_param_count(folder): | |
| """Sum of tensor element counts from the .safetensors headers (no weights are loaded).""" | |
| total = 0 | |
| for p in Path(folder).glob("*.safetensors"): | |
| with open(p, "rb") as f: | |
| (n,) = struct.unpack("<Q", f.read(8)) | |
| header = json.loads(f.read(n)) | |
| for k, v in header.items(): | |
| if k == "__metadata__": | |
| continue | |
| c = 1 | |
| for d in v["shape"]: | |
| c *= d | |
| total += c | |
| return total | |
| def check_adapter(folder): | |
| miss = [f for f in ("adapter_config.json", "adapter_model.safetensors") if not (Path(folder) / f).exists()] | |
| return [f"missing {m}" for m in miss] | |
| def check_merged(folder): | |
| folder = Path(folder) | |
| problems = [] | |
| if not (folder / "config.json").exists(): | |
| problems.append("missing config.json") | |
| idx = folder / "model.safetensors.index.json" | |
| if idx.exists(): | |
| shards = set(json.load(open(idx))["weight_map"].values()) | |
| problems += [f"missing shard {s}" for s in sorted(shards) if not (folder / s).exists()] | |
| elif not list(folder.glob("*.safetensors")): | |
| problems.append("no .safetensors weights found") | |
| return problems | |
| def model_card(kind, repo, base, dataset): | |
| if kind == "adapter": | |
| lib, what = "peft", "LoRA adapter" | |
| use = (f"```python\nfrom unsloth import FastModel\n" | |
| f"model, tok = FastModel.from_pretrained('{repo}', max_seq_length=2048, load_in_4bit=False)\n```") | |
| else: | |
| lib, what = "transformers", "merged 16-bit weights (LoRA merged into the base model)" | |
| use = (f"```python\nfrom unsloth import FastModel\n" | |
| f"model, tok = FastModel.from_pretrained('{repo}', max_seq_length=2048, load_in_4bit=False)\n```") | |
| return f"""--- | |
| base_model: {base} | |
| library_name: {lib} | |
| language: | |
| - si | |
| tags: | |
| - gemma | |
| - sinhala | |
| - continued-pretraining | |
| - unsloth | |
| datasets: | |
| - {dataset} | |
| --- | |
| # {repo.split('/')[-1]} | |
| Continued pretraining (CPT) of `{base}` (text only) on Sinhala text from `{dataset}` | |
| (`high_confidence_7_7M.csv`), trained with Unsloth + LoRA. This repo contains the {what}. | |
| This is a **base-style** model: it was trained with the next-token loss on raw Sinhala text, not on | |
| instructions or multiple-choice answers. Check the base model's licence before using or redistributing. | |
| ## Load | |
| {use} | |
| """ | |
| # ----------------------------------------------------------------------------- steps | |
| def make_merged(adapter_dir, merged_dir, max_seq_length): | |
| print(f"[merge] loading adapter from {adapter_dir} (GPU must be free: stop the training job first)") | |
| from unsloth import FastModel # noqa: E402 (imported late on purpose) | |
| model, tok = FastModel.from_pretrained(model_name=adapter_dir, max_seq_length=max_seq_length, | |
| dtype=None, load_in_4bit=False) | |
| print(f"[merge] saving merged 16-bit model to {merged_dir}") | |
| model.save_pretrained_merged(merged_dir, tok, save_method="merged_16bit") | |
| def upload(api, token, repo, folder, private, msg, large): | |
| create_repo(repo, repo_type="model", private=private, exist_ok=True, token=token) | |
| for attempt in range(1, 4): | |
| try: | |
| if large: | |
| api.upload_large_folder(repo_id=repo, folder_path=str(folder), repo_type="model", | |
| ignore_patterns=IGNORE) | |
| else: | |
| api.upload_folder(repo_id=repo, folder_path=str(folder), repo_type="model", | |
| commit_message=msg, ignore_patterns=IGNORE) | |
| return | |
| except Exception as e: # noqa: BLE001 | |
| print(f"[upload] attempt {attempt} failed: {type(e).__name__}: {e}") | |
| if attempt == 3: | |
| raise | |
| time.sleep(15 * attempt) | |
| def verify(api, token, repo, folder, do_hash): | |
| """Compare every local file with what the Hub actually holds. Returns a list of problems.""" | |
| info = api.model_info(repo, files_metadata=True, token=token) | |
| remote = {s.rfilename: s for s in info.siblings} | |
| problems, checked = [], 0 | |
| for rel, path in local_files(folder).items(): | |
| s = remote.get(rel) | |
| if s is None: | |
| problems.append(f"MISSING on Hub: {rel}") | |
| continue | |
| lsize = path.stat().st_size | |
| if s.size is not None and s.size != lsize: | |
| problems.append(f"SIZE mismatch: {rel} local={lsize} remote={s.size}") | |
| continue | |
| lfs = getattr(s, "lfs", None) | |
| rsha = (lfs.get("sha256") if isinstance(lfs, dict) else getattr(lfs, "sha256", None)) if lfs else None | |
| if rsha and do_hash: | |
| if sha256_file(path) != rsha: | |
| problems.append(f"SHA-256 mismatch: {rel}") | |
| continue | |
| checked += 1 | |
| extra = [r for r in remote if r not in local_files(folder) and r not in (".gitattributes",)] | |
| if extra: | |
| print(f"[verify] note: files on the Hub that are not in the local folder: {extra}") | |
| print(f"[verify] {repo}: {checked}/{len(local_files(folder))} local files confirmed " | |
| f"({'size + SHA-256' if do_hash else 'size only'})") | |
| return problems | |
| def smoke_test(repo, token, max_seq_length): | |
| print(f"[smoke] downloading {repo} from the Hub and generating a few tokens") | |
| from unsloth import FastModel # noqa: E402 | |
| model, tok = FastModel.from_pretrained(repo, max_seq_length=max_seq_length, dtype=None, | |
| load_in_4bit=False, token=token) | |
| t = getattr(tok, "tokenizer", tok) | |
| ids = t("ශ්රී ලංකාව", return_tensors="pt").to("cuda") | |
| out = model.generate(**ids, max_new_tokens=30, do_sample=False) | |
| print("[smoke] output:", t.decode(out[0], skip_special_tokens=True)) | |
| # ----------------------------------------------------------------------------- main | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--adapter_dir", default=f"{DEFAULT_OUT}/lora_adapter") | |
| ap.add_argument("--merged_dir", default=f"{DEFAULT_OUT}/merged_16bit") | |
| ap.add_argument("--adapter_repo", default="", help="e.g. yourname/gemma4-e4b-sinhala-cpt-lora") | |
| ap.add_argument("--merged_repo", default="", help="e.g. yourname/gemma4-e4b-sinhala-cpt") | |
| ap.add_argument("--make_merged", action="store_true", help="build merged_16bit from the adapter if missing") | |
| ap.add_argument("--extra", nargs="*", default=[], help="files to include under training/ in the adapter repo " | |
| "(e.g. cpt_run.log cpt_gemma4_sinhala.py)") | |
| ap.add_argument("--public", action="store_true", help="make the repos public (default: private)") | |
| ap.add_argument("--skip_hash", action="store_true", help="verify sizes only (faster)") | |
| ap.add_argument("--smoke_test", action="store_true", help="after upload, load the merged repo from the Hub") | |
| ap.add_argument("--base_model", default="google/gemma-4-E4B") | |
| ap.add_argument("--dataset", default="Minuri/diverse_sinhala_dataset") | |
| ap.add_argument("--max_seq_length", type=int, default=2048) | |
| ap.add_argument("--token", default="") | |
| args = ap.parse_args() | |
| if not (args.adapter_repo or args.merged_repo): | |
| sys.exit("Give --adapter_repo and/or --merged_repo") | |
| token = args.token or os.environ.get("HF_TOKEN") or get_token() | |
| if not token: | |
| sys.exit("No Hugging Face token. Run: export HF_TOKEN=hf_xxx (a token with WRITE access)") | |
| api = HfApi(token=token) | |
| me = api.whoami() | |
| print(f"[auth] logged in as {me['name']}") | |
| targets = [] # (kind, folder, repo) | |
| if args.adapter_repo: | |
| if not Path(args.adapter_dir).exists(): | |
| sys.exit(f"adapter_dir not found: {args.adapter_dir}") | |
| for f in args.extra: | |
| if Path(f).is_file(): | |
| dst = Path(args.adapter_dir) / "training" | |
| dst.mkdir(exist_ok=True) | |
| shutil.copy2(f, dst / Path(f).name) | |
| else: | |
| print(f"[extra] skipping {f} (not a file)") | |
| targets.append(("adapter", args.adapter_dir, args.adapter_repo)) | |
| if args.merged_repo: | |
| if not Path(args.merged_dir).exists() or not list(Path(args.merged_dir).glob("*.safetensors")): | |
| if args.make_merged: | |
| make_merged(args.adapter_dir, args.merged_dir, args.max_seq_length) | |
| else: | |
| sys.exit(f"merged_dir missing or empty: {args.merged_dir} (use --make_merged)") | |
| targets.append(("merged", args.merged_dir, args.merged_repo)) | |
| # ---- pre-flight ---- | |
| for kind, folder, repo in targets: | |
| problems = check_adapter(folder) if kind == "adapter" else check_merged(folder) | |
| if problems: | |
| sys.exit(f"[check] {kind} folder incomplete: {problems}") | |
| files = local_files(folder) | |
| print(f"[check] {kind}: {len(files)} files, {human(sum(p.stat().st_size for p in files.values()))}") | |
| if kind == "merged": | |
| n = safetensors_param_count(folder) | |
| flag = "OK (<= 8.0B)" if n <= 8_000_000_000 else "OVER the 8B shared-task limit!" | |
| print(f"[check] merged parameter count: {n:,} -> {flag}") | |
| if not (Path(folder) / "README.md").exists(): | |
| (Path(folder) / "README.md").write_text(model_card(kind, repo, args.base_model, args.dataset)) | |
| print(f"[card] wrote README.md in {folder}") | |
| # ---- upload + verify ---- | |
| all_problems = [] | |
| for kind, folder, repo in targets: | |
| print(f"\n[upload] {kind} -> https://huggingface.co/{repo}") | |
| upload(api, token, repo, folder, not args.public, f"Upload {kind} (Gemma 4 E4B Sinhala CPT)", | |
| large=(kind == "merged")) | |
| probs = verify(api, token, repo, folder, not args.skip_hash) | |
| all_problems += [f"{repo}: {p}" for p in probs] | |
| print() | |
| if all_problems: | |
| print("VERIFICATION FAILED:") | |
| for p in all_problems: | |
| print(" -", p) | |
| print("Re-run the same command: already-uploaded files are skipped and missing ones are retried.") | |
| sys.exit(1) | |
| print("ALL FILES UPLOADED AND VERIFIED") | |
| for _, _, repo in targets: | |
| print(f" https://huggingface.co/{repo}") | |
| if args.smoke_test and args.merged_repo: | |
| smoke_test(args.merged_repo, token, args.max_seq_length) | |
| if __name__ == "__main__": | |
| main() | |