YokoCoder-4B / kaggle_train_and_export.py
ahpah's picture
Update kaggle_train_and_export.py with exact Unsloth 2026.10.10 + transformers 5.17.0 + --no-mtp pipeline
cde4ff1 verified
Raw History Blame Contribute Delete
13.4 kB
#!/usr/bin/env python3
"""
YokoCoder:4B — Battle-Tested Unsloth 16-Bit LoRA Fine-Tuning, Safe Merge,
GGUF (Q4_K_M) Quantization (`--no-mtp`), and Hugging Face Export (`ahpah/YokoCoder-4B`).
"""
import gc
import hashlib
import json
import math
import os
import subprocess
import sys
import zipfile
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
os.environ["TOKENIZERS_PARALLELISM"] = "false"
os.environ["WANDB_DISABLED"] = "true"
# 1. Ensure exact battle-tested Unsloth + Transformers versions are installed
subprocess.check_call([
sys.executable, "-m", "pip", "install", "--quiet", "--upgrade",
"unsloth==2026.10.10", "unsloth_zoo", "transformers==5.17.0",
"peft", "accelerate", "datasets", "bitsandbytes",
"huggingface_hub", "sentencepiece", "protobuf", "gguf"
])
from unsloth import FastLanguageModel
import torch
from datasets import Dataset
from huggingface_hub import HfApi, hf_hub_download, snapshot_download
from peft import PeftModel
from transformers import AutoTokenizer, Trainer, TrainerCallback, TrainingArguments, set_seed
assert torch.cuda.is_available(), "Enable a GPU accelerator in Kaggle Settings!"
SEED = 3407
set_seed(SEED)
EPOCHS = 3
MAX_SEQ_LENGTH = 4096
OVERLAP = 512
GRADIENT_ACCUMULATION = 8
HF_TARGET_REPO = "ahpah/YokoCoder-4B"
BASE_NAME = "wangzhang/Qwen3.5-4B-abliterated"
BASE_REVISION = "bbf0cdd540907b95ca8430fd34e4b2971f73df57"
ROOT = Path("/kaggle/working/yokocoder_v2_run")
ROOT.mkdir(parents=True, exist_ok=True)
def get_hf_token() -> str | None:
token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN")
if token:
return token
try:
from kaggle_secrets import UserSecretsClient
return UserSecretsClient().get_secret("HF_TOKEN")
except Exception:
return None
HF_TOKEN = get_hf_token()
# ---------------------------------------------------------------------------
# 2. LOCATE OR DOWNLOAD STARTING ADAPTER FROM `ahpah/YokoCoder-4B`
# ---------------------------------------------------------------------------
found = list(Path("/kaggle/input").glob("**/final_adapter/adapter_config.json"))
if found:
ckpt = found[0].parent
else:
print(f"Downloading starting adapter from {HF_TARGET_REPO}...", flush=True)
snap_dir = snapshot_download(
repo_id=HF_TARGET_REPO,
allow_patterns=["adapters/dataset4-5epochs/final_adapter/*"],
token=HF_TOKEN,
)
ckpt = Path(snap_dir) / "adapters" / "dataset4-5epochs" / "final_adapter"
adapter_sha = hashlib.sha256((ckpt / "adapter_model.safetensors").read_bytes()).hexdigest()
print("START_ADAPTER " + json.dumps({"path": str(ckpt), "sha256": adapter_sha}), flush=True)
tokenizer = AutoTokenizer.from_pretrained(ckpt)
if tokenizer.pad_token_id is None:
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "right"
bf16 = torch.cuda.is_bf16_supported()
base, _ = FastLanguageModel.from_pretrained(
model_name=BASE_NAME,
revision=BASE_REVISION,
max_seq_length=MAX_SEQ_LENGTH,
dtype=torch.bfloat16 if bf16 else torch.float16,
load_in_4bit=False,
load_in_16bit=True,
full_finetuning=False,
fast_inference=False,
)
model = PeftModel.from_pretrained(base, ckpt, is_trainable=True)
FastLanguageModel.for_training(model, use_gradient_checkpointing="unsloth")
model.config.use_cache = False
model.print_trainable_parameters()
print("TRAINABLE_ADAPTER_LOADED", flush=True)
# ---------------------------------------------------------------------------
# 3. LOCATE OR DOWNLOAD DATASET (`yokocoder_4b_reasoning_gamedev_bugfix.jsonl`)
# ---------------------------------------------------------------------------
data_files = list(Path("/kaggle/input").glob("**/yokocoder_4b_reasoning_gamedev_bugfix.jsonl"))
if data_files:
dataset_path = data_files[0]
else:
print(f"Downloading dataset from {HF_TARGET_REPO}...", flush=True)
dataset_path = Path(
hf_hub_download(
repo_id=HF_TARGET_REPO,
filename="datasets/yokocoder_4b_reasoning_gamedev_bugfix.jsonl",
repo_type="model",
force_download=True,
token=HF_TOKEN,
)
)
raw = dataset_path.read_bytes()
records = [json.loads(line) for line in raw.decode("utf-8-sig").splitlines() if line.strip()]
for i, r in enumerate(records):
assert isinstance(r, dict) and isinstance(r.get("messages"), list), f"Bad record {i}"
assert [m.get("role") for m in r["messages"]] == ["system", "user", "assistant"], f"Role order error in {i}"
assert all(isinstance(m.get("content"), str) and m["content"].strip() for m in r["messages"]), f"Empty msg in {i}"
DATA_SHA = hashlib.sha256(raw).hexdigest()
print(
"DATA_VERIFIED "
+ json.dumps({"path": str(dataset_path), "examples": len(records), "bytes": len(raw), "sha256": DATA_SHA}),
flush=True,
)
def encode_record(record):
messages = record["messages"]
prefix = tokenizer.apply_chat_template(
messages[:-1], tokenize=False, add_generation_prompt=True, enable_thinking=True
)
full = tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=False, enable_thinking=True
)
assert full.startswith(prefix), "Chat template prefix mismatch"
encoded = tokenizer(full, add_special_tokens=False, return_offsets_mapping=True)
ids = encoded["input_ids"]
labels = [
tok if end > len(prefix) and start >= len(prefix) else -100
for tok, (start, end) in zip(ids, encoded["offset_mapping"])
]
return ids, labels
def make_windows(rows):
windows = []
stats = []
stride = MAX_SEQ_LENGTH - OVERLAP
for idx, record in enumerate(rows):
ids, labels = encode_record(record)
expected = sum(v != -100 for v in labels[1:])
actual = 0
for start in range(0, len(ids), stride):
end = min(start + MAX_SEQ_LENGTH, len(ids))
x = ids[start:end]
y = labels[start:end].copy()
for j in range(min(OVERLAP if start else 1, len(y))):
y[j] = -100
count = sum(v != -100 for v in y)
if count:
windows.append({"input_ids": x, "attention_mask": [1] * len(x), "labels": y})
actual += count
if end == len(ids):
break
assert actual == expected, (idx, actual, expected)
stats.append(
{"example": idx, "tokens": len(ids), "assistant_targets": expected, "covered_targets": actual}
)
assert windows
return windows, stats
train_windows, coverage = make_windows(records)
train_dataset = Dataset.from_list(train_windows)
(ROOT / "run_config.json").write_text(
json.dumps(
{
"dataset_sha256": DATA_SHA,
"starting_adapter_sha256": adapter_sha,
"base_model": BASE_NAME,
"base_revision": BASE_REVISION,
"additional_epochs": EPOCHS,
"train_examples": len(records),
"train_windows": len(train_windows),
"coverage": coverage,
},
indent=2,
)
)
print(
"TRAIN_DATA_READY "
+ json.dumps(
{
"examples": len(records),
"windows": len(train_windows),
"targets": sum(x["covered_targets"] for x in coverage),
}
),
flush=True,
)
# ---------------------------------------------------------------------------
# 4. FINE-TUNE WITH UNSLOTH 16-BIT + ADAMW_8BIT
# ---------------------------------------------------------------------------
def collate(features):
size = math.ceil(max(len(f["input_ids"]) for f in features) / 8) * 8
return {
"input_ids": torch.tensor(
[f["input_ids"] + [tokenizer.pad_token_id] * (size - len(f["input_ids"])) for f in features]
),
"attention_mask": torch.tensor(
[f["attention_mask"] + [0] * (size - len(f["input_ids"])) for f in features]
),
"labels": torch.tensor([f["labels"] + [-100] * (size - len(f["labels"])) for f in features]),
}
class Progress(TrainerCallback):
def on_log(self, args, state, control, logs=None, **kwargs):
if logs and "loss" in logs and not math.isfinite(logs["loss"]):
raise FloatingPointError("non-finite loss")
print(
"TRAIN_PROGRESS "
+ json.dumps({"epoch": state.epoch, "step": state.global_step, "metrics": logs}),
flush=True,
)
args = TrainingArguments(
output_dir=str(ROOT / "checkpoints"),
num_train_epochs=EPOCHS,
per_device_train_batch_size=1,
gradient_accumulation_steps=GRADIENT_ACCUMULATION,
learning_rate=2e-5,
warmup_steps=0,
lr_scheduler_type="constant",
weight_decay=0.01,
max_grad_norm=0.3,
fp16=False,
bf16=bf16,
optim="adamw_8bit",
save_strategy="epoch",
save_total_limit=2,
logging_steps=1,
logging_nan_inf_filter=False,
report_to="none",
disable_tqdm=True,
seed=SEED,
data_seed=SEED,
dataloader_num_workers=0,
remove_unused_columns=False,
)
trainer = Trainer(
model=model,
args=args,
train_dataset=train_dataset,
data_collator=collate,
processing_class=tokenizer,
callbacks=[Progress()],
)
result = trainer.train()
trainer.save_model(str(ROOT / "final_adapter"))
tokenizer.save_pretrained(ROOT / "final_adapter")
trainer.save_state()
trainer.save_metrics("train", result.metrics)
epoch = float(trainer.state.epoch or 0)
status = {
"completed": epoch >= EPOCHS - 1e-5,
"requested_epochs": EPOCHS,
"completed_epochs": epoch,
"global_step": trainer.state.global_step,
"final_adapter": str(ROOT / "final_adapter"),
"metrics": result.metrics,
}
(ROOT / "training_status.json").write_text(json.dumps(status, indent=2))
print("TRAINING_COMPLETE " + json.dumps(status), flush=True)
assert status["completed"], f"Training stopped before {EPOCHS} epochs"
# ---------------------------------------------------------------------------
# 5. MERGE TO 16-BIT & QUANTIZE TO Q4_K_M GGUF (`--no-mtp`)
# ---------------------------------------------------------------------------
def build_tools():
repo = Path("/kaggle/temp/llama.cpp")
if not repo.exists():
subprocess.check_call(["git", "clone", "--depth", "1", "https://github.com/ggml-org/llama.cpp", str(repo)])
subprocess.check_call(
[
"cmake", "-S", str(repo), "-B", str(repo / "build"),
"-DGGML_CUDA=OFF", "-DLLAMA_CURL=OFF", "-DLLAMA_BUILD_TESTS=OFF",
"-DLLAMA_BUILD_EXAMPLES=OFF", "-DLLAMA_BUILD_TOOLS=ON", "-DBUILD_SHARED_LIBS=OFF",
],
stdout=subprocess.DEVNULL,
)
subprocess.check_call(
["cmake", "--build", str(repo / "build"), "--config", "Release", "--target", "llama-quantize", "-j", "2"],
stdout=subprocess.DEVNULL,
)
return repo
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(build_tools)
print("MERGE_START", flush=True)
merged = Path("/kaggle/temp/YokoCoder-4B-v2-merged")
model = model.merge_and_unload(safe_merge=True)
model = model.to(dtype=torch.bfloat16)
model.config.eos_token_id = tokenizer.eos_token_id
model.generation_config.eos_token_id = tokenizer.eos_token_id
model.save_pretrained(merged, safe_serialization=True, max_shard_size="4GB")
tokenizer.save_pretrained(merged)
print("MERGED_WEIGHTS_SAVED", flush=True)
del model, base, trainer
gc.collect()
torch.cuda.empty_cache()
repo = future.result()
pool.shutdown()
intermediate = Path("/kaggle/temp/YokoCoder-4B-v2-f16.gguf")
subprocess.check_call(
[
sys.executable,
str(repo / "convert_hf_to_gguf.py"),
str(merged),
"--no-mtp",
"--outtype",
"f16",
"--outfile",
str(intermediate),
],
stdout=subprocess.DEVNULL,
)
print("GGUF_F16_READY", flush=True)
output = Path("/kaggle/working/YokoCoder-4B-5epochs-Q4_K_M.gguf")
subprocess.check_call(
[str(repo / "build/bin/llama-quantize"), str(intermediate), str(output), "Q4_K_M", "4"],
stdout=subprocess.DEVNULL,
)
intermediate.unlink(missing_ok=True)
print(f"GGUF_Q4_K_M_READY: {output} ({output.stat().st_size:,} bytes)", flush=True)
# ---------------------------------------------------------------------------
# 6. PUSH UPDATED GGUF & ADAPTER TO HUGGING FACE (`ahpah/YokoCoder-4B`)
# ---------------------------------------------------------------------------
if HF_TOKEN:
print(f"Pushing {output.name} and v2 adapter to Hugging Face ({HF_TARGET_REPO})...", flush=True)
api = HfApi(token=HF_TOKEN)
api.upload_file(
path_or_fileobj=str(output),
path_in_repo="YokoCoder-4B-5epochs-Q4_K_M.gguf",
repo_id=HF_TARGET_REPO,
repo_type="model",
commit_message="Update YokoCoder-4B-5epochs-Q4_K_M.gguf with v2 Structured Reasoning, Bug-Fixing & 2D/3D GameDev fine-tune",
)
api.upload_folder(
folder_path=str(ROOT / "final_adapter"),
path_in_repo="adapters/v2-reasoning-gamedev-bugfix/final_adapter",
repo_id=HF_TARGET_REPO,
repo_type="model",
commit_message="Upload YokoCoder:4B v2 final_adapter",
)
print(f"HF_UPLOAD_COMPLETE: https://huggingface.co/{HF_TARGET_REPO}", flush=True)
else:
print("WARNING: HF_TOKEN not found; GGUF saved to /kaggle/working/YokoCoder-4B-5epochs-Q4_K_M.gguf", flush=True)