Upload train_gptoss.sh with huggingface_hub

train_gptoss.sh

@@ -2,8 +2,8 @@
 set -e
 pip install torch transformers peft trl datasets accelerate sentencepiece huggingface_hub -q 2>&1 | tail -1
 
-TOKEN="${HF_TOKEN:?Set HF_TOKEN first}"
-REPO="${HF_REPO_ID:-HeeBive/ZeroSec}"
+export HF_TOKEN="${HF_TOKEN:?Set HF_TOKEN first}"
+export HF_REPO_ID="${HF_REPO_ID:-HeeBive/ZeroSec}"
 
 python3 << 'PYEOF' > training.log 2>&1
 import os, torch, shutil, json
@@ -30,11 +30,11 @@ if tok.pad_token is None: tok.pad_token = tok.eos_token
 ds = ds.map(lambda x: tok(x["text"], truncation=True, max_length=512), batched=True, remove_columns=ds.column_names)
 
 print("Load model...")
-m = AutoModelForCausalLM.from_pretrained(MODEL, device_map="auto", torch_dtype=torch.bfloat16, trust_remote_code=True)
+m = AutoModelForCausalLM.from_pretrained(MODEL, device_map="auto", torch_dtype=torch.float16, trust_remote_code=True)
 for p in m.parameters(): p.requires_grad = False
 m.gradient_checkpointing_enable()
 m = get_peft_model(m, LoraConfig(r=16, lora_alpha=32, target_modules=["q_proj","k_proj","v_proj","o_proj"], task_type="CAUSAL_LM", bias="none"))
-args = TrainingArguments(output_dir=OUT, per_device_train_batch_size=1, gradient_accumulation_steps=8, num_train_epochs=1, learning_rate=1e-4, bf16=True, logging_steps=5, optim="adamw_torch", report_to="none")
+args = TrainingArguments(output_dir=OUT, per_device_train_batch_size=1, gradient_accumulation_steps=8, num_train_epochs=1, learning_rate=1e-4, fp16=True, logging_steps=5, optim="adamw_torch", report_to="none")
 SFTTrainer(model=m, processing_class=tok, train_dataset=ds, args=args).train()
 m.save_pretrained(OUT); tok.save_pretrained(OUT)
 print("Uploading...")