taketakedaiki commited on
Commit
1cd75a6
·
verified ·
1 Parent(s): a162c78

Upload SFT training script

Browse files
Files changed (1) hide show
  1. sft_cell4_complete.py +256 -0
sft_cell4_complete.py ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import random
3
+ import json
4
+ import shutil
5
+ from dataclasses import dataclass
6
+ from typing import Any, Dict, List, Tuple
7
+ from datasets import load_dataset, Dataset
8
+ from transformers import TrainingArguments, Trainer, TrainerCallback
9
+ import numpy as np
10
+ import torch
11
+ from unsloth import FastLanguageModel
12
+
13
+ def _getenv(name, default):
14
+ return os.environ.get(name, default)
15
+ def _getenv_int(name, default):
16
+ try: return int(os.environ.get(name, str(default)))
17
+ except: return default
18
+ def _getenv_float(name, default):
19
+ try: return float(os.environ.get(name, str(default)))
20
+ except: return default
21
+
22
+ BASE_MODEL_ID = _getenv("SFT_BASE_MODEL", "Qwen/Qwen3-4B-Instruct-2507")
23
+ DATASET_ID = _getenv("SFT_DATASET_ID", "u-10bei/structured_data_with_cot_dataset_512_v4")
24
+ OUT_LORA_DIR = _getenv("SFT_OUT_LORA_DIR", "/kaggle/working/lora_structeval_t_qwen3_4b")
25
+ SEED = _getenv_int("SFT_SEED", 3407)
26
+ VAL_RATIO = _getenv_float("SFT_VAL_RATIO", 0.05)
27
+ MAX_SEQ_LEN = _getenv_int("SFT_MAX_SEQ_LEN", 512)
28
+ LORA_R = _getenv_int("SFT_LORA_R", 64)
29
+ LORA_ALPHA = _getenv_int("SFT_LORA_ALPHA", 128)
30
+ LORA_DROPOUT = _getenv_float("SFT_LORA_DROPOUT", 0)
31
+ LORA_TARGET_MODULES = _getenv("SFT_LORA_TARGET_MODULES", "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj").split(",")
32
+ NUM_TRAIN_EPOCHS = _getenv_int("SFT_EPOCHS", 1)
33
+ PER_DEVICE_TRAIN_BATCH_SIZE = _getenv_int("SFT_PER_DEVICE_TRAIN_BS", 2)
34
+ PER_DEVICE_EVAL_BATCH_SIZE = _getenv_int("SFT_PER_DEVICE_EVAL_BS", 2)
35
+ GRAD_ACCUM = _getenv_int("SFT_GRAD_ACCUM", 8)
36
+ LR = _getenv_float("SFT_LR", 1e-6)
37
+ WARMUP_RATIO = _getenv_float("SFT_WARMUP_RATIO", 0.1)
38
+ MAX_STEPS = _getenv_int("SFT_MAX_STEPS", -1)
39
+ LOGGING_STEPS = _getenv_int("SFT_LOGGING_STEPS", 10)
40
+ EVAL_STEPS = _getenv_int("SFT_EVAL_STEPS", 50)
41
+ SAVE_STEPS = _getenv_int("SFT_SAVE_STEPS", 100)
42
+ SAVE_TOTAL_LIMIT = _getenv_int("SFT_SAVE_TOTAL_LIMIT", 2)
43
+ WEIGHT_DECAY = _getenv_float("SFT_WEIGHT_DECAY", 0.05)
44
+ UPSAMPLE_ENABLE = _getenv("SFT_USE_UPSAMPLING", "0") in ("1","true","True")
45
+ UPSAMPLE_RULES_JSON = _getenv("SFT_UPSAMPLE_RULES", "")
46
+
47
+ def seed_everything(seed):
48
+ random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)
49
+ if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed)
50
+
51
+ seed_everything(SEED)
52
+
53
+ def ensure_openai_messages(ds, msg_col="messages"):
54
+ ex = ds[0].get(msg_col, None)
55
+ if not isinstance(ex, list):
56
+ raise ValueError(f"Dataset must have list-style messages. Got {type(ex)}")
57
+
58
+ def has_any_nonempty_assistant_turn(msgs):
59
+ return any(m.get("role")=="assistant" and str(m.get("content","")).strip()!="" for m in msgs)
60
+
61
+ def ends_with_nonempty_assistant(ex):
62
+ msgs = ex.get("messages", [])
63
+ if not msgs or msgs[-1].get("role")!="assistant": return False
64
+ c = msgs[-1].get("content","")
65
+ return isinstance(c, str) and c.strip()!=""
66
+
67
+ def shuffle_split(ds, val_ratio, seed):
68
+ ds_shuf = ds.shuffle(seed=seed)
69
+ n = len(ds_shuf)
70
+ n_val = max(1, int(round(n * val_ratio)))
71
+ return ds_shuf.select(range(n_val, n)), ds_shuf.select(range(n_val))
72
+
73
+ def make_text_cache_builder(tokenizer):
74
+ def _build(batch):
75
+ full_out, prefix_out, full_len_out, prefix_len_out = [], [], [], []
76
+ for msgs in batch["messages"]:
77
+ full = tokenizer.apply_chat_template(msgs, tokenize=False, add_generation_prompt=False)
78
+ prefix = tokenizer.apply_chat_template(msgs[:-1], tokenize=False, add_generation_prompt=True)
79
+ full_out.append(full); prefix_out.append(prefix)
80
+ full_ids = tokenizer(full, add_special_tokens=False, truncation=False)["input_ids"]
81
+ prefix_ids = tokenizer(prefix, add_special_tokens=False, truncation=False)["input_ids"]
82
+ full_len_out.append(len(full_ids)); prefix_len_out.append(len(prefix_ids))
83
+ return {"full_text": full_out, "prefix_text": prefix_out, "full_input_ids_len": full_len_out, "prefix_input_ids_len": prefix_len_out}
84
+ return _build
85
+
86
+ MASK_COT = _getenv("SFT_MASK_COT", "1") in ("1","true","True")
87
+ OUTPUT_MARKERS = [s.strip() for s in _getenv("SFT_OUTPUT_MARKERS", "Output:,OUTPUT:,Final:,Answer:,Result:,Response:").split(",") if s.strip()]
88
+ OUTPUT_LEARN_MODE = _getenv("SFT_OUTPUT_LEARN_MODE", "after_marker")
89
+
90
+ @dataclass
91
+ class AssistantOnlyCollatorCached:
92
+ tokenizer: Any
93
+ max_length: int = MAX_SEQ_LEN
94
+
95
+ def _find_subseq(self, seq, sub):
96
+ if not sub or len(sub) > len(seq): return -1
97
+ for i in range(len(seq) - len(sub) + 1):
98
+ if seq[i:i+len(sub)] == sub: return i
99
+ return -1
100
+
101
+ def __call__(self, batch):
102
+ tok = self.tokenizer
103
+ full_texts = [ex["full_text"] for ex in batch]
104
+ prefix_texts = [ex["prefix_text"] for ex in batch]
105
+ old_trunc = getattr(tok, "truncation_side", "right")
106
+ old_pad = getattr(tok, "padding_side", "right")
107
+ tok.truncation_side = "left"; tok.padding_side = "right"
108
+ try:
109
+ enc = tok(full_texts, return_tensors="pt", padding=True, truncation=True, max_length=self.max_length, add_special_tokens=False)
110
+ input_ids = enc["input_ids"]; attention_mask = enc["attention_mask"]
111
+ labels = torch.full_like(input_ids, fill_value=-100)
112
+ full_ids_nt = tok(full_texts, return_tensors=None, padding=False, truncation=False, add_special_tokens=False)["input_ids"]
113
+ prefix_ids_nt = tok(prefix_texts, return_tensors=None, padding=False, truncation=False, add_special_tokens=False)["input_ids"]
114
+ marker_seqs = []
115
+ if MASK_COT and OUTPUT_MARKERS:
116
+ for m in OUTPUT_MARKERS:
117
+ mid = tok(m, add_special_tokens=False, truncation=False)["input_ids"]
118
+ if not mid: continue
119
+ mid_nl = tok(m+"\n", add_special_tokens=False, truncation=False)["input_ids"]
120
+ marker_seqs.append((mid, mid_nl))
121
+ for i in range(input_ids.size(0)):
122
+ trunc_left = max(0, len(full_ids_nt[i]) - self.max_length)
123
+ boundary = len(prefix_ids_nt[i]) - trunc_left
124
+ full_len_tr = int(attention_mask[i].sum().item())
125
+ if boundary <= 0 or boundary >= full_len_tr: continue
126
+ span_start = boundary; span_end = full_len_tr; learn_start = span_start
127
+ if MASK_COT and marker_seqs:
128
+ visible_ids = input_ids[i, :full_len_tr].tolist()
129
+ assistant_ids = visible_ids[span_start:span_end]
130
+ best_out = None
131
+ for mid, mid_nl in marker_seqs:
132
+ p = self._find_subseq(assistant_ids, mid_nl)
133
+ if p != -1:
134
+ out_pos = span_start + p; after_pos = out_pos + len(mid_nl)
135
+ else:
136
+ p = self._find_subseq(assistant_ids, mid)
137
+ if p == -1: continue
138
+ out_pos = span_start + p; after_pos = out_pos + len(mid)
139
+ if best_out is None or out_pos < best_out[0]: best_out = (out_pos, after_pos)
140
+ if best_out is not None:
141
+ out_pos, after_pos = best_out
142
+ learn_start = after_pos if OUTPUT_LEARN_MODE != "from_marker" else out_pos
143
+ learn_start = max(span_start, min(learn_start, span_end))
144
+ if learn_start < span_end:
145
+ labels[i, learn_start:span_end] = input_ids[i, learn_start:span_end]
146
+ labels[attention_mask == 0] = -100
147
+ return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}
148
+ finally:
149
+ tok.truncation_side = old_trunc; tok.padding_side = old_pad
150
+
151
+ @torch.no_grad()
152
+ def filter_has_supervision(ds, collator):
153
+ keep = []
154
+ for i in range(len(ds)):
155
+ out = collator([ds[i]])
156
+ if (out["labels"][0] != -100).sum().item() > 0: keep.append(i)
157
+ return ds.select(keep)
158
+
159
+ def count_all_masked(ds, collator, n=200, seed=3407):
160
+ rng = random.Random(seed); n = min(n, len(ds))
161
+ idxs = [rng.randrange(0, len(ds)) for _ in range(n)]
162
+ all_masked = 0
163
+ for i in idxs:
164
+ out = collator([ds[i]])
165
+ if (out["labels"][0] != -100).sum().item() == 0: all_masked += 1
166
+ print(f"[CHECK] all-masked in {n}: {all_masked} ({all_masked/max(1,n):.1%})")
167
+
168
+ def apply_upsampling(train_ds):
169
+ if not UPSAMPLE_ENABLE or not UPSAMPLE_RULES_JSON: return train_ds
170
+ try:
171
+ rules = json.loads(UPSAMPLE_RULES_JSON)
172
+ if not isinstance(rules, dict) or not rules: return train_ds
173
+ except: return train_ds
174
+ packs = train_ds["subcategory"] if "subcategory" in train_ds.column_names else [None]*len(train_ds)
175
+ pack_field = train_ds["pack"] if "pack" in train_ds.column_names else [None]*len(train_ds)
176
+ w = []
177
+ for sub, pk in zip(packs, pack_field):
178
+ wt = 1.0; ss = str(sub or ""); sp = str(pk or "")
179
+ for pat, mult in rules.items():
180
+ try: m = float(mult)
181
+ except: m = 1.0
182
+ if pat.startswith("pack:"):
183
+ if sp == pat.split(":",1)[1]: wt *= max(0.0, m)
184
+ else:
185
+ if pat in ss: wt *= max(0.0, m)
186
+ w.append(wt)
187
+ w = np.asarray(w, dtype=np.float64)
188
+ if (w <= 0).all() or w.sum() == 0: return train_ds
189
+ p = w / w.sum(); n = len(train_ds)
190
+ idx = np.random.choice(np.arange(n), size=n, replace=True, p=p)
191
+ return train_ds.select(idx.tolist())
192
+
193
+ class LabelStatsCallback(TrainerCallback):
194
+ def __init__(self, dataset, collator, name="train", every_n_steps=100):
195
+ self.dataset, self.collator, self.name, self.every_n_steps = dataset, collator, name, every_n_steps
196
+ @torch.no_grad()
197
+ def on_step_end(self, args, state, control, **kwargs):
198
+ if (state.global_step % self.every_n_steps) == 0:
199
+ batch = [self.dataset[random.randint(0, len(self.dataset)-1)] for _ in range(8)]
200
+ out = self.collator(batch)
201
+ valid = (out["labels"] != -100).sum().item()
202
+ total = (out["attention_mask"] == 1).sum().item()
203
+ print(f"\n[LabelStats:{self.name}] step={state.global_step} valid_ratio={valid/max(1,total):.4f}")
204
+
205
+ def main():
206
+ os.makedirs(OUT_LORA_DIR, exist_ok=True)
207
+ print(f"[INFO] Loading dataset: {DATASET_ID}")
208
+ ds_all = load_dataset(DATASET_ID, split="train")
209
+ ensure_openai_messages(ds_all)
210
+ ds_all = ds_all.filter(lambda ex: has_any_nonempty_assistant_turn(ex["messages"]))
211
+ ds_all = ds_all.filter(ends_with_nonempty_assistant)
212
+ train_ds, val_ds = shuffle_split(ds_all, VAL_RATIO, SEED)
213
+ train_ds = apply_upsampling(train_ds)
214
+ print("[INFO] Loading base model:", BASE_MODEL_ID)
215
+ model, tokenizer = FastLanguageModel.from_pretrained(
216
+ model_name=BASE_MODEL_ID, max_seq_length=MAX_SEQ_LEN, dtype=None, load_in_4bit=True)
217
+ build_cache = make_text_cache_builder(tokenizer)
218
+ train_ds = train_ds.map(build_cache, batched=True, num_proc=1, desc="Caching train")
219
+ val_ds = val_ds.map(build_cache, batched=True, num_proc=1, desc="Caching val")
220
+ model = FastLanguageModel.get_peft_model(
221
+ model, r=LORA_R, target_modules=LORA_TARGET_MODULES,
222
+ lora_alpha=LORA_ALPHA, lora_dropout=LORA_DROPOUT,
223
+ use_gradient_checkpointing="unsloth", random_state=SEED)
224
+ args = TrainingArguments(
225
+ output_dir=OUT_LORA_DIR, num_train_epochs=NUM_TRAIN_EPOCHS,
226
+ per_device_train_batch_size=PER_DEVICE_TRAIN_BATCH_SIZE,
227
+ per_device_eval_batch_size=PER_DEVICE_EVAL_BATCH_SIZE,
228
+ gradient_accumulation_steps=GRAD_ACCUM, learning_rate=LR,
229
+ warmup_ratio=WARMUP_RATIO, lr_scheduler_type="cosine",
230
+ weight_decay=WEIGHT_DECAY, logging_steps=LOGGING_STEPS,
231
+ eval_strategy="steps", eval_steps=EVAL_STEPS,
232
+ save_strategy="steps", save_steps=SAVE_STEPS,
233
+ save_total_limit=SAVE_TOTAL_LIMIT, max_steps=MAX_STEPS,
234
+ bf16=False, fp16=True, push_to_hub=False, report_to="none",
235
+ group_by_length=False, remove_unused_columns=False)
236
+ collator = AssistantOnlyCollatorCached(tokenizer=tokenizer, max_length=MAX_SEQ_LEN)
237
+ print("[INFO] Checking all-masked before filtering...")
238
+ count_all_masked(val_ds, collator, n=len(val_ds), seed=SEED)
239
+ print("[INFO] Filtering train/val...")
240
+ train_ds = filter_has_supervision(train_ds, collator)
241
+ val_ds = filter_has_supervision(val_ds, collator)
242
+ print("[INFO] New sizes: train =", len(train_ds), "val =", len(val_ds))
243
+ count_all_masked(val_ds, collator, n=len(val_ds), seed=SEED)
244
+ trainer = Trainer(
245
+ model=model, args=args, train_dataset=train_ds, eval_dataset=val_ds,
246
+ data_collator=collator, tokenizer=tokenizer)
247
+ trainer.add_callback(LabelStatsCallback(train_ds, collator, name="train", every_n_steps=LOGGING_STEPS))
248
+ print("[INFO] Starting training...")
249
+ trainer.train()
250
+ print("[INFO] Saving adapter & tokenizer...")
251
+ model.save_pretrained(OUT_LORA_DIR)
252
+ tokenizer.save_pretrained(OUT_LORA_DIR)
253
+ print(f"[INFO] Done. Saved to {OUT_LORA_DIR}")
254
+
255
+ if __name__ == "__main__":
256
+ main()