diff --git a/.gitattributes b/.gitattributes
index 857b422b0ed5ffae087d4f2d1d008c1b4cf004ce..7e35f53ef70bc50484bed9b3e3ff12d30108e836 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -81,3 +81,28 @@ qwen3/hpd/checkpoint-314/tokenizer.json filter=lfs diff=lfs merge=lfs -text
qwen3/hpd/checkpoint-628/tokenizer.json filter=lfs diff=lfs merge=lfs -text
qwen3/hpd/checkpoint-942/tokenizer.json filter=lfs diff=lfs merge=lfs -text
qwen3/hpd/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/rkl/checkpoint-1256/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/rkl/checkpoint-1570/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/rkl/checkpoint-314/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/rkl/checkpoint-628/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/rkl/checkpoint-942/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/rkl/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/sfkl/checkpoint-1256/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/sfkl/checkpoint-1570/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/sfkl/checkpoint-314/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/sfkl/checkpoint-628/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/sfkl/checkpoint-942/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/sfkl/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/sft/checkpoint-1256/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/sft/checkpoint-1570/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/sft/checkpoint-314/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/sft/checkpoint-628/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/sft/checkpoint-942/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/sft/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/srkl/checkpoint-1256/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/srkl/checkpoint-1570/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/srkl/checkpoint-314/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/srkl/checkpoint-628/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/srkl/checkpoint-942/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/srkl/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen3/teacher_lora/checkpoint-1256/tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/qwen3/hpd/trainer_state.json b/qwen3/hpd/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c1309b84e2d48065e370506f8519837c5d59dcc
--- /dev/null
+++ b/qwen3/hpd/trainer_state.json
@@ -0,0 +1,1909 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 5.0,
+ "eval_steps": 500,
+ "global_step": 1570,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "adv1_neg_pct": 0.2358350306749344,
+ "adv2_active_pct": 17.716575622558594,
+ "ce_loss": 0.6496895551681519,
+ "epoch": 0.06382130035899482,
+ "grad_norm": 4.9955339431762695,
+ "hpd_loss": 0.7524586915969849,
+ "k1_gt_raw_max": 0.3395599126815796,
+ "k1_gt_raw_mean": 0.08363302797079086,
+ "k1_gt_raw_min": -0.006161125842481852,
+ "k1_gt_raw_neg_pct": 0.2358350306749344,
+ "k1_sample_max": -1.2287571430206299,
+ "k1_sample_mean": -5.443553924560547,
+ "k1_sample_min": -17.516345977783203,
+ "k1_sample_neg_pct": 97.79017639160156,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.7524587631225585,
+ "mask3_pct": 17.716575622558594,
+ "neg_student_nll_loss": 0.6496895551681519,
+ "sampled_diff_pct": 17.718591690063477,
+ "step": 20
+ },
+ {
+ "adv1_neg_pct": 0.24569129943847656,
+ "adv2_active_pct": 6.35067081451416,
+ "ce_loss": 0.12904959917068481,
+ "epoch": 0.12764260071798963,
+ "grad_norm": 3.5179519653320312,
+ "hpd_loss": 0.07163389772176743,
+ "k1_gt_raw_max": 0.3387124538421631,
+ "k1_gt_raw_mean": 0.03725019469857216,
+ "k1_gt_raw_min": -0.009099474176764488,
+ "k1_gt_raw_neg_pct": 0.24569129943847656,
+ "k1_sample_max": -1.1939605474472046,
+ "k1_sample_mean": -4.561788082122803,
+ "k1_sample_min": -10.153608322143555,
+ "k1_sample_neg_pct": 96.875,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.07163387537002563,
+ "mask3_pct": 6.35067081451416,
+ "neg_student_nll_loss": 0.12904959917068481,
+ "sampled_diff_pct": 6.35067081451416,
+ "step": 40
+ },
+ {
+ "adv1_neg_pct": 0.18458101153373718,
+ "adv2_active_pct": 5.008382320404053,
+ "ce_loss": 0.10459579527378082,
+ "epoch": 0.19146390107698444,
+ "grad_norm": 2.890872001647949,
+ "hpd_loss": 0.09348365664482117,
+ "k1_gt_raw_max": 0.3109760880470276,
+ "k1_gt_raw_mean": 0.026082728058099747,
+ "k1_gt_raw_min": -0.01705370843410492,
+ "k1_gt_raw_neg_pct": 0.18458101153373718,
+ "k1_sample_max": -2.0660922527313232,
+ "k1_sample_mean": -4.749420166015625,
+ "k1_sample_min": -8.562150955200195,
+ "k1_sample_neg_pct": 89.6875,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.09348366260528565,
+ "mask3_pct": 5.008382320404053,
+ "neg_student_nll_loss": 0.10459579527378082,
+ "sampled_diff_pct": 5.008382320404053,
+ "step": 60
+ },
+ {
+ "adv1_neg_pct": 0.3009452819824219,
+ "adv2_active_pct": 3.9314260482788086,
+ "ce_loss": 0.08063317835330963,
+ "epoch": 0.25528520143597927,
+ "grad_norm": 2.033250570297241,
+ "hpd_loss": 0.08107596635818481,
+ "k1_gt_raw_max": 0.2881737947463989,
+ "k1_gt_raw_mean": 0.019415946677327156,
+ "k1_gt_raw_min": -0.008919830434024334,
+ "k1_gt_raw_neg_pct": 0.3009452819824219,
+ "k1_sample_max": -1.6159262657165527,
+ "k1_sample_mean": -3.7506155967712402,
+ "k1_sample_min": -6.360409259796143,
+ "k1_sample_neg_pct": 86.39583587646484,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.0810759723186493,
+ "mask3_pct": 3.9314260482788086,
+ "neg_student_nll_loss": 0.08063317835330963,
+ "sampled_diff_pct": 3.9378738403320312,
+ "step": 80
+ },
+ {
+ "adv1_neg_pct": 0.22911369800567627,
+ "adv2_active_pct": 2.694115161895752,
+ "ce_loss": 0.052447859197854996,
+ "epoch": 0.3191065017949741,
+ "grad_norm": 1.751448392868042,
+ "hpd_loss": 0.027988094836473465,
+ "k1_gt_raw_max": 0.3031776547431946,
+ "k1_gt_raw_mean": 0.01864592172205448,
+ "k1_gt_raw_min": -0.007533406838774681,
+ "k1_gt_raw_neg_pct": 0.22911369800567627,
+ "k1_sample_max": -1.57374906539917,
+ "k1_sample_mean": -3.1450629234313965,
+ "k1_sample_min": -5.145340919494629,
+ "k1_sample_neg_pct": 84.70052337646484,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.027988094091415405,
+ "mask3_pct": 2.6920177936553955,
+ "neg_student_nll_loss": 0.052447859197854996,
+ "sampled_diff_pct": 2.7128100395202637,
+ "step": 100
+ },
+ {
+ "adv1_neg_pct": 0.1952832043170929,
+ "adv2_active_pct": 3.306915044784546,
+ "ce_loss": 0.04980571195483208,
+ "epoch": 0.3829278021539689,
+ "grad_norm": 3.930035352706909,
+ "hpd_loss": 0.015151199884712696,
+ "k1_gt_raw_max": 0.28564560413360596,
+ "k1_gt_raw_mean": 0.01884515956044197,
+ "k1_gt_raw_min": -0.001684204675257206,
+ "k1_gt_raw_neg_pct": 0.1952832043170929,
+ "k1_sample_max": -1.9181127548217773,
+ "k1_sample_mean": -3.2553763389587402,
+ "k1_sample_min": -4.8449602127075195,
+ "k1_sample_neg_pct": 75.20833587646484,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.015151196718215942,
+ "mask3_pct": 3.306915044784546,
+ "neg_student_nll_loss": 0.04980571195483208,
+ "sampled_diff_pct": 3.3092129230499268,
+ "step": 120
+ },
+ {
+ "adv1_neg_pct": 0.3264433741569519,
+ "adv2_active_pct": 2.25433087348938,
+ "ce_loss": 0.04018855467438698,
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.6639359593391418,
+ "hpd_loss": 0.02619136869907379,
+ "k1_gt_raw_max": 0.24626323580741882,
+ "k1_gt_raw_mean": 0.015010751783847809,
+ "k1_gt_raw_min": -0.004701526835560799,
+ "k1_gt_raw_neg_pct": 0.3264433741569519,
+ "k1_sample_max": -1.902454137802124,
+ "k1_sample_mean": -2.805896282196045,
+ "k1_sample_min": -3.829012393951416,
+ "k1_sample_neg_pct": 71.25,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.026191365718841553,
+ "mask3_pct": 2.25433087348938,
+ "neg_student_nll_loss": 0.04018855467438698,
+ "sampled_diff_pct": 2.2565953731536865,
+ "step": 140
+ },
+ {
+ "adv1_neg_pct": 0.3137112259864807,
+ "adv2_active_pct": 1.3726557493209839,
+ "ce_loss": 0.03148110955953598,
+ "epoch": 0.5105704028719585,
+ "grad_norm": 1.548140525817871,
+ "hpd_loss": 0.03039727918803692,
+ "k1_gt_raw_max": 0.20344290137290955,
+ "k1_gt_raw_mean": 0.009032711386680603,
+ "k1_gt_raw_min": -0.0009857998229563236,
+ "k1_gt_raw_neg_pct": 0.3137112259864807,
+ "k1_sample_max": -2.153831958770752,
+ "k1_sample_mean": -2.81235933303833,
+ "k1_sample_min": -3.547999858856201,
+ "k1_sample_neg_pct": 58.75,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.03039727509021759,
+ "mask3_pct": 1.3726557493209839,
+ "neg_student_nll_loss": 0.03148110955953598,
+ "sampled_diff_pct": 1.3726557493209839,
+ "step": 160
+ },
+ {
+ "adv1_neg_pct": 0.4895250201225281,
+ "adv2_active_pct": 1.1766502857208252,
+ "ce_loss": 0.02162190154194832,
+ "epoch": 0.5743917032309533,
+ "grad_norm": 2.528654098510742,
+ "hpd_loss": 0.011955421417951584,
+ "k1_gt_raw_max": 0.20187309384346008,
+ "k1_gt_raw_mean": 0.008551979437470436,
+ "k1_gt_raw_min": -0.0070910281501710415,
+ "k1_gt_raw_neg_pct": 0.4895250201225281,
+ "k1_sample_max": -1.979987621307373,
+ "k1_sample_mean": -2.5711493492126465,
+ "k1_sample_min": -3.2561821937561035,
+ "k1_sample_neg_pct": 56.09375,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.011955419182777404,
+ "mask3_pct": 1.1766502857208252,
+ "neg_student_nll_loss": 0.02162190154194832,
+ "sampled_diff_pct": 1.185581922531128,
+ "step": 180
+ },
+ {
+ "adv1_neg_pct": 0.5023188591003418,
+ "adv2_active_pct": 1.5687706470489502,
+ "ce_loss": 0.027880553156137466,
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.9650227427482605,
+ "hpd_loss": 0.010340285487473011,
+ "k1_gt_raw_max": 0.2007327377796173,
+ "k1_gt_raw_mean": 0.009667925536632538,
+ "k1_gt_raw_min": -0.01589183136820793,
+ "k1_gt_raw_neg_pct": 0.5023188591003418,
+ "k1_sample_max": -1.5354162454605103,
+ "k1_sample_mean": -2.1268630027770996,
+ "k1_sample_min": -2.771559238433838,
+ "k1_sample_neg_pct": 56.40625,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.010340283811092376,
+ "mask3_pct": 1.5687706470489502,
+ "neg_student_nll_loss": 0.027880553156137466,
+ "sampled_diff_pct": 1.5725816488265991,
+ "step": 200
+ },
+ {
+ "adv1_neg_pct": 0.48300591111183167,
+ "adv2_active_pct": 1.1448569297790527,
+ "ce_loss": 0.023295864462852478,
+ "epoch": 0.702034303948943,
+ "grad_norm": 2.59177303314209,
+ "hpd_loss": 0.019972147420048714,
+ "k1_gt_raw_max": 0.18090441823005676,
+ "k1_gt_raw_mean": 0.0075406162068247795,
+ "k1_gt_raw_min": -0.014693280681967735,
+ "k1_gt_raw_neg_pct": 0.48300591111183167,
+ "k1_sample_max": -1.6416490077972412,
+ "k1_sample_mean": -2.2296323776245117,
+ "k1_sample_min": -2.930915355682373,
+ "k1_sample_neg_pct": 48.515625,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.01997213959693909,
+ "mask3_pct": 1.1448569297790527,
+ "neg_student_nll_loss": 0.023295864462852478,
+ "sampled_diff_pct": 1.154510736465454,
+ "step": 220
+ },
+ {
+ "adv1_neg_pct": 0.765082597732544,
+ "adv2_active_pct": 0.9244414567947388,
+ "ce_loss": 0.02368738129734993,
+ "epoch": 0.7658556043079378,
+ "grad_norm": 1.1686080694198608,
+ "hpd_loss": 0.02641882374882698,
+ "k1_gt_raw_max": 0.1654866635799408,
+ "k1_gt_raw_mean": 0.005874675698578358,
+ "k1_gt_raw_min": -0.009244760498404503,
+ "k1_gt_raw_neg_pct": 0.765082597732544,
+ "k1_sample_max": -1.3134702444076538,
+ "k1_sample_mean": -1.9339661598205566,
+ "k1_sample_min": -2.6181559562683105,
+ "k1_sample_neg_pct": 45.3125,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.026418817043304444,
+ "mask3_pct": 0.9244414567947388,
+ "neg_student_nll_loss": 0.02368738129734993,
+ "sampled_diff_pct": 0.9244414567947388,
+ "step": 240
+ },
+ {
+ "adv1_neg_pct": 0.8438892364501953,
+ "adv2_active_pct": 0.929736852645874,
+ "ce_loss": 0.020094729959964752,
+ "epoch": 0.8296769046669326,
+ "grad_norm": 1.2779905796051025,
+ "hpd_loss": 0.015618223696947098,
+ "k1_gt_raw_max": 0.16853322088718414,
+ "k1_gt_raw_mean": 0.006190822925418615,
+ "k1_gt_raw_min": -0.009831264615058899,
+ "k1_gt_raw_neg_pct": 0.8438892364501953,
+ "k1_sample_max": -1.3172353506088257,
+ "k1_sample_mean": -1.7796552181243896,
+ "k1_sample_min": -2.261643409729004,
+ "k1_sample_neg_pct": 44.95738983154297,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.015618222951889037,
+ "mask3_pct": 0.9239488840103149,
+ "neg_student_nll_loss": 0.020094729959964752,
+ "sampled_diff_pct": 0.9378187656402588,
+ "step": 260
+ },
+ {
+ "adv1_neg_pct": 0.9945487976074219,
+ "adv2_active_pct": 0.7392160892486572,
+ "ce_loss": 0.015887977555394173,
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.26237931847572327,
+ "hpd_loss": 0.015452454797923565,
+ "k1_gt_raw_max": 0.14525339007377625,
+ "k1_gt_raw_mean": 0.0046372562646865845,
+ "k1_gt_raw_min": -0.003686224576085806,
+ "k1_gt_raw_neg_pct": 0.9945487976074219,
+ "k1_sample_max": -1.206074595451355,
+ "k1_sample_mean": -1.4332499504089355,
+ "k1_sample_min": -1.671927571296692,
+ "k1_sample_neg_pct": 35.625,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.015452450513839722,
+ "mask3_pct": 0.7392160892486572,
+ "neg_student_nll_loss": 0.015887977555394173,
+ "sampled_diff_pct": 0.7392160892486572,
+ "step": 280
+ },
+ {
+ "adv1_neg_pct": 0.9700182676315308,
+ "adv2_active_pct": 0.6018879413604736,
+ "ce_loss": 0.014513559639453888,
+ "epoch": 0.9573195053849222,
+ "grad_norm": 2.0338079929351807,
+ "hpd_loss": 0.013913260772824287,
+ "k1_gt_raw_max": 0.13846856355667114,
+ "k1_gt_raw_mean": 0.004541991278529167,
+ "k1_gt_raw_min": -0.002533096820116043,
+ "k1_gt_raw_neg_pct": 0.9700182676315308,
+ "k1_sample_max": -1.2417628765106201,
+ "k1_sample_mean": -1.4318351745605469,
+ "k1_sample_min": -1.6163195371627808,
+ "k1_sample_neg_pct": 32.864585876464844,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.013913258910179138,
+ "mask3_pct": 0.6018879413604736,
+ "neg_student_nll_loss": 0.014513559639453888,
+ "sampled_diff_pct": 0.607985258102417,
+ "step": 300
+ },
+ {
+ "adv1_neg_pct": 0.8508870601654053,
+ "adv2_active_pct": 0.859713077545166,
+ "ce_loss": 0.019685909152030945,
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 86.6822,
+ "eval_generator_rouge1": 99.4006,
+ "eval_generator_rouge2": 99.0209,
+ "eval_generator_rougeL": 98.951,
+ "eval_loss": 0.010082065127789974,
+ "eval_runtime": 595.0313,
+ "eval_samples_per_second": 5.738,
+ "eval_selector_accuracy": 93.9836,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.18,
+ "hpd_loss": 0.02267773076891899,
+ "k1_gt_raw_max": 0.14172008633613586,
+ "k1_gt_raw_mean": 0.0054550133645534515,
+ "k1_gt_raw_min": -0.005414052866399288,
+ "k1_gt_raw_neg_pct": 0.8508870601654053,
+ "k1_sample_max": -1.4249026775360107,
+ "k1_sample_mean": -1.6359399557113647,
+ "k1_sample_min": -1.8393499851226807,
+ "k1_sample_neg_pct": 36.4485969543457,
+ "mask3_pct": 0.859713077545166,
+ "neg_student_nll_loss": 0.019685909152030945,
+ "sampled_diff_pct": 0.859713077545166,
+ "step": 314
+ },
+ {
+ "adv1_neg_pct": 0.6673219203948975,
+ "adv2_active_pct": 1.2173185348510742,
+ "ce_loss": 0.011882005259394646,
+ "epoch": 1.0191463901076985,
+ "grad_norm": 4.386560916900635,
+ "hpd_loss": -0.0019697409588843584,
+ "k1_gt_raw_max": 0.14985336363315582,
+ "k1_gt_raw_mean": 0.006570600904524326,
+ "k1_gt_raw_min": -0.0011412458261474967,
+ "k1_gt_raw_neg_pct": 0.6673219203948975,
+ "k1_sample_max": -0.5733932256698608,
+ "k1_sample_mean": -0.7650949358940125,
+ "k1_sample_min": -0.9706472158432007,
+ "k1_sample_neg_pct": 36.458335876464844,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.014574740827083588,
+ "mask3_pct": 1.2173185348510742,
+ "neg_student_nll_loss": 0.011882005259394646,
+ "sampled_diff_pct": 1.2173185348510742,
+ "step": 320
+ },
+ {
+ "adv1_neg_pct": 1.1466679573059082,
+ "adv2_active_pct": 0.4476942718029022,
+ "ce_loss": 0.009723825380206108,
+ "epoch": 1.0829676904666932,
+ "grad_norm": 1.2898244857788086,
+ "hpd_loss": 0.009684150107204914,
+ "k1_gt_raw_max": 0.10980867594480515,
+ "k1_gt_raw_mean": 0.0032046493142843246,
+ "k1_gt_raw_min": -0.009925473481416702,
+ "k1_gt_raw_neg_pct": 1.1466679573059082,
+ "k1_sample_max": -0.76816326379776,
+ "k1_sample_mean": -0.8626709580421448,
+ "k1_sample_min": -0.9573334455490112,
+ "k1_sample_neg_pct": 28.95833396911621,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.0096841499209404,
+ "mask3_pct": 0.4476942718029022,
+ "neg_student_nll_loss": 0.009723825380206108,
+ "sampled_diff_pct": 0.4520599842071533,
+ "step": 340
+ },
+ {
+ "adv1_neg_pct": 1.657383918762207,
+ "adv2_active_pct": 0.7416136860847473,
+ "ce_loss": 0.01363883912563324,
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.8189982175827026,
+ "hpd_loss": 0.012504098936915398,
+ "k1_gt_raw_max": 0.11255955696105957,
+ "k1_gt_raw_mean": 0.0030380175448954105,
+ "k1_gt_raw_min": -0.01135759986937046,
+ "k1_gt_raw_neg_pct": 1.657383918762207,
+ "k1_sample_max": -1.1351618766784668,
+ "k1_sample_mean": -1.3799619674682617,
+ "k1_sample_min": -1.6486986875534058,
+ "k1_sample_neg_pct": 30.0,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.012504099309444428,
+ "mask3_pct": 0.7416136860847473,
+ "neg_student_nll_loss": 0.01363883912563324,
+ "sampled_diff_pct": 0.7438143491744995,
+ "step": 360
+ },
+ {
+ "adv1_neg_pct": 1.3510011434555054,
+ "adv2_active_pct": 0.3445201814174652,
+ "ce_loss": 0.007902171462774277,
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.5892553329467773,
+ "hpd_loss": 0.008744428865611553,
+ "k1_gt_raw_max": 0.09419713914394379,
+ "k1_gt_raw_mean": 0.0022562379017472267,
+ "k1_gt_raw_min": -0.006403174251317978,
+ "k1_gt_raw_neg_pct": 1.3510011434555054,
+ "k1_sample_max": -0.583869457244873,
+ "k1_sample_mean": -0.7341683506965637,
+ "k1_sample_min": -0.8994753360748291,
+ "k1_sample_neg_pct": 23.75,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.008744429051876067,
+ "mask3_pct": 0.3445201814174652,
+ "neg_student_nll_loss": 0.007902171462774277,
+ "sampled_diff_pct": 0.35682976245880127,
+ "step": 380
+ },
+ {
+ "adv1_neg_pct": 1.4141113758087158,
+ "adv2_active_pct": 0.3374101221561432,
+ "ce_loss": 0.008193380199372768,
+ "epoch": 1.2744315915436777,
+ "grad_norm": 1.2929012775421143,
+ "hpd_loss": 0.006689602509140968,
+ "k1_gt_raw_max": 0.10671139508485794,
+ "k1_gt_raw_mean": 0.002954404801130295,
+ "k1_gt_raw_min": -0.006222248077392578,
+ "k1_gt_raw_neg_pct": 1.4141114950180054,
+ "k1_sample_max": -0.6661227941513062,
+ "k1_sample_mean": -0.7585095763206482,
+ "k1_sample_min": -0.8607568740844727,
+ "k1_sample_neg_pct": 25.760417938232422,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.0066896028816699985,
+ "mask3_pct": 0.3344895839691162,
+ "neg_student_nll_loss": 0.008193380199372768,
+ "sampled_diff_pct": 0.3449546694755554,
+ "step": 400
+ },
+ {
+ "adv1_neg_pct": 2.1220993995666504,
+ "adv2_active_pct": 0.4313049912452698,
+ "ce_loss": 0.011112527921795845,
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.8164143562316895,
+ "hpd_loss": 0.010808300226926804,
+ "k1_gt_raw_max": 0.10569541901350021,
+ "k1_gt_raw_mean": 0.0035689054057002068,
+ "k1_gt_raw_min": -0.004470471758395433,
+ "k1_gt_raw_neg_pct": 2.1220993995666504,
+ "k1_sample_max": -0.6986867189407349,
+ "k1_sample_mean": -0.7543543577194214,
+ "k1_sample_min": -0.8270018100738525,
+ "k1_sample_neg_pct": 25.279605865478516,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.010808297991752624,
+ "mask3_pct": 0.4313049912452698,
+ "neg_student_nll_loss": 0.011112527921795845,
+ "sampled_diff_pct": 0.4342671036720276,
+ "step": 420
+ },
+ {
+ "adv1_neg_pct": 1.9999275207519531,
+ "adv2_active_pct": 0.2967550456523895,
+ "ce_loss": 0.007173327729105949,
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.959374189376831,
+ "hpd_loss": 0.005844146944582462,
+ "k1_gt_raw_max": 0.08694229274988174,
+ "k1_gt_raw_mean": 0.002139701507985592,
+ "k1_gt_raw_min": -0.005928537808358669,
+ "k1_gt_raw_neg_pct": 1.9999275207519531,
+ "k1_sample_max": -0.8034830093383789,
+ "k1_sample_mean": -0.8980427980422974,
+ "k1_sample_min": -0.9926026463508606,
+ "k1_sample_neg_pct": 20.78125,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.005844147875905037,
+ "mask3_pct": 0.2967550456523895,
+ "neg_student_nll_loss": 0.007173327729105949,
+ "sampled_diff_pct": 0.30670204758644104,
+ "step": 440
+ },
+ {
+ "adv1_neg_pct": 1.9445667266845703,
+ "adv2_active_pct": 0.28356558084487915,
+ "ce_loss": 0.007787229493260384,
+ "epoch": 1.4658954926206622,
+ "grad_norm": 1.3586090803146362,
+ "hpd_loss": 0.009073459543287754,
+ "k1_gt_raw_max": 0.07425311952829361,
+ "k1_gt_raw_mean": 0.0016224482096731663,
+ "k1_gt_raw_min": -0.0022226697765290737,
+ "k1_gt_raw_neg_pct": 1.9445667266845703,
+ "k1_sample_max": -0.6454492807388306,
+ "k1_sample_mean": -0.686149001121521,
+ "k1_sample_min": -0.7300035953521729,
+ "k1_sample_neg_pct": 17.1875,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.009073460102081298,
+ "mask3_pct": 0.27911925315856934,
+ "neg_student_nll_loss": 0.007787229493260384,
+ "sampled_diff_pct": 0.28356558084487915,
+ "step": 460
+ },
+ {
+ "adv1_neg_pct": 1.8402546644210815,
+ "adv2_active_pct": 0.41872814297676086,
+ "ce_loss": 0.005175907164812088,
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.2023886740207672,
+ "hpd_loss": 0.0015706174308434129,
+ "k1_gt_raw_max": 0.07684825360774994,
+ "k1_gt_raw_mean": 0.0022208630107343197,
+ "k1_gt_raw_min": -0.008054896257817745,
+ "k1_gt_raw_neg_pct": 1.8402546644210815,
+ "k1_sample_max": -0.47196078300476074,
+ "k1_sample_mean": -0.4973880648612976,
+ "k1_sample_min": -0.5228154063224792,
+ "k1_sample_neg_pct": 17.5,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.0015706172212958335,
+ "mask3_pct": 0.41872814297676086,
+ "neg_student_nll_loss": 0.005175907164812088,
+ "sampled_diff_pct": 0.4208254814147949,
+ "step": 480
+ },
+ {
+ "adv1_neg_pct": 2.5322799682617188,
+ "adv2_active_pct": 0.33805567026138306,
+ "ce_loss": 0.00585869699716568,
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.030877001583576202,
+ "hpd_loss": 0.003286010818555951,
+ "k1_gt_raw_max": 0.058261577039957047,
+ "k1_gt_raw_mean": 0.002064550295472145,
+ "k1_gt_raw_min": -0.0029995564837008715,
+ "k1_gt_raw_neg_pct": 2.5322799682617188,
+ "k1_sample_max": -0.3366956412792206,
+ "k1_sample_mean": -0.39731863141059875,
+ "k1_sample_min": -0.45794159173965454,
+ "k1_sample_neg_pct": 14.0625,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.0032860103994607924,
+ "mask3_pct": 0.33805567026138306,
+ "neg_student_nll_loss": 0.00585869699716568,
+ "sampled_diff_pct": 0.3419136703014374,
+ "step": 500
+ },
+ {
+ "adv1_neg_pct": 2.701911449432373,
+ "adv2_active_pct": 0.3643760681152344,
+ "ce_loss": 0.010007714852690697,
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.14292795956134796,
+ "hpd_loss": 0.012855440378189087,
+ "k1_gt_raw_max": 0.06845426559448242,
+ "k1_gt_raw_mean": 0.0017418819479644299,
+ "k1_gt_raw_min": -0.004030767362564802,
+ "k1_gt_raw_neg_pct": 2.701911449432373,
+ "k1_sample_max": -0.4802917242050171,
+ "k1_sample_mean": -0.5421380996704102,
+ "k1_sample_min": -0.6135058999061584,
+ "k1_sample_neg_pct": 17.03125,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.012855446338653565,
+ "mask3_pct": 0.3643760681152344,
+ "neg_student_nll_loss": 0.010007714852690697,
+ "sampled_diff_pct": 0.3710411787033081,
+ "step": 520
+ },
+ {
+ "adv1_neg_pct": 2.248600959777832,
+ "adv2_active_pct": 0.5108850598335266,
+ "ce_loss": 0.01003289595246315,
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.6681362390518188,
+ "hpd_loss": 0.009245548397302628,
+ "k1_gt_raw_max": 0.08858497440814972,
+ "k1_gt_raw_mean": 0.002853624988347292,
+ "k1_gt_raw_min": -0.004460392985492945,
+ "k1_gt_raw_neg_pct": 2.248600959777832,
+ "k1_sample_max": -0.7408795356750488,
+ "k1_sample_mean": -0.8169400691986084,
+ "k1_sample_min": -0.8906024694442749,
+ "k1_sample_neg_pct": 22.65625,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.009245550632476807,
+ "mask3_pct": 0.5108850598335266,
+ "neg_student_nll_loss": 0.01003289595246315,
+ "sampled_diff_pct": 0.5210334062576294,
+ "step": 540
+ },
+ {
+ "adv1_neg_pct": 2.4635496139526367,
+ "adv2_active_pct": 0.22115065157413483,
+ "ce_loss": 0.005004607141017914,
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.08243642747402191,
+ "hpd_loss": 0.0062531642615795135,
+ "k1_gt_raw_max": 0.0580272376537323,
+ "k1_gt_raw_mean": 0.0011763360816985369,
+ "k1_gt_raw_min": -0.005644598975777626,
+ "k1_gt_raw_neg_pct": 2.4635493755340576,
+ "k1_sample_max": -0.2404015064239502,
+ "k1_sample_mean": -0.2807300090789795,
+ "k1_sample_min": -0.32241004705429077,
+ "k1_sample_neg_pct": 14.921875,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.006253162026405334,
+ "mask3_pct": 0.22115065157413483,
+ "neg_student_nll_loss": 0.005004607141017914,
+ "sampled_diff_pct": 0.22733174264431,
+ "step": 560
+ },
+ {
+ "adv1_neg_pct": 3.3381173610687256,
+ "adv2_active_pct": 0.23258033394813538,
+ "ce_loss": 0.006839662324637175,
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.0869053527712822,
+ "hpd_loss": 0.00969111267477274,
+ "k1_gt_raw_max": 0.05798381567001343,
+ "k1_gt_raw_mean": 0.0016341875307261944,
+ "k1_gt_raw_min": -0.009443331509828568,
+ "k1_gt_raw_neg_pct": 3.3381173610687256,
+ "k1_sample_max": -0.29867684841156006,
+ "k1_sample_mean": -0.31859445571899414,
+ "k1_sample_min": -0.338512122631073,
+ "k1_sample_neg_pct": 10.625,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.009691111743450165,
+ "mask3_pct": 0.23258033394813538,
+ "neg_student_nll_loss": 0.006839662324637175,
+ "sampled_diff_pct": 0.24274389445781708,
+ "step": 580
+ },
+ {
+ "adv1_neg_pct": 4.040760040283203,
+ "adv2_active_pct": 0.20796719193458557,
+ "ce_loss": 0.002632163930684328,
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.2512037456035614,
+ "hpd_loss": 0.0005975529784336686,
+ "k1_gt_raw_max": 0.049394719302654266,
+ "k1_gt_raw_mean": 0.0014062562258914113,
+ "k1_gt_raw_min": -0.002836518455296755,
+ "k1_gt_raw_neg_pct": 4.040760040283203,
+ "k1_sample_max": -0.23663434386253357,
+ "k1_sample_mean": -0.255886435508728,
+ "k1_sample_min": -0.2802615463733673,
+ "k1_sample_neg_pct": 10.208333969116211,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.0005975536536425352,
+ "mask3_pct": 0.20796719193458557,
+ "neg_student_nll_loss": 0.002632163930684328,
+ "sampled_diff_pct": 0.21867018938064575,
+ "step": 600
+ },
+ {
+ "adv1_neg_pct": 4.097904205322266,
+ "adv2_active_pct": 0.1848699152469635,
+ "ce_loss": 0.0032025063410401344,
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.338096022605896,
+ "hpd_loss": 0.002149906475096941,
+ "k1_gt_raw_max": 0.043635837733745575,
+ "k1_gt_raw_mean": 0.0007720005232840776,
+ "k1_gt_raw_min": -0.0036990540102124214,
+ "k1_gt_raw_neg_pct": 4.097904205322266,
+ "k1_sample_max": -0.2395995557308197,
+ "k1_sample_mean": -0.2737259864807129,
+ "k1_sample_min": -0.3078524172306061,
+ "k1_sample_neg_pct": 13.4375,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.0021499061957001688,
+ "mask3_pct": 0.1848699152469635,
+ "neg_student_nll_loss": 0.0032025063410401344,
+ "sampled_diff_pct": 0.18927133083343506,
+ "step": 620
+ },
+ {
+ "adv1_neg_pct": 3.390739917755127,
+ "adv2_active_pct": 0.10791555047035217,
+ "ce_loss": 0.001353576430119574,
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.1519,
+ "eval_generator_rouge1": 99.7488,
+ "eval_generator_rouge2": 99.6229,
+ "eval_generator_rougeL": 99.5782,
+ "eval_loss": 0.004672760609537363,
+ "eval_runtime": 596.1717,
+ "eval_samples_per_second": 5.727,
+ "eval_selector_accuracy": 98.014,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.179,
+ "hpd_loss": 0.0011793917510658503,
+ "k1_gt_raw_max": 0.03581507131457329,
+ "k1_gt_raw_mean": 0.0006900117732584476,
+ "k1_gt_raw_min": -0.002846459159627557,
+ "k1_gt_raw_neg_pct": 3.390739917755127,
+ "k1_sample_max": -0.06917751580476761,
+ "k1_sample_mean": -0.06961424648761749,
+ "k1_sample_min": -0.07005097717046738,
+ "k1_sample_neg_pct": 10.1694917678833,
+ "mask3_pct": 0.10791555047035217,
+ "neg_student_nll_loss": 0.001353576430119574,
+ "sampled_diff_pct": 0.10791555047035217,
+ "step": 628
+ },
+ {
+ "adv1_neg_pct": 3.476264715194702,
+ "adv2_active_pct": 0.12990054488182068,
+ "ce_loss": 0.004336793441325426,
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.6891085505485535,
+ "hpd_loss": 0.002609923481941223,
+ "k1_gt_raw_max": 0.038067810237407684,
+ "k1_gt_raw_mean": 0.0007896563620306551,
+ "k1_gt_raw_min": -0.005201837047934532,
+ "k1_gt_raw_neg_pct": 3.476264715194702,
+ "k1_sample_max": -0.2407418042421341,
+ "k1_sample_mean": -0.2449159324169159,
+ "k1_sample_min": -0.2522057890892029,
+ "k1_sample_neg_pct": 9.798177719116211,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.0020211638882756235,
+ "mask3_pct": 0.12990054488182068,
+ "neg_student_nll_loss": 0.004336793441325426,
+ "sampled_diff_pct": 0.14956170320510864,
+ "step": 640
+ },
+ {
+ "adv1_neg_pct": 3.8216357231140137,
+ "adv2_active_pct": 0.16393554210662842,
+ "ce_loss": 0.0034592957235872746,
+ "epoch": 2.102114080574392,
+ "grad_norm": 1.311052918434143,
+ "hpd_loss": 0.004151628818362951,
+ "k1_gt_raw_max": 0.04581638425588608,
+ "k1_gt_raw_mean": 0.0008853404433466494,
+ "k1_gt_raw_min": -0.0063157654367387295,
+ "k1_gt_raw_neg_pct": 3.8216357231140137,
+ "k1_sample_max": -0.250868022441864,
+ "k1_sample_mean": -0.30226844549179077,
+ "k1_sample_min": -0.3555181622505188,
+ "k1_sample_neg_pct": 10.833333969116211,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.004151628911495208,
+ "mask3_pct": 0.16393554210662842,
+ "neg_student_nll_loss": 0.0034592957235872746,
+ "sampled_diff_pct": 0.16612085700035095,
+ "step": 660
+ },
+ {
+ "adv1_neg_pct": 4.672894477844238,
+ "adv2_active_pct": 0.17785239219665527,
+ "ce_loss": 0.0037891073152422905,
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.20577986538410187,
+ "hpd_loss": 0.004049420822411776,
+ "k1_gt_raw_max": 0.047759026288986206,
+ "k1_gt_raw_mean": 0.0010929291602224112,
+ "k1_gt_raw_min": -0.008511744439601898,
+ "k1_gt_raw_neg_pct": 4.672894477844238,
+ "k1_sample_max": -0.2957162857055664,
+ "k1_sample_mean": -0.3683391511440277,
+ "k1_sample_min": -0.440962016582489,
+ "k1_sample_neg_pct": 12.8125,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.004049421846866607,
+ "mask3_pct": 0.17785239219665527,
+ "neg_student_nll_loss": 0.0037891073152422905,
+ "sampled_diff_pct": 0.17980551719665527,
+ "step": 680
+ },
+ {
+ "adv1_neg_pct": 3.504452705383301,
+ "adv2_active_pct": 0.33364731073379517,
+ "ce_loss": 0.00837249867618084,
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.05891747772693634,
+ "hpd_loss": 0.012418361380696297,
+ "k1_gt_raw_max": 0.044552117586135864,
+ "k1_gt_raw_mean": 0.001460403436794877,
+ "k1_gt_raw_min": -0.0013761408627033234,
+ "k1_gt_raw_neg_pct": 3.504452705383301,
+ "k1_sample_max": -0.41403040289878845,
+ "k1_sample_mean": -0.4155550003051758,
+ "k1_sample_min": -0.4170795679092407,
+ "k1_sample_neg_pct": 12.8125,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.012418360263109208,
+ "mask3_pct": 0.33364731073379517,
+ "neg_student_nll_loss": 0.00837249867618084,
+ "sampled_diff_pct": 0.33364731073379517,
+ "step": 700
+ },
+ {
+ "adv1_neg_pct": 2.9557998180389404,
+ "adv2_active_pct": 0.2356344759464264,
+ "ce_loss": 0.002732612658292055,
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.03069828823208809,
+ "hpd_loss": 0.0013308569323271513,
+ "k1_gt_raw_max": 0.043037086725234985,
+ "k1_gt_raw_mean": 0.001189029891975224,
+ "k1_gt_raw_min": -0.00335265276953578,
+ "k1_gt_raw_neg_pct": 2.9557998180389404,
+ "k1_sample_max": -0.12545080482959747,
+ "k1_sample_mean": -0.13167737424373627,
+ "k1_sample_min": -0.13885264098644257,
+ "k1_sample_neg_pct": 12.1875,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.0013308576308190823,
+ "mask3_pct": 0.2356344759464264,
+ "neg_student_nll_loss": 0.002732612658292055,
+ "sampled_diff_pct": 0.24056380987167358,
+ "step": 720
+ },
+ {
+ "adv1_neg_pct": 4.362781524658203,
+ "adv2_active_pct": 0.14987114071846008,
+ "ce_loss": 0.00366773409768939,
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.03650576248764992,
+ "hpd_loss": 0.0028897470328956842,
+ "k1_gt_raw_max": 0.03678428381681442,
+ "k1_gt_raw_mean": 0.000713786983396858,
+ "k1_gt_raw_min": -0.007670572958886623,
+ "k1_gt_raw_neg_pct": 4.362782001495361,
+ "k1_sample_max": -0.16289430856704712,
+ "k1_sample_mean": -0.19424015283584595,
+ "k1_sample_min": -0.2255859375,
+ "k1_sample_neg_pct": 9.0625,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.002889746427536011,
+ "mask3_pct": 0.14987114071846008,
+ "neg_student_nll_loss": 0.00366773409768939,
+ "sampled_diff_pct": 0.1667182892560959,
+ "step": 740
+ },
+ {
+ "adv1_neg_pct": 4.3088059425354,
+ "adv2_active_pct": 0.073929063975811,
+ "ce_loss": 0.0018987958319485188,
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.03706243261694908,
+ "hpd_loss": 0.001820039818994701,
+ "k1_gt_raw_max": 0.030663518235087395,
+ "k1_gt_raw_mean": 0.0005116415559314191,
+ "k1_gt_raw_min": -0.003718154039233923,
+ "k1_gt_raw_neg_pct": 4.3088059425354,
+ "k1_sample_max": -0.11870913207530975,
+ "k1_sample_mean": -0.12011738121509552,
+ "k1_sample_min": -0.12152564525604248,
+ "k1_sample_neg_pct": 7.03125,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0018200401216745376,
+ "mask3_pct": 0.073929063975811,
+ "neg_student_nll_loss": 0.0018987958319485188,
+ "sampled_diff_pct": 0.08056812733411789,
+ "step": 760
+ },
+ {
+ "adv1_neg_pct": 4.6282758712768555,
+ "adv2_active_pct": 0.08583934605121613,
+ "ce_loss": 0.000978614203631878,
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.0633927583694458,
+ "hpd_loss": 0.0008434242918156087,
+ "k1_gt_raw_max": 0.02713455632328987,
+ "k1_gt_raw_mean": 0.0004272414662409574,
+ "k1_gt_raw_min": -0.006445502862334251,
+ "k1_gt_raw_neg_pct": 4.628275394439697,
+ "k1_sample_max": -0.10302496701478958,
+ "k1_sample_mean": -0.11016625910997391,
+ "k1_sample_min": -0.11730755120515823,
+ "k1_sample_neg_pct": 8.90625,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0008434242568910122,
+ "mask3_pct": 0.08583934605121613,
+ "neg_student_nll_loss": 0.000978614203631878,
+ "sampled_diff_pct": 0.09829449653625488,
+ "step": 780
+ },
+ {
+ "adv1_neg_pct": 5.204377174377441,
+ "adv2_active_pct": 0.15303727984428406,
+ "ce_loss": 0.004278196021914482,
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.2113279402256012,
+ "hpd_loss": 0.00520919868722558,
+ "k1_gt_raw_max": 0.0305880606174469,
+ "k1_gt_raw_mean": 0.0005127020995132625,
+ "k1_gt_raw_min": -0.0020859134383499622,
+ "k1_gt_raw_neg_pct": 5.204377174377441,
+ "k1_sample_max": -0.07243171334266663,
+ "k1_sample_mean": -0.08607004582881927,
+ "k1_sample_min": -0.10009756684303284,
+ "k1_sample_neg_pct": 10.0,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.005209198221564293,
+ "mask3_pct": 0.15303727984428406,
+ "neg_student_nll_loss": 0.004278196021914482,
+ "sampled_diff_pct": 0.1585448682308197,
+ "step": 800
+ },
+ {
+ "adv1_neg_pct": 5.268265724182129,
+ "adv2_active_pct": 0.10696102678775787,
+ "ce_loss": 0.003248082473874092,
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.32251229882240295,
+ "hpd_loss": 0.002392602153122425,
+ "k1_gt_raw_max": 0.03637195751070976,
+ "k1_gt_raw_mean": 0.0010833492269739509,
+ "k1_gt_raw_min": -0.0027411947958171368,
+ "k1_gt_raw_neg_pct": 5.268265724182129,
+ "k1_sample_max": -0.06464927643537521,
+ "k1_sample_mean": -0.06609892845153809,
+ "k1_sample_min": -0.06948131322860718,
+ "k1_sample_neg_pct": 6.826923370361328,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.0023926014080643656,
+ "mask3_pct": 0.1040404736995697,
+ "neg_student_nll_loss": 0.003248082473874092,
+ "sampled_diff_pct": 0.11661312729120255,
+ "step": 820
+ },
+ {
+ "adv1_neg_pct": 5.766000270843506,
+ "adv2_active_pct": 0.06631548702716827,
+ "ce_loss": 0.0020227693021297455,
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.2545914947986603,
+ "hpd_loss": 0.002442866563796997,
+ "k1_gt_raw_max": 0.027828488498926163,
+ "k1_gt_raw_mean": 0.0007106282282620668,
+ "k1_gt_raw_min": -0.004438548348844051,
+ "k1_gt_raw_neg_pct": 5.766000270843506,
+ "k1_sample_max": -0.10070931911468506,
+ "k1_sample_mean": -0.10070931911468506,
+ "k1_sample_min": -0.10070931911468506,
+ "k1_sample_neg_pct": 7.1875,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.002442866563796997,
+ "mask3_pct": 0.06631548702716827,
+ "neg_student_nll_loss": 0.0020227693021297455,
+ "sampled_diff_pct": 0.07493838667869568,
+ "step": 840
+ },
+ {
+ "adv1_neg_pct": 6.06646728515625,
+ "adv2_active_pct": 0.10010633617639542,
+ "ce_loss": 0.0019979095086455345,
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.2798786759376526,
+ "hpd_loss": 0.0019781002774834633,
+ "k1_gt_raw_max": 0.0327674075961113,
+ "k1_gt_raw_mean": 0.0005696950247511268,
+ "k1_gt_raw_min": -0.003440997563302517,
+ "k1_gt_raw_neg_pct": 6.06646728515625,
+ "k1_sample_max": -0.08324240148067474,
+ "k1_sample_mean": -0.13795003294944763,
+ "k1_sample_min": -0.1926576793193817,
+ "k1_sample_neg_pct": 9.375,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.0019781002774834633,
+ "mask3_pct": 0.10010633617639542,
+ "neg_student_nll_loss": 0.0019979095086455345,
+ "sampled_diff_pct": 0.10264699161052704,
+ "step": 860
+ },
+ {
+ "adv1_neg_pct": 5.891697883605957,
+ "adv2_active_pct": 0.07234375178813934,
+ "ce_loss": 0.0014330728445202112,
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.09429220855236053,
+ "hpd_loss": 0.0010787295177578926,
+ "k1_gt_raw_max": 0.038590773940086365,
+ "k1_gt_raw_mean": 0.0007731987861916423,
+ "k1_gt_raw_min": -0.0038950724992901087,
+ "k1_gt_raw_neg_pct": 5.891697883605957,
+ "k1_sample_max": -0.1224956065416336,
+ "k1_sample_mean": -0.1250840276479721,
+ "k1_sample_min": -0.12767243385314941,
+ "k1_sample_neg_pct": 7.34375,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.001078729424625635,
+ "mask3_pct": 0.07234375178813934,
+ "neg_student_nll_loss": 0.0014330728445202112,
+ "sampled_diff_pct": 0.07657574117183685,
+ "step": 880
+ },
+ {
+ "adv1_neg_pct": 6.011615753173828,
+ "adv2_active_pct": 0.09838910400867462,
+ "ce_loss": 0.0020957919768989086,
+ "epoch": 2.8679696848823295,
+ "grad_norm": 1.0214177370071411,
+ "hpd_loss": 0.0015987120568752289,
+ "k1_gt_raw_max": 0.03688127547502518,
+ "k1_gt_raw_mean": 0.0006160076009109616,
+ "k1_gt_raw_min": -0.005018594674766064,
+ "k1_gt_raw_neg_pct": 6.011615753173828,
+ "k1_sample_max": -0.2794632613658905,
+ "k1_sample_mean": -0.2854064106941223,
+ "k1_sample_min": -0.2913495600223541,
+ "k1_sample_neg_pct": 8.75,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0015987120568752289,
+ "mask3_pct": 0.09838910400867462,
+ "neg_student_nll_loss": 0.0020957919768989086,
+ "sampled_diff_pct": 0.10469574481248856,
+ "step": 900
+ },
+ {
+ "adv1_neg_pct": 5.8451714515686035,
+ "adv2_active_pct": 0.12536077201366425,
+ "ce_loss": 0.0031956052407622337,
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.009789629839360714,
+ "hpd_loss": 0.005076252389699221,
+ "k1_gt_raw_max": 0.03244764730334282,
+ "k1_gt_raw_mean": 0.0005789952119812369,
+ "k1_gt_raw_min": -0.004300437867641449,
+ "k1_gt_raw_neg_pct": 5.8451714515686035,
+ "k1_sample_max": -0.09452944993972778,
+ "k1_sample_mean": -0.1174035519361496,
+ "k1_sample_min": -0.1418294906616211,
+ "k1_sample_neg_pct": 8.4375,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.005076252669095993,
+ "mask3_pct": 0.12536077201366425,
+ "neg_student_nll_loss": 0.0031956052407622337,
+ "sampled_diff_pct": 0.129419207572937,
+ "step": 920
+ },
+ {
+ "adv1_neg_pct": 6.187288284301758,
+ "adv2_active_pct": 0.08470146358013153,
+ "ce_loss": 0.001648780656978488,
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.1643683910369873,
+ "hpd_loss": 0.0010852484265342355,
+ "k1_gt_raw_max": 0.03225347399711609,
+ "k1_gt_raw_mean": 0.0005872219335287809,
+ "k1_gt_raw_min": -0.0068657128140330315,
+ "k1_gt_raw_neg_pct": 6.187288284301758,
+ "k1_sample_max": -0.11387737095355988,
+ "k1_sample_mean": -0.13910287618637085,
+ "k1_sample_min": -0.1643284261226654,
+ "k1_sample_neg_pct": 7.8125,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0010852484032511712,
+ "mask3_pct": 0.08470146358013153,
+ "neg_student_nll_loss": 0.001648780656978488,
+ "sampled_diff_pct": 0.08656158298254013,
+ "step": 940
+ },
+ {
+ "adv1_neg_pct": 5.000968933105469,
+ "adv2_active_pct": 0.110027015209198,
+ "ce_loss": 0.001415811711922288,
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.6776,
+ "eval_generator_rouge1": 99.8328,
+ "eval_generator_rouge2": 99.7335,
+ "eval_generator_rougeL": 99.6469,
+ "eval_loss": 0.00400863541290164,
+ "eval_runtime": 591.8273,
+ "eval_samples_per_second": 5.769,
+ "eval_selector_accuracy": 97.7804,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.181,
+ "hpd_loss": 0.0011558050755411386,
+ "k1_gt_raw_max": 0.03289224952459335,
+ "k1_gt_raw_mean": 0.0007148425211198628,
+ "k1_gt_raw_min": -0.0005029054591432214,
+ "k1_gt_raw_neg_pct": 5.000969409942627,
+ "k1_sample_max": -0.4340977072715759,
+ "k1_sample_mean": -0.4340977072715759,
+ "k1_sample_min": -0.4340977072715759,
+ "k1_sample_neg_pct": 9.090909004211426,
+ "mask3_pct": 0.110027015209198,
+ "neg_student_nll_loss": 0.001415811711922288,
+ "sampled_diff_pct": 0.110027015209198,
+ "step": 942
+ },
+ {
+ "adv1_neg_pct": 5.784729957580566,
+ "adv2_active_pct": 0.1198313981294632,
+ "ce_loss": 0.0024845735169947147,
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.0822562500834465,
+ "hpd_loss": 0.0026429281570017338,
+ "k1_gt_raw_max": 0.030722517520189285,
+ "k1_gt_raw_mean": 0.0009094014531001449,
+ "k1_gt_raw_min": -0.006959723308682442,
+ "k1_gt_raw_neg_pct": 5.784729957580566,
+ "k1_sample_max": -0.06599755585193634,
+ "k1_sample_mean": -0.06599755585193634,
+ "k1_sample_min": -0.06599755585193634,
+ "k1_sample_neg_pct": 6.94444465637207,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.0024656012654304506,
+ "mask3_pct": 0.1198313981294632,
+ "neg_student_nll_loss": 0.0024845735169947147,
+ "sampled_diff_pct": 0.12411808967590332,
+ "step": 960
+ },
+ {
+ "adv1_neg_pct": 6.275859832763672,
+ "adv2_active_pct": 0.1653088629245758,
+ "ce_loss": 0.002910045674070716,
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.002782815834507346,
+ "hpd_loss": 0.002390575595200062,
+ "k1_gt_raw_max": 0.03876848146319389,
+ "k1_gt_raw_mean": 0.001082484028302133,
+ "k1_gt_raw_min": -0.005310694221407175,
+ "k1_gt_raw_neg_pct": 6.275859832763672,
+ "k1_sample_max": -0.10106666386127472,
+ "k1_sample_mean": -0.10889029502868652,
+ "k1_sample_min": -0.11623653769493103,
+ "k1_sample_neg_pct": 7.8125,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.0023905765265226364,
+ "mask3_pct": 0.1653088629245758,
+ "neg_student_nll_loss": 0.002910045674070716,
+ "sampled_diff_pct": 0.17157939076423645,
+ "step": 980
+ },
+ {
+ "adv1_neg_pct": 6.713335037231445,
+ "adv2_active_pct": 0.07989098131656647,
+ "ce_loss": 0.000890995841473341,
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.025590306147933006,
+ "hpd_loss": 0.0010361053282395005,
+ "k1_gt_raw_max": 0.02309339866042137,
+ "k1_gt_raw_mean": 0.000387923966627568,
+ "k1_gt_raw_min": -0.0018807905726134777,
+ "k1_gt_raw_neg_pct": 6.713335037231445,
+ "k1_sample_max": -0.07003647089004517,
+ "k1_sample_mean": -0.0700383111834526,
+ "k1_sample_min": -0.07004015147686005,
+ "k1_sample_neg_pct": 6.5625,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.0010361054912209511,
+ "mask3_pct": 0.07989098131656647,
+ "neg_student_nll_loss": 0.000890995841473341,
+ "sampled_diff_pct": 0.08394942432641983,
+ "step": 1000
+ },
+ {
+ "adv1_neg_pct": 6.980464458465576,
+ "adv2_active_pct": 0.04861818999052048,
+ "ce_loss": 0.0008955475641414523,
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.04064895957708359,
+ "hpd_loss": 0.0005541563150472939,
+ "k1_gt_raw_max": 0.023353416472673416,
+ "k1_gt_raw_mean": 0.00037372263614088297,
+ "k1_gt_raw_min": -0.0031757489778101444,
+ "k1_gt_raw_neg_pct": 6.980464458465576,
+ "k1_sample_max": -0.02265203185379505,
+ "k1_sample_mean": -0.02265203185379505,
+ "k1_sample_min": -0.02265203185379505,
+ "k1_sample_neg_pct": 5.625,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.0005541563965380192,
+ "mask3_pct": 0.04861818999052048,
+ "neg_student_nll_loss": 0.0008955475641414523,
+ "sampled_diff_pct": 0.04861818999052048,
+ "step": 1020
+ },
+ {
+ "adv1_neg_pct": 6.828683853149414,
+ "adv2_active_pct": 0.0889447033405304,
+ "ce_loss": 0.0019629367161542177,
+ "epoch": 3.3127243717590744,
+ "grad_norm": 2.8549108505249023,
+ "hpd_loss": 0.002801127266138792,
+ "k1_gt_raw_max": 0.021204540506005287,
+ "k1_gt_raw_mean": 0.00043139539775438607,
+ "k1_gt_raw_min": -0.006724623031914234,
+ "k1_gt_raw_neg_pct": 6.828684329986572,
+ "k1_sample_max": -0.07739919424057007,
+ "k1_sample_mean": -0.07739919424057007,
+ "k1_sample_min": -0.07739919424057007,
+ "k1_sample_neg_pct": 5.0,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.00280112698674202,
+ "mask3_pct": 0.0889447033405304,
+ "neg_student_nll_loss": 0.0019629367161542177,
+ "sampled_diff_pct": 0.10269542038440704,
+ "step": 1040
+ },
+ {
+ "adv1_neg_pct": 7.2235517501831055,
+ "adv2_active_pct": 0.10061224550008774,
+ "ce_loss": 0.004100818186998367,
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.004515279550105333,
+ "hpd_loss": 0.006908780429512262,
+ "k1_gt_raw_max": 0.017903421074151993,
+ "k1_gt_raw_mean": 0.0002864316338673234,
+ "k1_gt_raw_min": -0.003864757716655731,
+ "k1_gt_raw_neg_pct": 7.2235517501831055,
+ "k1_sample_max": -0.07832760363817215,
+ "k1_sample_mean": -0.07832760363817215,
+ "k1_sample_min": -0.07832760363817215,
+ "k1_sample_neg_pct": 5.9375,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.006908781081438065,
+ "mask3_pct": 0.10061224550008774,
+ "neg_student_nll_loss": 0.004100818186998367,
+ "sampled_diff_pct": 0.10612311214208603,
+ "step": 1060
+ },
+ {
+ "adv1_neg_pct": 7.121249198913574,
+ "adv2_active_pct": 0.035785384476184845,
+ "ce_loss": 0.0010942043736577034,
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.006863700225949287,
+ "hpd_loss": 0.0009617547038942575,
+ "k1_gt_raw_max": 0.016735289245843887,
+ "k1_gt_raw_mean": 0.0003524690982885659,
+ "k1_gt_raw_min": -0.003919572569429874,
+ "k1_gt_raw_neg_pct": 7.121249198913574,
+ "k1_sample_max": -0.02336699143052101,
+ "k1_sample_mean": -0.02336699143052101,
+ "k1_sample_min": -0.02336699143052101,
+ "k1_sample_neg_pct": 4.0625,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.0009617548435926438,
+ "mask3_pct": 0.032108910381793976,
+ "neg_student_nll_loss": 0.0010942043736577034,
+ "sampled_diff_pct": 0.05000416934490204,
+ "step": 1080
+ },
+ {
+ "adv1_neg_pct": 7.497800827026367,
+ "adv2_active_pct": 0.06837265938520432,
+ "ce_loss": 0.0012112363474443555,
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.009582476690411568,
+ "hpd_loss": 0.0011069111060351133,
+ "k1_gt_raw_max": 0.01962250843644142,
+ "k1_gt_raw_mean": 0.0003861070144921541,
+ "k1_gt_raw_min": -0.003895876696333289,
+ "k1_gt_raw_neg_pct": 7.497800827026367,
+ "k1_sample_max": -0.03470407798886299,
+ "k1_sample_mean": -0.038440171629190445,
+ "k1_sample_min": -0.0421762689948082,
+ "k1_sample_neg_pct": 5.78125,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0011069110594689846,
+ "mask3_pct": 0.06837265938520432,
+ "neg_student_nll_loss": 0.0012112363474443555,
+ "sampled_diff_pct": 0.07695083320140839,
+ "step": 1100
+ },
+ {
+ "adv1_neg_pct": 7.4989423751831055,
+ "adv2_active_pct": 0.09932753443717957,
+ "ce_loss": 0.001159896608442068,
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.07570958882570267,
+ "hpd_loss": 0.0004915354074910283,
+ "k1_gt_raw_max": 0.021993957459926605,
+ "k1_gt_raw_mean": 0.0004521916271187365,
+ "k1_gt_raw_min": -0.005119780544191599,
+ "k1_gt_raw_neg_pct": 7.4989423751831055,
+ "k1_sample_max": -0.03237403556704521,
+ "k1_sample_mean": -0.06356782466173172,
+ "k1_sample_min": -0.09476162493228912,
+ "k1_sample_neg_pct": 6.5625,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.0004915353842079639,
+ "mask3_pct": 0.09932753443717957,
+ "neg_student_nll_loss": 0.001159896608442068,
+ "sampled_diff_pct": 0.10849244892597198,
+ "step": 1120
+ },
+ {
+ "adv1_neg_pct": 7.469758033752441,
+ "adv2_active_pct": 0.03868325054645538,
+ "ce_loss": 0.0013367629144340754,
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.030605977401137352,
+ "hpd_loss": 0.0013018195750191808,
+ "k1_gt_raw_max": 0.01742619276046753,
+ "k1_gt_raw_mean": 0.00042788474820554256,
+ "k1_gt_raw_min": -0.002206374891102314,
+ "k1_gt_raw_neg_pct": 7.469758033752441,
+ "k1_sample_max": -0.026432175189256668,
+ "k1_sample_mean": -0.026432175189256668,
+ "k1_sample_min": -0.026432175189256668,
+ "k1_sample_neg_pct": 4.375,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0013018193654716014,
+ "mask3_pct": 0.036279406398534775,
+ "neg_student_nll_loss": 0.0013367629144340754,
+ "sampled_diff_pct": 0.04094774276018143,
+ "step": 1140
+ },
+ {
+ "adv1_neg_pct": 7.294529914855957,
+ "adv2_active_pct": 0.11667037010192871,
+ "ce_loss": 0.0022262034472078085,
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.1906837671995163,
+ "hpd_loss": 0.0014206715859472752,
+ "k1_gt_raw_max": 0.021399978548288345,
+ "k1_gt_raw_mean": 0.00044373038690537214,
+ "k1_gt_raw_min": -0.0028102025389671326,
+ "k1_gt_raw_neg_pct": 7.294529914855957,
+ "k1_sample_max": -0.02788049727678299,
+ "k1_sample_mean": -0.028852656483650208,
+ "k1_sample_min": -0.03075457364320755,
+ "k1_sample_neg_pct": 4.9479169845581055,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.0014206715859472752,
+ "mask3_pct": 0.11374981701374054,
+ "neg_student_nll_loss": 0.0022262034472078085,
+ "sampled_diff_pct": 0.1271422654390335,
+ "step": 1160
+ },
+ {
+ "adv1_neg_pct": 7.579638957977295,
+ "adv2_active_pct": 0.06752925366163254,
+ "ce_loss": 0.0028973212465643883,
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.011799277737736702,
+ "hpd_loss": 0.0015698154456913471,
+ "k1_gt_raw_max": 0.01623934879899025,
+ "k1_gt_raw_mean": 0.00032681378070265055,
+ "k1_gt_raw_min": -0.0041321092285215855,
+ "k1_gt_raw_neg_pct": 7.579638957977295,
+ "k1_sample_max": -0.0820823684334755,
+ "k1_sample_mean": -0.08302680402994156,
+ "k1_sample_min": -0.08577128499746323,
+ "k1_sample_neg_pct": 3.680555582046509,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0015698155388236047,
+ "mask3_pct": 0.06483529508113861,
+ "neg_student_nll_loss": 0.0028973212465643883,
+ "sampled_diff_pct": 0.08027052134275436,
+ "step": 1180
+ },
+ {
+ "adv1_neg_pct": 8.004940032958984,
+ "adv2_active_pct": 0.17861586809158325,
+ "ce_loss": 0.0022735765669494867,
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.014485353603959084,
+ "hpd_loss": 0.0013695353409275413,
+ "k1_gt_raw_max": 0.022271674126386642,
+ "k1_gt_raw_mean": 0.00047311594244092703,
+ "k1_gt_raw_min": -0.005409082397818565,
+ "k1_gt_raw_neg_pct": 8.004940032958984,
+ "k1_sample_max": -0.07869651913642883,
+ "k1_sample_mean": -0.1084669977426529,
+ "k1_sample_min": -0.13823749125003815,
+ "k1_sample_neg_pct": 6.5625,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.0013695357367396355,
+ "mask3_pct": 0.17861586809158325,
+ "neg_student_nll_loss": 0.0022735765669494867,
+ "sampled_diff_pct": 0.1825408637523651,
+ "step": 1200
+ },
+ {
+ "adv1_neg_pct": 7.9708051681518555,
+ "adv2_active_pct": 0.14446821808815002,
+ "ce_loss": 0.002031442243605852,
+ "epoch": 3.8871160749900278,
+ "grad_norm": 1.8391499519348145,
+ "hpd_loss": 0.0014964366564527154,
+ "k1_gt_raw_max": 0.019910190254449844,
+ "k1_gt_raw_mean": 0.0005614210967905819,
+ "k1_gt_raw_min": -0.003311981214210391,
+ "k1_gt_raw_neg_pct": 7.9708051681518555,
+ "k1_sample_max": -0.06566861271858215,
+ "k1_sample_mean": -0.06566861271858215,
+ "k1_sample_min": -0.06566861271858215,
+ "k1_sample_neg_pct": 5.3125,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.0014964366331696511,
+ "mask3_pct": 0.14446821808815002,
+ "neg_student_nll_loss": 0.002031442243605852,
+ "sampled_diff_pct": 0.1486356258392334,
+ "step": 1220
+ },
+ {
+ "adv1_neg_pct": 7.752712249755859,
+ "adv2_active_pct": 0.05451521277427673,
+ "ce_loss": 0.0008983995649032295,
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.015798989683389664,
+ "hpd_loss": 0.0006963292835280299,
+ "k1_gt_raw_max": 0.017497122287750244,
+ "k1_gt_raw_mean": 0.00027755158953368664,
+ "k1_gt_raw_min": -0.0025472629349678755,
+ "k1_gt_raw_neg_pct": 7.752712249755859,
+ "k1_sample_max": -0.0381598025560379,
+ "k1_sample_mean": -0.03892132267355919,
+ "k1_sample_min": -0.03968284651637077,
+ "k1_sample_neg_pct": 4.375,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0006963292602449655,
+ "mask3_pct": 0.05451521277427673,
+ "neg_student_nll_loss": 0.0008983995649032295,
+ "sampled_diff_pct": 0.061319805681705475,
+ "step": 1240
+ },
+ {
+ "adv1_neg_pct": 7.9339280128479,
+ "adv2_active_pct": 0.0740404948592186,
+ "ce_loss": 0.0014802285004407167,
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.4953,
+ "eval_generator_rouge1": 99.8312,
+ "eval_generator_rouge2": 99.7559,
+ "eval_generator_rougeL": 99.645,
+ "eval_loss": 0.003369994228705764,
+ "eval_runtime": 1103.3525,
+ "eval_samples_per_second": 3.094,
+ "eval_selector_accuracy": 98.5981,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.097,
+ "hpd_loss": 0.0012612877180799842,
+ "k1_gt_raw_max": 0.01940547674894333,
+ "k1_gt_raw_mean": 0.0002844607806764543,
+ "k1_gt_raw_min": -0.003394282190129161,
+ "k1_gt_raw_neg_pct": 7.9339280128479,
+ "k1_sample_max": -0.06970421969890594,
+ "k1_sample_mean": -0.06970421969890594,
+ "k1_sample_min": -0.06970421969890594,
+ "k1_sample_neg_pct": 6.910569190979004,
+ "mask3_pct": 0.0740404948592186,
+ "neg_student_nll_loss": 0.0014802285004407167,
+ "sampled_diff_pct": 0.08771276473999023,
+ "step": 1256
+ },
+ {
+ "adv1_neg_pct": 8.280838966369629,
+ "adv2_active_pct": 0.13949164748191833,
+ "ce_loss": 0.005130399018526077,
+ "epoch": 4.0127642600717985,
+ "grad_norm": 0.059252846986055374,
+ "hpd_loss": 0.003739401465281844,
+ "k1_gt_raw_max": 0.02487422712147236,
+ "k1_gt_raw_mean": 0.00037547305691987276,
+ "k1_gt_raw_min": -0.009174536913633347,
+ "k1_gt_raw_neg_pct": 8.280838012695312,
+ "k1_sample_max": -0.009557662531733513,
+ "k1_sample_mean": -0.013778358697891235,
+ "k1_sample_min": -0.020752470940351486,
+ "k1_sample_neg_pct": 7.5,
+ "learning_rate": 9.464081721960592e-06,
+ "loss": 0.001723523810505867,
+ "mask3_pct": 0.13949164748191833,
+ "neg_student_nll_loss": 0.005130399018526077,
+ "sampled_diff_pct": 0.17727938294410706,
+ "step": 1260
+ },
+ {
+ "adv1_neg_pct": 7.800583839416504,
+ "adv2_active_pct": 0.043229397386312485,
+ "ce_loss": 0.0006054828409105539,
+ "epoch": 4.076585560430794,
+ "grad_norm": 0.0579792857170105,
+ "hpd_loss": 0.0004133617621846497,
+ "k1_gt_raw_max": 0.015528247691690922,
+ "k1_gt_raw_mean": 0.0002333328011445701,
+ "k1_gt_raw_min": -0.0038630864582955837,
+ "k1_gt_raw_neg_pct": 7.800584316253662,
+ "k1_sample_max": -0.014678173698484898,
+ "k1_sample_mean": -0.014678173698484898,
+ "k1_sample_min": -0.014678173698484898,
+ "k1_sample_neg_pct": 4.0625,
+ "learning_rate": 8.331628936475818e-06,
+ "loss": 0.0004133617505431175,
+ "mask3_pct": 0.043229397386312485,
+ "neg_student_nll_loss": 0.0006054828409105539,
+ "sampled_diff_pct": 0.045146577060222626,
+ "step": 1280
+ },
+ {
+ "adv1_neg_pct": 8.047872543334961,
+ "adv2_active_pct": 0.08839298039674759,
+ "ce_loss": 0.0009541312465444207,
+ "epoch": 4.140406860789788,
+ "grad_norm": 0.003797554410994053,
+ "hpd_loss": 0.0009471764788031578,
+ "k1_gt_raw_max": 0.013323026709258556,
+ "k1_gt_raw_mean": 0.00020666152704507113,
+ "k1_gt_raw_min": -0.00606294721364975,
+ "k1_gt_raw_neg_pct": 8.047872543334961,
+ "k1_sample_max": -0.008535138331353664,
+ "k1_sample_mean": -0.008535138331353664,
+ "k1_sample_min": -0.008535138331353664,
+ "k1_sample_neg_pct": 5.3125,
+ "learning_rate": 7.2659843568748385e-06,
+ "loss": 0.0009471764788031578,
+ "mask3_pct": 0.08839298039674759,
+ "neg_student_nll_loss": 0.0009541312465444207,
+ "sampled_diff_pct": 0.09454742819070816,
+ "step": 1300
+ },
+ {
+ "adv1_neg_pct": 8.000151634216309,
+ "adv2_active_pct": 0.034580253064632416,
+ "ce_loss": 0.0007429468678310513,
+ "epoch": 4.204228161148784,
+ "grad_norm": 0.022960586473345757,
+ "hpd_loss": 0.0005513728247024119,
+ "k1_gt_raw_max": 0.015379039570689201,
+ "k1_gt_raw_mean": 0.0002641355968080461,
+ "k1_gt_raw_min": -0.002805485622957349,
+ "k1_gt_raw_neg_pct": 8.000151634216309,
+ "k1_sample_max": -0.006287828553467989,
+ "k1_sample_mean": -0.006287828553467989,
+ "k1_sample_min": -0.006287828553467989,
+ "k1_sample_neg_pct": 3.4375,
+ "learning_rate": 6.268854516770008e-06,
+ "loss": 0.000551372766494751,
+ "mask3_pct": 0.034580253064632416,
+ "neg_student_nll_loss": 0.0007429468678310513,
+ "sampled_diff_pct": 0.047987110912799835,
+ "step": 1320
+ },
+ {
+ "adv1_neg_pct": 8.06116008758545,
+ "adv2_active_pct": 0.02886439487338066,
+ "ce_loss": 0.0009998048190027475,
+ "epoch": 4.268049461507778,
+ "grad_norm": 0.05514178052544594,
+ "hpd_loss": 0.0010553341126069427,
+ "k1_gt_raw_max": 0.014247752726078033,
+ "k1_gt_raw_mean": 0.0003266474523115903,
+ "k1_gt_raw_min": -0.0025498915929347277,
+ "k1_gt_raw_neg_pct": 8.06116008758545,
+ "k1_sample_max": -0.023031100630760193,
+ "k1_sample_mean": -0.023031100630760193,
+ "k1_sample_min": -0.023031100630760193,
+ "k1_sample_neg_pct": 3.4375,
+ "learning_rate": 5.341836229602331e-06,
+ "loss": 0.0010553342290222645,
+ "mask3_pct": 0.02886439487338066,
+ "neg_student_nll_loss": 0.0009998048190027475,
+ "sampled_diff_pct": 0.03517622873187065,
+ "step": 1340
+ },
+ {
+ "adv1_neg_pct": 8.273700714111328,
+ "adv2_active_pct": 0.022702474147081375,
+ "ce_loss": 0.0009629722917452455,
+ "epoch": 4.331870761866773,
+ "grad_norm": 0.02074464224278927,
+ "hpd_loss": 0.0008157925913110375,
+ "k1_gt_raw_max": 0.01817188784480095,
+ "k1_gt_raw_mean": 0.0004169899912085384,
+ "k1_gt_raw_min": -0.003591842483729124,
+ "k1_gt_raw_neg_pct": 8.273700714111328,
+ "k1_sample_max": -0.006041577085852623,
+ "k1_sample_mean": -0.006041577085852623,
+ "k1_sample_min": -0.006041577085852623,
+ "k1_sample_neg_pct": 2.8125,
+ "learning_rate": 4.4864140314888216e-06,
+ "loss": 0.0008157927542924881,
+ "mask3_pct": 0.022702474147081375,
+ "neg_student_nll_loss": 0.0009629722917452455,
+ "sampled_diff_pct": 0.030936820432543755,
+ "step": 1360
+ },
+ {
+ "adv1_neg_pct": 7.839452743530273,
+ "adv2_active_pct": 0.049231112003326416,
+ "ce_loss": 0.0010351771488785744,
+ "epoch": 4.395692062225768,
+ "grad_norm": 0.038118693977594376,
+ "hpd_loss": 0.0007022822974249721,
+ "k1_gt_raw_max": 0.013979194685816765,
+ "k1_gt_raw_mean": 0.00021736414055339992,
+ "k1_gt_raw_min": -0.005359914153814316,
+ "k1_gt_raw_neg_pct": 7.839452743530273,
+ "k1_sample_max": -0.04112572968006134,
+ "k1_sample_mean": -0.04112572968006134,
+ "k1_sample_min": -0.04112572968006134,
+ "k1_sample_neg_pct": 4.6875,
+ "learning_rate": 3.703957803871932e-06,
+ "loss": 0.0007022821810096502,
+ "mask3_pct": 0.049231112003326416,
+ "neg_student_nll_loss": 0.0010351771488785744,
+ "sampled_diff_pct": 0.056373219937086105,
+ "step": 1380
+ },
+ {
+ "adv1_neg_pct": 8.548715591430664,
+ "adv2_active_pct": 0.06189306825399399,
+ "ce_loss": 0.002533749444410205,
+ "epoch": 4.4595133625847625,
+ "grad_norm": 0.04451121762394905,
+ "hpd_loss": 0.0015958126168698072,
+ "k1_gt_raw_max": 0.017123829573392868,
+ "k1_gt_raw_mean": 0.0002977260737679899,
+ "k1_gt_raw_min": -0.0033434557262808084,
+ "k1_gt_raw_neg_pct": 8.548715591430664,
+ "k1_sample_max": -0.018765060231089592,
+ "k1_sample_mean": -0.02024500072002411,
+ "k1_sample_min": -0.022792842239141464,
+ "k1_sample_neg_pct": 4.699519157409668,
+ "learning_rate": 2.9957205797780495e-06,
+ "loss": 0.0015958130359649658,
+ "mask3_pct": 0.05997588858008385,
+ "neg_student_nll_loss": 0.002533749444410205,
+ "sampled_diff_pct": 0.07589613646268845,
+ "step": 1400
+ },
+ {
+ "adv1_neg_pct": 8.14011001586914,
+ "adv2_active_pct": 0.028554929420351982,
+ "ce_loss": 0.0016532940790057182,
+ "epoch": 4.523334662943758,
+ "grad_norm": 0.03609246015548706,
+ "hpd_loss": 0.0018588351085782051,
+ "k1_gt_raw_max": 0.015877414494752884,
+ "k1_gt_raw_mean": 0.0004283803573343903,
+ "k1_gt_raw_min": -0.004388327710330486,
+ "k1_gt_raw_neg_pct": 8.14011001586914,
+ "k1_sample_max": -0.019240127876400948,
+ "k1_sample_mean": -0.019240127876400948,
+ "k1_sample_min": -0.019240127876400948,
+ "k1_sample_neg_pct": 3.125,
+ "learning_rate": 2.362836537198199e-06,
+ "loss": 0.0018588351085782051,
+ "mask3_pct": 0.028554929420351982,
+ "neg_student_nll_loss": 0.0016532940790057182,
+ "sampled_diff_pct": 0.041689325124025345,
+ "step": 1420
+ },
+ {
+ "adv1_neg_pct": 8.328222274780273,
+ "adv2_active_pct": 0.050987400114536285,
+ "ce_loss": 0.0007530204020440578,
+ "epoch": 4.587155963302752,
+ "grad_norm": 0.01626456528902054,
+ "hpd_loss": 0.0006849722703918815,
+ "k1_gt_raw_max": 0.014860251918435097,
+ "k1_gt_raw_mean": 0.0002935512748081237,
+ "k1_gt_raw_min": -0.003724183887243271,
+ "k1_gt_raw_neg_pct": 8.328222274780273,
+ "k1_sample_max": -0.028437752276659012,
+ "k1_sample_mean": -0.028437752276659012,
+ "k1_sample_min": -0.028437752276659012,
+ "k1_sample_neg_pct": 4.6875,
+ "learning_rate": 1.8063191828043703e-06,
+ "loss": 0.0006849721074104309,
+ "mask3_pct": 0.050987400114536285,
+ "neg_student_nll_loss": 0.0007530204020440578,
+ "sampled_diff_pct": 0.06721149384975433,
+ "step": 1440
+ },
+ {
+ "adv1_neg_pct": 7.96031379699707,
+ "adv2_active_pct": 0.03354815021157265,
+ "ce_loss": 0.0007837152807042003,
+ "epoch": 4.650977263661747,
+ "grad_norm": 0.7522907257080078,
+ "hpd_loss": 0.0007729316130280495,
+ "k1_gt_raw_max": 0.014540698379278183,
+ "k1_gt_raw_mean": 0.00021094034309498966,
+ "k1_gt_raw_min": -0.003531917929649353,
+ "k1_gt_raw_neg_pct": 7.96031379699707,
+ "k1_sample_max": -0.007455674931406975,
+ "k1_sample_mean": -0.036636389791965485,
+ "k1_sample_min": -0.06581709533929825,
+ "k1_sample_neg_pct": 3.28125,
+ "learning_rate": 1.3270597289100437e-06,
+ "loss": 0.000772931519895792,
+ "mask3_pct": 0.03354815021157265,
+ "neg_student_nll_loss": 0.0007837152807042003,
+ "sampled_diff_pct": 0.04775560647249222,
+ "step": 1460
+ },
+ {
+ "adv1_neg_pct": 8.403984069824219,
+ "adv2_active_pct": 0.02325102873146534,
+ "ce_loss": 0.0005917788948863745,
+ "epoch": 4.714798564020742,
+ "grad_norm": 0.0069316113367676735,
+ "hpd_loss": 0.000479920010548085,
+ "k1_gt_raw_max": 0.01145990937948227,
+ "k1_gt_raw_mean": 0.0001835223811212927,
+ "k1_gt_raw_min": -0.004522635601460934,
+ "k1_gt_raw_neg_pct": 8.403984069824219,
+ "k1_sample_max": -0.0059456657618284225,
+ "k1_sample_mean": -0.008637822233140469,
+ "k1_sample_min": -0.012212925590574741,
+ "k1_sample_neg_pct": 2.7083334922790527,
+ "learning_rate": 9.258256662741205e-07,
+ "loss": 0.00047991983592510223,
+ "mask3_pct": 0.02325102873146534,
+ "neg_student_nll_loss": 0.0005917788948863745,
+ "sampled_diff_pct": 0.029702603816986084,
+ "step": 1480
+ },
+ {
+ "adv1_neg_pct": 7.90092658996582,
+ "adv2_active_pct": 0.05479162558913231,
+ "ce_loss": 0.0017136812675744295,
+ "epoch": 4.778619864379737,
+ "grad_norm": 0.08339332044124603,
+ "hpd_loss": 0.0012177170719951391,
+ "k1_gt_raw_max": 0.015806099399924278,
+ "k1_gt_raw_mean": 0.00033896442619152367,
+ "k1_gt_raw_min": -0.0018794997595250607,
+ "k1_gt_raw_neg_pct": 7.90092658996582,
+ "k1_sample_max": -0.02365684136748314,
+ "k1_sample_mean": -0.02365684136748314,
+ "k1_sample_min": -0.02365684136748314,
+ "k1_sample_neg_pct": 5.9375,
+ "learning_rate": 6.032595350337341e-07,
+ "loss": 0.001217717118561268,
+ "mask3_pct": 0.05479162558913231,
+ "neg_student_nll_loss": 0.0017136812675744295,
+ "sampled_diff_pct": 0.0594295933842659,
+ "step": 1500
+ },
+ {
+ "adv1_neg_pct": 8.259902954101562,
+ "adv2_active_pct": 0.09987135231494904,
+ "ce_loss": 0.004173032008111477,
+ "epoch": 4.842441164738732,
+ "grad_norm": 0.010871863923966885,
+ "hpd_loss": 0.007525380700826645,
+ "k1_gt_raw_max": 0.01568448543548584,
+ "k1_gt_raw_mean": 0.0002367629058426246,
+ "k1_gt_raw_min": -0.0033387483563274145,
+ "k1_gt_raw_neg_pct": 8.259902954101562,
+ "k1_sample_max": -0.04992266744375229,
+ "k1_sample_mean": -0.05137576535344124,
+ "k1_sample_min": -0.05282886326313019,
+ "k1_sample_neg_pct": 5.625,
+ "learning_rate": 3.5987789573421356e-07,
+ "loss": 0.007525377720594406,
+ "mask3_pct": 0.09987135231494904,
+ "neg_student_nll_loss": 0.004173032008111477,
+ "sampled_diff_pct": 0.11135074496269226,
+ "step": 1520
+ },
+ {
+ "adv1_neg_pct": 8.341958045959473,
+ "adv2_active_pct": 0.044122375547885895,
+ "ce_loss": 0.0007239175029098988,
+ "epoch": 4.9062624650977265,
+ "grad_norm": 0.09346315264701843,
+ "hpd_loss": 0.0006393846706487238,
+ "k1_gt_raw_max": 0.012377542443573475,
+ "k1_gt_raw_mean": 0.0001652358623687178,
+ "k1_gt_raw_min": -0.004152372479438782,
+ "k1_gt_raw_neg_pct": 8.341958999633789,
+ "k1_sample_max": -0.007577377371490002,
+ "k1_sample_mean": -0.009641861543059349,
+ "k1_sample_min": -0.012904488481581211,
+ "k1_sample_neg_pct": 4.6875,
+ "learning_rate": 1.9607050210401192e-07,
+ "loss": 0.0006393846124410629,
+ "mask3_pct": 0.044122375547885895,
+ "neg_student_nll_loss": 0.0007239175029098988,
+ "sampled_diff_pct": 0.04648980498313904,
+ "step": 1540
+ },
+ {
+ "adv1_neg_pct": 8.45923137664795,
+ "adv2_active_pct": 0.06292080879211426,
+ "ce_loss": 0.0011851892340928316,
+ "epoch": 4.970083765456721,
+ "grad_norm": 0.028900589793920517,
+ "hpd_loss": 0.0013436225708574057,
+ "k1_gt_raw_max": 0.013746695592999458,
+ "k1_gt_raw_mean": 0.0003474469413049519,
+ "k1_gt_raw_min": -0.0036701993085443974,
+ "k1_gt_raw_neg_pct": 8.45923137664795,
+ "k1_sample_max": -0.009252920746803284,
+ "k1_sample_mean": -0.01030099019408226,
+ "k1_sample_min": -0.011349059641361237,
+ "k1_sample_neg_pct": 2.34375,
+ "learning_rate": 1.1209967689925832e-07,
+ "loss": 0.0013436229899525642,
+ "mask3_pct": 0.05779785290360451,
+ "neg_student_nll_loss": 0.0011851892340928316,
+ "sampled_diff_pct": 0.07132385671138763,
+ "step": 1560
+ },
+ {
+ "adv1_neg_pct": 8.752664566040039,
+ "adv2_active_pct": 0.050552137196063995,
+ "ce_loss": 0.0007130501326173544,
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.4953,
+ "eval_generator_rouge1": 99.897,
+ "eval_generator_rouge2": 99.8245,
+ "eval_generator_rougeL": 99.7185,
+ "eval_loss": 0.0032775094732642174,
+ "eval_runtime": 593.2024,
+ "eval_samples_per_second": 5.755,
+ "eval_selector_accuracy": 98.5981,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.18,
+ "hpd_loss": 0.0003927574143745005,
+ "k1_gt_raw_max": 0.015969211235642433,
+ "k1_gt_raw_mean": 0.00024475069949403405,
+ "k1_gt_raw_min": -0.00466952845454216,
+ "k1_gt_raw_neg_pct": 8.752664566040039,
+ "k1_sample_max": -0.008284520357847214,
+ "k1_sample_mean": -0.008284520357847214,
+ "k1_sample_min": -0.008284520357847214,
+ "k1_sample_neg_pct": 4.6666669845581055,
+ "mask3_pct": 0.050552137196063995,
+ "neg_student_nll_loss": 0.0007130501326173544,
+ "sampled_diff_pct": 0.061017539352178574,
+ "step": 1570
+ },
+ {
+ "epoch": 5.0,
+ "step": 1570,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.018279938655608827,
+ "train_runtime": 8124.6936,
+ "train_samples_per_second": 6.171,
+ "train_steps_per_second": 0.193
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 5.871444840310374e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/hpd/training_args.bin b/qwen3/hpd/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..1caa0e72cc5edd590e105073725cf4d832684e44
--- /dev/null
+++ b/qwen3/hpd/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f04fac21a72f04221af020c3bb88a0dc0ccf53bb7a0c0da21d1ab444937cb9d
+size 7185
diff --git a/qwen3/rkl/README.md b/qwen3/rkl/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..af1bcdbe3eb23dc40513bab05a1469e4ba0b15ce
--- /dev/null
+++ b/qwen3/rkl/README.md
@@ -0,0 +1,79 @@
+---
+library_name: peft
+license: other
+base_model: Qwen/Qwen3-0.6B
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+pipeline_tag: text-generation
+model-index:
+- name: rkl
+ results: []
+---
+
+
+
+# rkl
+
+This model is a fine-tuned version of [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) on the cypher_prepared_train dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0039
+- Selector Count: 1712.0
+- Generator Count: 1712.0
+- Selector Accuracy: 98.7734
+- Generator Exact Match: 96.1449
+- Generator Rouge1: 99.9473
+- Generator Rouge2: 99.8921
+- Generator Rougel: 99.7553
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 2
+- eval_batch_size: 16
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine_with_min_lr
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Selector Count | Generator Count | Selector Accuracy | Generator Exact Match | Generator Rouge1 | Generator Rouge2 | Generator Rougel |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:---------------:|:-----------------:|:---------------------:|:----------------:|:----------------:|:----------------:|
+| 0.0352 | 1.0 | 314 | 0.0162 | 1712.0 | 1712.0 | 95.8528 | 82.8855 | 99.1271 | 98.6581 | 98.5916 |
+| 0.0066 | 2.0 | 628 | 0.0062 | 1712.0 | 1712.0 | 97.9556 | 93.8084 | 99.8307 | 99.7138 | 99.5563 |
+| 0.0035 | 3.0 | 942 | 0.0053 | 1712.0 | 1712.0 | 97.8972 | 95.8528 | 99.8679 | 99.7892 | 99.6975 |
+| 0.0008 | 4.0 | 1256 | 0.0040 | 1712.0 | 1712.0 | 98.7734 | 96.0864 | 99.9527 | 99.8894 | 99.7552 |
+| 0.0023 | 5.0 | 1570 | 0.0039 | 1712.0 | 1712.0 | 98.6565 | 96.2617 | 99.9585 | 99.8979 | 99.795 |
+
+
+### Framework versions
+
+- PEFT 0.18.1
+- Transformers 5.8.0
+- Pytorch 2.9.1+cu128
+- Datasets 4.0.0
+- Tokenizers 0.22.2
\ No newline at end of file
diff --git a/qwen3/rkl/adapter_config.json b/qwen3/rkl/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e870ea0655d09e18c34d7c826373471fbdcbf384
--- /dev/null
+++ b/qwen3/rkl/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "down_proj",
+ "up_proj",
+ "q_proj",
+ "v_proj",
+ "k_proj",
+ "o_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/rkl/adapter_model.safetensors b/qwen3/rkl/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..834562f054e90249b7521cdd219fdb689892d266
--- /dev/null
+++ b/qwen3/rkl/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:449e1623c1ec264231a16314b4aa4316bdfdb75bf206dd613bb1af0a1a0502ef
+size 40422560
diff --git a/qwen3/rkl/all_results.json b/qwen3/rkl/all_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..ade6f008ed1ac071fa15854287dbd8df389f6c8f
--- /dev/null
+++ b/qwen3/rkl/all_results.json
@@ -0,0 +1,19 @@
+{
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.1449,
+ "eval_generator_rouge1": 99.9473,
+ "eval_generator_rouge2": 99.8921,
+ "eval_generator_rougeL": 99.7553,
+ "eval_loss": 0.003887052647769451,
+ "eval_runtime": 585.9552,
+ "eval_samples_per_second": 5.826,
+ "eval_selector_accuracy": 98.7734,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.050996783147655,
+ "train_runtime": 7746.3372,
+ "train_samples_per_second": 6.472,
+ "train_steps_per_second": 0.203
+}
\ No newline at end of file
diff --git a/qwen3/rkl/chat_template.jinja b/qwen3/rkl/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/rkl/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-1256/README.md b/qwen3/rkl/checkpoint-1256/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-1256/adapter_config.json b/qwen3/rkl/checkpoint-1256/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e870ea0655d09e18c34d7c826373471fbdcbf384
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "down_proj",
+ "up_proj",
+ "q_proj",
+ "v_proj",
+ "k_proj",
+ "o_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-1256/adapter_model.safetensors b/qwen3/rkl/checkpoint-1256/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2b6856e64fde3611fd69f1bfc807f98048186f95
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49dd8abe9d38d25baf17ccb7c853a9003a04a8355108fa57b308c7b19c3c038c
+size 40422560
diff --git a/qwen3/rkl/checkpoint-1256/chat_template.jinja b/qwen3/rkl/checkpoint-1256/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/rkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bebd35c791790a5c98cadafcd64b62d51c3e23dd
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38d1059dde6a77b6c1482b3d875459f2a8caaefb76253929a858b42b8ab4df17
+size 121135557
diff --git a/qwen3/rkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/rkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6baa3d9c127ce6205127bef4ce4cf3544ae994aa
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49a60bd75ae50987d6d67449de985b12e350d627ca0b6935b19a3228ce03d657
+size 121135749
diff --git a/qwen3/rkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt b/qwen3/rkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..592782b54e305e92acf976e8469e8bc64e29b831
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ba94a8028bae0ba14a30ed59224be061b20c8a544bb9b769febc757165be06f
+size 351841765
diff --git a/qwen3/rkl/checkpoint-1256/latest b/qwen3/rkl/checkpoint-1256/latest
new file mode 100644
index 0000000000000000000000000000000000000000..74abb3d23ea83ebdac59918752db5d1782834ca7
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/latest
@@ -0,0 +1 @@
+global_step1256
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-1256/rng_state_0.pth b/qwen3/rkl/checkpoint-1256/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..190466218471bcc68ea34393c4aef7b081f3112e
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3beae65d37e0fde2b2b2d57d84db365d2d2c2ff221c3a1ff6299332e0201b81
+size 14917
diff --git a/qwen3/rkl/checkpoint-1256/rng_state_1.pth b/qwen3/rkl/checkpoint-1256/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..e06e1293d88586b6fddf5ac83f5e5915a2e9da8b
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d9b405e9dd4d072fde095004f0791da1a8a35a2d7aa71ad233ad959e87b3b11
+size 14917
diff --git a/qwen3/rkl/checkpoint-1256/scheduler.pt b/qwen3/rkl/checkpoint-1256/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40e863d07cfed22cd912014eb3ba3fd2b16b173f
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1775297f14e75f7463ae51983a66296f81f4b2e2e5d7234f1a6d295abfa4cf23
+size 1465
diff --git a/qwen3/rkl/checkpoint-1256/tokenizer.json b/qwen3/rkl/checkpoint-1256/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/rkl/checkpoint-1256/tokenizer_config.json b/qwen3/rkl/checkpoint-1256/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/rkl/checkpoint-1256/trainer_state.json b/qwen3/rkl/checkpoint-1256/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..1b4bb1eb2f4178763afe607fd88cc44d11d6e3db
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/trainer_state.json
@@ -0,0 +1,528 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 4.0,
+ "eval_steps": 500,
+ "global_step": 1256,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 6.2327961921691895,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 1.962670135498047,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 4.827701568603516,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.43457612991333006,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 2.8958382606506348,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.29139797687530516,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 1.0482853651046753,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.1546297550201416,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 1.5923899412155151,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.1172574520111084,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.9633629322052002,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.08894596099853516,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 1.1507108211517334,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.07953760027885437,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 2.0309667587280273,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.08130698204040528,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 1.1275790929794312,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.054671406745910645,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 1.0041522979736328,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.08292654752731324,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 1.9121218919754028,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.05164262652397156,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.26428329944610596,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.032167059183120725,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.4897637665271759,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.02957485318183899,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.6934449672698975,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.049537676572799685,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.21298079192638397,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.035175830125808716,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 82.8855,
+ "eval_generator_rouge1": 99.1271,
+ "eval_generator_rouge2": 98.6581,
+ "eval_generator_rougeL": 98.5916,
+ "eval_loss": 0.01617370918393135,
+ "eval_runtime": 611.3283,
+ "eval_samples_per_second": 5.585,
+ "eval_selector_accuracy": 95.8528,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 8.917404174804688,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.0453259289264679,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.24316377937793732,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.03384425640106201,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.1466674506664276,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.028050321340560912,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.4751643240451813,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.014143100380897522,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.45425429940223694,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.011174042522907258,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.36087778210639954,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.021321842074394227,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.20291069149971008,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.01833752542734146,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.313603937625885,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.01772385835647583,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.8575339913368225,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.007667127996683121,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.7694051265716553,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.012477626651525497,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.19115960597991943,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.014969517290592194,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.5240126252174377,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.011565862596035004,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.29221007227897644,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.007554464042186737,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.6604180932044983,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.021175915002822877,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.24356646835803986,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.00589771680533886,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.6210055351257324,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.006590907275676727,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.8084,
+ "eval_generator_rouge1": 99.8307,
+ "eval_generator_rouge2": 99.7138,
+ "eval_generator_rougeL": 99.5563,
+ "eval_loss": 0.00618594279512763,
+ "eval_runtime": 615.6447,
+ "eval_samples_per_second": 5.545,
+ "eval_selector_accuracy": 97.9556,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.174,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.10265772044658661,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.006059937924146652,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.5004086494445801,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.012070311605930329,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.07480980455875397,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0035718731582164765,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.01576077565550804,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.004927867650985717,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.0444757305085659,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.00403532050549984,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.09332595765590668,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.004731206595897675,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.0037110887933522463,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0022928833961486815,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.011130036786198616,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.005582543462514878,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.10506990551948547,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.006190977245569229,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.23082585632801056,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.010367032140493393,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.5436015725135803,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.0035324115306138992,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.36652716994285583,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.004156689718365669,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.11949720978736877,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.003846340999007225,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.18794448673725128,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.005602637305855751,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.30812641978263855,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.006678573787212372,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.09404585510492325,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0035119958221912382,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.8528,
+ "eval_generator_rouge1": 99.8679,
+ "eval_generator_rouge2": 99.7892,
+ "eval_generator_rougeL": 99.6975,
+ "eval_loss": 0.0053189839236438274,
+ "eval_runtime": 604.6245,
+ "eval_samples_per_second": 5.646,
+ "eval_selector_accuracy": 97.8972,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.177,
+ "step": 942
+ },
+ {
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.16364775598049164,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.004304204136133194,
+ "step": 960
+ },
+ {
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.042379967868328094,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.001880204677581787,
+ "step": 980
+ },
+ {
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.007348992861807346,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.0023815888911485673,
+ "step": 1000
+ },
+ {
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.08003639429807663,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.006686699390411377,
+ "step": 1020
+ },
+ {
+ "epoch": 3.3127243717590744,
+ "grad_norm": 0.3456681966781616,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.00394720807671547,
+ "step": 1040
+ },
+ {
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.006578488275408745,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.004488777369260788,
+ "step": 1060
+ },
+ {
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.08876754343509674,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.006795570254325867,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.03748416155576706,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0018583754077553748,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.048245642334222794,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.0009692028164863586,
+ "step": 1120
+ },
+ {
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.3362617790699005,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0013890210539102554,
+ "step": 1140
+ },
+ {
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.06560458987951279,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.006552443653345108,
+ "step": 1160
+ },
+ {
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.04897458106279373,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0024676153436303137,
+ "step": 1180
+ },
+ {
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.03441966325044632,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.002887801453471184,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8871160749900278,
+ "grad_norm": 0.1817827820777893,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.005654159188270569,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.01794292777776718,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0008344464935362339,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.0864,
+ "eval_generator_rouge1": 99.9527,
+ "eval_generator_rouge2": 99.8894,
+ "eval_generator_rougeL": 99.7552,
+ "eval_loss": 0.003953293431550264,
+ "eval_runtime": 713.0036,
+ "eval_samples_per_second": 4.788,
+ "eval_selector_accuracy": 98.7734,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.15,
+ "step": 1256
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 4.697643190059008e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/rkl/checkpoint-1256/training_args.bin b/qwen3/rkl/checkpoint-1256/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3862019231d51094f64737a17b9e90c9702be5bb
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc7dbd9317362495edd0aabf350a46bb54254e0f793b789083e43c4c49777755
+size 7185
diff --git a/qwen3/rkl/checkpoint-1256/zero_to_fp32.py b/qwen3/rkl/checkpoint-1256/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1256/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/rkl/checkpoint-1570/README.md b/qwen3/rkl/checkpoint-1570/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-1570/adapter_config.json b/qwen3/rkl/checkpoint-1570/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e870ea0655d09e18c34d7c826373471fbdcbf384
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "down_proj",
+ "up_proj",
+ "q_proj",
+ "v_proj",
+ "k_proj",
+ "o_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-1570/adapter_model.safetensors b/qwen3/rkl/checkpoint-1570/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..834562f054e90249b7521cdd219fdb689892d266
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:449e1623c1ec264231a16314b4aa4316bdfdb75bf206dd613bb1af0a1a0502ef
+size 40422560
diff --git a/qwen3/rkl/checkpoint-1570/chat_template.jinja b/qwen3/rkl/checkpoint-1570/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/rkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7568ade65ab2b9bffdf8b125e24a4eae690f316d
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8ad0afbf9a548f703369da872951e1d9564702aec4ac7cbd6310d13eb3994a4
+size 121135557
diff --git a/qwen3/rkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/rkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c178a0ea0ef304a2e77332e9906ba870cdb80fba
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f59c01db4519a0d6c4329549540a85396b5ac9497686b28c5e4162d7687a8e01
+size 121135749
diff --git a/qwen3/rkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt b/qwen3/rkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e15f8de7d04e89e543d1f85c26a2517036cd4fe4
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7e117cfc87a45c1e34058a6c40d3aed33988819648271296b0b07e2165408c5
+size 351841765
diff --git a/qwen3/rkl/checkpoint-1570/latest b/qwen3/rkl/checkpoint-1570/latest
new file mode 100644
index 0000000000000000000000000000000000000000..18c2fcff58fa6daa0a2e7c3a015552a69085778b
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/latest
@@ -0,0 +1 @@
+global_step1570
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-1570/rng_state_0.pth b/qwen3/rkl/checkpoint-1570/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..75f1d2e845e901a85036f573e43f6596b5295b1e
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2fdb82561540b183a87f7470ab1c56ed743d9a227f225d79b7d8e8146aa51c7
+size 14917
diff --git a/qwen3/rkl/checkpoint-1570/rng_state_1.pth b/qwen3/rkl/checkpoint-1570/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7d19810e93c38d6893eb632dd47a3d343717a7c0
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62bcd73f58cb09b07c70b0042d5a2b436f7385333a81e3f2ea69ecea23fd3690
+size 14917
diff --git a/qwen3/rkl/checkpoint-1570/scheduler.pt b/qwen3/rkl/checkpoint-1570/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e22954463492d8a070ae427c95418df61893c54c
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6332fac17ae99a07597512c705658da2fd8429d1c39bb776b30e107c2cb7aae
+size 1465
diff --git a/qwen3/rkl/checkpoint-1570/tokenizer.json b/qwen3/rkl/checkpoint-1570/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/rkl/checkpoint-1570/tokenizer_config.json b/qwen3/rkl/checkpoint-1570/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/rkl/checkpoint-1570/trainer_state.json b/qwen3/rkl/checkpoint-1570/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..05c18dd3d43feeabfe94afe6fec3e0f51e0620ef
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/trainer_state.json
@@ -0,0 +1,655 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 5.0,
+ "eval_steps": 500,
+ "global_step": 1570,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 6.2327961921691895,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 1.962670135498047,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 4.827701568603516,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.43457612991333006,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 2.8958382606506348,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.29139797687530516,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 1.0482853651046753,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.1546297550201416,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 1.5923899412155151,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.1172574520111084,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.9633629322052002,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.08894596099853516,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 1.1507108211517334,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.07953760027885437,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 2.0309667587280273,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.08130698204040528,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 1.1275790929794312,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.054671406745910645,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 1.0041522979736328,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.08292654752731324,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 1.9121218919754028,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.05164262652397156,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.26428329944610596,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.032167059183120725,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.4897637665271759,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.02957485318183899,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.6934449672698975,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.049537676572799685,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.21298079192638397,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.035175830125808716,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 82.8855,
+ "eval_generator_rouge1": 99.1271,
+ "eval_generator_rouge2": 98.6581,
+ "eval_generator_rougeL": 98.5916,
+ "eval_loss": 0.01617370918393135,
+ "eval_runtime": 611.3283,
+ "eval_samples_per_second": 5.585,
+ "eval_selector_accuracy": 95.8528,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 8.917404174804688,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.0453259289264679,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.24316377937793732,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.03384425640106201,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.1466674506664276,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.028050321340560912,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.4751643240451813,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.014143100380897522,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.45425429940223694,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.011174042522907258,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.36087778210639954,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.021321842074394227,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.20291069149971008,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.01833752542734146,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.313603937625885,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.01772385835647583,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.8575339913368225,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.007667127996683121,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.7694051265716553,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.012477626651525497,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.19115960597991943,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.014969517290592194,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.5240126252174377,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.011565862596035004,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.29221007227897644,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.007554464042186737,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.6604180932044983,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.021175915002822877,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.24356646835803986,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.00589771680533886,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.6210055351257324,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.006590907275676727,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.8084,
+ "eval_generator_rouge1": 99.8307,
+ "eval_generator_rouge2": 99.7138,
+ "eval_generator_rougeL": 99.5563,
+ "eval_loss": 0.00618594279512763,
+ "eval_runtime": 615.6447,
+ "eval_samples_per_second": 5.545,
+ "eval_selector_accuracy": 97.9556,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.174,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.10265772044658661,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.006059937924146652,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.5004086494445801,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.012070311605930329,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.07480980455875397,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0035718731582164765,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.01576077565550804,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.004927867650985717,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.0444757305085659,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.00403532050549984,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.09332595765590668,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.004731206595897675,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.0037110887933522463,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0022928833961486815,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.011130036786198616,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.005582543462514878,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.10506990551948547,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.006190977245569229,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.23082585632801056,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.010367032140493393,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.5436015725135803,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.0035324115306138992,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.36652716994285583,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.004156689718365669,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.11949720978736877,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.003846340999007225,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.18794448673725128,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.005602637305855751,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.30812641978263855,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.006678573787212372,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.09404585510492325,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0035119958221912382,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.8528,
+ "eval_generator_rouge1": 99.8679,
+ "eval_generator_rouge2": 99.7892,
+ "eval_generator_rougeL": 99.6975,
+ "eval_loss": 0.0053189839236438274,
+ "eval_runtime": 604.6245,
+ "eval_samples_per_second": 5.646,
+ "eval_selector_accuracy": 97.8972,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.177,
+ "step": 942
+ },
+ {
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.16364775598049164,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.004304204136133194,
+ "step": 960
+ },
+ {
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.042379967868328094,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.001880204677581787,
+ "step": 980
+ },
+ {
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.007348992861807346,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.0023815888911485673,
+ "step": 1000
+ },
+ {
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.08003639429807663,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.006686699390411377,
+ "step": 1020
+ },
+ {
+ "epoch": 3.3127243717590744,
+ "grad_norm": 0.3456681966781616,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.00394720807671547,
+ "step": 1040
+ },
+ {
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.006578488275408745,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.004488777369260788,
+ "step": 1060
+ },
+ {
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.08876754343509674,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.006795570254325867,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.03748416155576706,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0018583754077553748,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.048245642334222794,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.0009692028164863586,
+ "step": 1120
+ },
+ {
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.3362617790699005,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0013890210539102554,
+ "step": 1140
+ },
+ {
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.06560458987951279,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.006552443653345108,
+ "step": 1160
+ },
+ {
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.04897458106279373,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0024676153436303137,
+ "step": 1180
+ },
+ {
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.03441966325044632,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.002887801453471184,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8871160749900278,
+ "grad_norm": 0.1817827820777893,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.005654159188270569,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.01794292777776718,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0008344464935362339,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.0864,
+ "eval_generator_rouge1": 99.9527,
+ "eval_generator_rouge2": 99.8894,
+ "eval_generator_rougeL": 99.7552,
+ "eval_loss": 0.003953293431550264,
+ "eval_runtime": 713.0036,
+ "eval_samples_per_second": 4.788,
+ "eval_selector_accuracy": 98.7734,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.15,
+ "step": 1256
+ },
+ {
+ "epoch": 4.0127642600717985,
+ "grad_norm": 0.6850450038909912,
+ "learning_rate": 9.464081721960592e-06,
+ "loss": 0.0026287056505680086,
+ "step": 1260
+ },
+ {
+ "epoch": 4.076585560430794,
+ "grad_norm": 0.16716192662715912,
+ "learning_rate": 8.331628936475818e-06,
+ "loss": 0.0009219101630151272,
+ "step": 1280
+ },
+ {
+ "epoch": 4.140406860789788,
+ "grad_norm": 0.0726974830031395,
+ "learning_rate": 7.2659843568748385e-06,
+ "loss": 0.0009689806029200554,
+ "step": 1300
+ },
+ {
+ "epoch": 4.204228161148784,
+ "grad_norm": 0.014396958984434605,
+ "learning_rate": 6.268854516770008e-06,
+ "loss": 0.0010982549749314785,
+ "step": 1320
+ },
+ {
+ "epoch": 4.268049461507778,
+ "grad_norm": 0.035876017063856125,
+ "learning_rate": 5.341836229602331e-06,
+ "loss": 0.00099984435364604,
+ "step": 1340
+ },
+ {
+ "epoch": 4.331870761866773,
+ "grad_norm": 2.9912006855010986,
+ "learning_rate": 4.4864140314888216e-06,
+ "loss": 0.006877808272838593,
+ "step": 1360
+ },
+ {
+ "epoch": 4.395692062225768,
+ "grad_norm": 0.09273593127727509,
+ "learning_rate": 3.703957803871932e-06,
+ "loss": 0.00488009825348854,
+ "step": 1380
+ },
+ {
+ "epoch": 4.4595133625847625,
+ "grad_norm": 0.030779356136918068,
+ "learning_rate": 2.9957205797780495e-06,
+ "loss": 0.0020341061055660246,
+ "step": 1400
+ },
+ {
+ "epoch": 4.523334662943758,
+ "grad_norm": 0.14085310697555542,
+ "learning_rate": 2.362836537198199e-06,
+ "loss": 0.0019017530605196953,
+ "step": 1420
+ },
+ {
+ "epoch": 4.587155963302752,
+ "grad_norm": 0.013417008332908154,
+ "learning_rate": 1.8063191828043703e-06,
+ "loss": 0.0017165429890155792,
+ "step": 1440
+ },
+ {
+ "epoch": 4.650977263661747,
+ "grad_norm": 0.015544545836746693,
+ "learning_rate": 1.3270597289100437e-06,
+ "loss": 0.0006548282690346241,
+ "step": 1460
+ },
+ {
+ "epoch": 4.714798564020742,
+ "grad_norm": 0.019597982987761497,
+ "learning_rate": 9.258256662741205e-07,
+ "loss": 0.0006419078446924687,
+ "step": 1480
+ },
+ {
+ "epoch": 4.778619864379737,
+ "grad_norm": 0.018851926550269127,
+ "learning_rate": 6.032595350337341e-07,
+ "loss": 0.001545669510960579,
+ "step": 1500
+ },
+ {
+ "epoch": 4.842441164738732,
+ "grad_norm": 7.279566287994385,
+ "learning_rate": 3.5987789573421356e-07,
+ "loss": 0.009023332595825195,
+ "step": 1520
+ },
+ {
+ "epoch": 4.9062624650977265,
+ "grad_norm": 0.08475331962108612,
+ "learning_rate": 1.9607050210401192e-07,
+ "loss": 0.0006238400470465422,
+ "step": 1540
+ },
+ {
+ "epoch": 4.970083765456721,
+ "grad_norm": 0.04514538496732712,
+ "learning_rate": 1.1209967689925832e-07,
+ "loss": 0.0023126199841499327,
+ "step": 1560
+ },
+ {
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.2617,
+ "eval_generator_rouge1": 99.9585,
+ "eval_generator_rouge2": 99.8979,
+ "eval_generator_rougeL": 99.795,
+ "eval_loss": 0.003887052647769451,
+ "eval_runtime": 600.6059,
+ "eval_samples_per_second": 5.684,
+ "eval_selector_accuracy": 98.6565,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.178,
+ "step": 1570
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 5.871444840310374e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/rkl/checkpoint-1570/training_args.bin b/qwen3/rkl/checkpoint-1570/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3862019231d51094f64737a17b9e90c9702be5bb
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc7dbd9317362495edd0aabf350a46bb54254e0f793b789083e43c4c49777755
+size 7185
diff --git a/qwen3/rkl/checkpoint-1570/zero_to_fp32.py b/qwen3/rkl/checkpoint-1570/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/rkl/checkpoint-1570/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/rkl/checkpoint-314/README.md b/qwen3/rkl/checkpoint-314/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-314/adapter_config.json b/qwen3/rkl/checkpoint-314/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e870ea0655d09e18c34d7c826373471fbdcbf384
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "down_proj",
+ "up_proj",
+ "q_proj",
+ "v_proj",
+ "k_proj",
+ "o_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-314/adapter_model.safetensors b/qwen3/rkl/checkpoint-314/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8567670f0c7fa6e384637bea3f02a3e564be6345
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60909bcb7be42193d204715cb7a7b6dc0795ceca604e686cdd4264a3ab6f9831
+size 40422560
diff --git a/qwen3/rkl/checkpoint-314/chat_template.jinja b/qwen3/rkl/checkpoint-314/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/rkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f59f6ebcef064c244153312f3bc251f8251149f7
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5083536bcc7408de0bf1b97680f18a3fcf32ea4ad7bef22eb789d9140c665ab1
+size 121135557
diff --git a/qwen3/rkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/rkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f738f1611b19d23763d988dac00e3cb3ce3b3065
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a6b9eb6bc8336c3eb795dc55ce5bdff10f89f631e1a65cf3ea5893c9f9c27a0
+size 121135749
diff --git a/qwen3/rkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt b/qwen3/rkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6e400662f44278607e471faa57dd444b08cd8b9a
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:356401f40ed313b45f502ae710237c169fac408dabdc462723253780728ddc70
+size 351841765
diff --git a/qwen3/rkl/checkpoint-314/latest b/qwen3/rkl/checkpoint-314/latest
new file mode 100644
index 0000000000000000000000000000000000000000..a9786513b1a514db2c3a9da7408844635663d55d
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/latest
@@ -0,0 +1 @@
+global_step314
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-314/rng_state_0.pth b/qwen3/rkl/checkpoint-314/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..c19d182f7da7e01e3125c88df59af455448cb599
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f2d5291a83119dec9bbc7b89132c07385432cac09a49c454147928c92ee7fb2
+size 14917
diff --git a/qwen3/rkl/checkpoint-314/rng_state_1.pth b/qwen3/rkl/checkpoint-314/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d5c9bc64659877d1b64da510823572bfb3a6f25f
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac6b8ede8cb4bb1d8252ed32deaef002a4dec03d3dd98dca92ba879d170030e9
+size 14917
diff --git a/qwen3/rkl/checkpoint-314/scheduler.pt b/qwen3/rkl/checkpoint-314/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3cb460802cda03bf0833a98a6f1428a3fd4f585
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17e254b5642bb75983fd5105bec79b35376cfbc99d29702287668b8767d46149
+size 1465
diff --git a/qwen3/rkl/checkpoint-314/tokenizer.json b/qwen3/rkl/checkpoint-314/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/rkl/checkpoint-314/tokenizer_config.json b/qwen3/rkl/checkpoint-314/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/rkl/checkpoint-314/trainer_state.json b/qwen3/rkl/checkpoint-314/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..71db5192dc326c65765e5145759cfaa250c17fd9
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/trainer_state.json
@@ -0,0 +1,154 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 1.0,
+ "eval_steps": 500,
+ "global_step": 314,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 6.2327961921691895,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 1.962670135498047,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 4.827701568603516,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.43457612991333006,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 2.8958382606506348,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.29139797687530516,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 1.0482853651046753,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.1546297550201416,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 1.5923899412155151,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.1172574520111084,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.9633629322052002,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.08894596099853516,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 1.1507108211517334,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.07953760027885437,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 2.0309667587280273,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.08130698204040528,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 1.1275790929794312,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.054671406745910645,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 1.0041522979736328,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.08292654752731324,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 1.9121218919754028,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.05164262652397156,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.26428329944610596,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.032167059183120725,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.4897637665271759,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.02957485318183899,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.6934449672698975,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.049537676572799685,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.21298079192638397,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.035175830125808716,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 82.8855,
+ "eval_generator_rouge1": 99.1271,
+ "eval_generator_rouge2": 98.6581,
+ "eval_generator_rougeL": 98.5916,
+ "eval_loss": 0.01617370918393135,
+ "eval_runtime": 611.3283,
+ "eval_samples_per_second": 5.585,
+ "eval_selector_accuracy": 95.8528,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 314
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 1.172842003562496e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/rkl/checkpoint-314/training_args.bin b/qwen3/rkl/checkpoint-314/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3862019231d51094f64737a17b9e90c9702be5bb
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc7dbd9317362495edd0aabf350a46bb54254e0f793b789083e43c4c49777755
+size 7185
diff --git a/qwen3/rkl/checkpoint-314/zero_to_fp32.py b/qwen3/rkl/checkpoint-314/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/rkl/checkpoint-314/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/rkl/checkpoint-628/README.md b/qwen3/rkl/checkpoint-628/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-628/adapter_config.json b/qwen3/rkl/checkpoint-628/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e870ea0655d09e18c34d7c826373471fbdcbf384
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "down_proj",
+ "up_proj",
+ "q_proj",
+ "v_proj",
+ "k_proj",
+ "o_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-628/adapter_model.safetensors b/qwen3/rkl/checkpoint-628/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..076e2f8e65aeb1a536101c6b82624b526550ca3f
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f975b1e012707fc9428476ee3f5e24e697ecb55dd4911c453dca7cdacc1af69
+size 40422560
diff --git a/qwen3/rkl/checkpoint-628/chat_template.jinja b/qwen3/rkl/checkpoint-628/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/rkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a569a97becb38dee35ee503b7e3aa14a31ae597b
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:110ca9a3f0373158e1db79c8db7b76a939c391b4eb3bb3fc2c9a35f670c7f5a9
+size 121135557
diff --git a/qwen3/rkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/rkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f6c3e0a3e0ca2c07c3cee674334d8f2abe78083
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1702f4bcaa69237b313b30dbd70e6d1313524a59dafcd0dafd5a43bc81f4c062
+size 121135749
diff --git a/qwen3/rkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt b/qwen3/rkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..065be553a394ad6a3958fd88a22bd159a0a29aa9
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a20cd0c4c6de58af33a3554220783195ffd528de116adad140acf577789682a5
+size 351841765
diff --git a/qwen3/rkl/checkpoint-628/latest b/qwen3/rkl/checkpoint-628/latest
new file mode 100644
index 0000000000000000000000000000000000000000..af055c3db05dcb2af541005f22308dbd946aba65
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/latest
@@ -0,0 +1 @@
+global_step628
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-628/rng_state_0.pth b/qwen3/rkl/checkpoint-628/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..09da9eb118e054db6e8710b60d85ba41c779a41c
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56026660f59155b19639072162bc8176a670667cf28669147c900d7817aa85c1
+size 14917
diff --git a/qwen3/rkl/checkpoint-628/rng_state_1.pth b/qwen3/rkl/checkpoint-628/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..e4d5567b152d32ea7a94f3f5f896044a42fe86f3
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9de1c85ea59b5ba32da2b05d04c7595bc85dd244a3fa6f91a184bf6619ae765
+size 14917
diff --git a/qwen3/rkl/checkpoint-628/scheduler.pt b/qwen3/rkl/checkpoint-628/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96c0cabcd99be4945ffbd7897ecc5367ac4beb28
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb823a2c51d7fb42dfe3b693a7159784285dbf38c5a797e6e73292096123ab7a
+size 1465
diff --git a/qwen3/rkl/checkpoint-628/tokenizer.json b/qwen3/rkl/checkpoint-628/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/rkl/checkpoint-628/tokenizer_config.json b/qwen3/rkl/checkpoint-628/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/rkl/checkpoint-628/trainer_state.json b/qwen3/rkl/checkpoint-628/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..8d982ac8340202c632548a2616817fbcccd10e4f
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/trainer_state.json
@@ -0,0 +1,281 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 2.0,
+ "eval_steps": 500,
+ "global_step": 628,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 6.2327961921691895,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 1.962670135498047,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 4.827701568603516,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.43457612991333006,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 2.8958382606506348,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.29139797687530516,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 1.0482853651046753,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.1546297550201416,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 1.5923899412155151,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.1172574520111084,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.9633629322052002,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.08894596099853516,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 1.1507108211517334,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.07953760027885437,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 2.0309667587280273,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.08130698204040528,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 1.1275790929794312,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.054671406745910645,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 1.0041522979736328,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.08292654752731324,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 1.9121218919754028,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.05164262652397156,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.26428329944610596,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.032167059183120725,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.4897637665271759,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.02957485318183899,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.6934449672698975,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.049537676572799685,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.21298079192638397,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.035175830125808716,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 82.8855,
+ "eval_generator_rouge1": 99.1271,
+ "eval_generator_rouge2": 98.6581,
+ "eval_generator_rougeL": 98.5916,
+ "eval_loss": 0.01617370918393135,
+ "eval_runtime": 611.3283,
+ "eval_samples_per_second": 5.585,
+ "eval_selector_accuracy": 95.8528,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 8.917404174804688,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.0453259289264679,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.24316377937793732,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.03384425640106201,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.1466674506664276,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.028050321340560912,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.4751643240451813,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.014143100380897522,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.45425429940223694,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.011174042522907258,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.36087778210639954,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.021321842074394227,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.20291069149971008,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.01833752542734146,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.313603937625885,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.01772385835647583,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.8575339913368225,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.007667127996683121,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.7694051265716553,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.012477626651525497,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.19115960597991943,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.014969517290592194,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.5240126252174377,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.011565862596035004,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.29221007227897644,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.007554464042186737,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.6604180932044983,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.021175915002822877,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.24356646835803986,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.00589771680533886,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.6210055351257324,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.006590907275676727,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.8084,
+ "eval_generator_rouge1": 99.8307,
+ "eval_generator_rouge2": 99.7138,
+ "eval_generator_rougeL": 99.5563,
+ "eval_loss": 0.00618594279512763,
+ "eval_runtime": 615.6447,
+ "eval_samples_per_second": 5.545,
+ "eval_selector_accuracy": 97.9556,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.174,
+ "step": 628
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 2.346351770520781e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/rkl/checkpoint-628/training_args.bin b/qwen3/rkl/checkpoint-628/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3862019231d51094f64737a17b9e90c9702be5bb
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc7dbd9317362495edd0aabf350a46bb54254e0f793b789083e43c4c49777755
+size 7185
diff --git a/qwen3/rkl/checkpoint-628/zero_to_fp32.py b/qwen3/rkl/checkpoint-628/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/rkl/checkpoint-628/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/rkl/checkpoint-942/README.md b/qwen3/rkl/checkpoint-942/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-942/adapter_config.json b/qwen3/rkl/checkpoint-942/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e870ea0655d09e18c34d7c826373471fbdcbf384
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "down_proj",
+ "up_proj",
+ "q_proj",
+ "v_proj",
+ "k_proj",
+ "o_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-942/adapter_model.safetensors b/qwen3/rkl/checkpoint-942/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..594d6b882d29f054615774d0c6c880abc5a9c1cf
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3133da46bc7494b6b503ee48cfd311d08b4c3055357242024c8a9dbde2fa74f4
+size 40422560
diff --git a/qwen3/rkl/checkpoint-942/chat_template.jinja b/qwen3/rkl/checkpoint-942/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/rkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6931a08c94de7a9f37c2467bed644c88dff26283
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0540c9a376bd25ed0fee441d3fd5a692fa71aee6fa355dcc4380b24438fbcfa7
+size 121135557
diff --git a/qwen3/rkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/rkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7ca57827cf8806964dc0ea0a6e0f1c866eaf259
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b3d9ee9bde9b47bb788ebe6ad1fe734538be8695809eeb9453c6b8f13924ff2
+size 121135749
diff --git a/qwen3/rkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt b/qwen3/rkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..463fc1f7c0df81a41ce1915f4348a90c31c4e149
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd29ae2e5f84f5361cb3171524e6bc827cd8573c8bf257405049c8cc49d843cf
+size 351841765
diff --git a/qwen3/rkl/checkpoint-942/latest b/qwen3/rkl/checkpoint-942/latest
new file mode 100644
index 0000000000000000000000000000000000000000..4e9a68e4460aca593c480611a486d2a4cd73ef42
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/latest
@@ -0,0 +1 @@
+global_step942
\ No newline at end of file
diff --git a/qwen3/rkl/checkpoint-942/rng_state_0.pth b/qwen3/rkl/checkpoint-942/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..581b6f970ee18b22f0dde019e096507d57f9e9cc
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8f0300abee2f4a5dfe70668daa0bb6f92a4eadda3bfa2811c6205e2425edca5
+size 14917
diff --git a/qwen3/rkl/checkpoint-942/rng_state_1.pth b/qwen3/rkl/checkpoint-942/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..1a00ca0984a90d5930a1d963dc524f341ccbd0a5
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a9b0702f878f854cfa02bca1a4a0b014407e5e807b189294a3eac7bc5e36eb7
+size 14917
diff --git a/qwen3/rkl/checkpoint-942/scheduler.pt b/qwen3/rkl/checkpoint-942/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b1f8af0480bf83ab728abfeff11bc3288214349
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8076da6f897951c5bdf8649e4fba596e8c49819927746017442ffd85f40b18a3
+size 1465
diff --git a/qwen3/rkl/checkpoint-942/tokenizer.json b/qwen3/rkl/checkpoint-942/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/rkl/checkpoint-942/tokenizer_config.json b/qwen3/rkl/checkpoint-942/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/rkl/checkpoint-942/trainer_state.json b/qwen3/rkl/checkpoint-942/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..c146dada4433b8e800acbab80fe1f1eac2881ef2
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/trainer_state.json
@@ -0,0 +1,408 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 3.0,
+ "eval_steps": 500,
+ "global_step": 942,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 6.2327961921691895,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 1.962670135498047,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 4.827701568603516,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.43457612991333006,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 2.8958382606506348,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.29139797687530516,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 1.0482853651046753,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.1546297550201416,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 1.5923899412155151,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.1172574520111084,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.9633629322052002,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.08894596099853516,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 1.1507108211517334,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.07953760027885437,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 2.0309667587280273,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.08130698204040528,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 1.1275790929794312,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.054671406745910645,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 1.0041522979736328,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.08292654752731324,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 1.9121218919754028,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.05164262652397156,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.26428329944610596,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.032167059183120725,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.4897637665271759,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.02957485318183899,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.6934449672698975,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.049537676572799685,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.21298079192638397,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.035175830125808716,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 82.8855,
+ "eval_generator_rouge1": 99.1271,
+ "eval_generator_rouge2": 98.6581,
+ "eval_generator_rougeL": 98.5916,
+ "eval_loss": 0.01617370918393135,
+ "eval_runtime": 611.3283,
+ "eval_samples_per_second": 5.585,
+ "eval_selector_accuracy": 95.8528,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 8.917404174804688,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.0453259289264679,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.24316377937793732,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.03384425640106201,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.1466674506664276,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.028050321340560912,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.4751643240451813,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.014143100380897522,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.45425429940223694,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.011174042522907258,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.36087778210639954,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.021321842074394227,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.20291069149971008,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.01833752542734146,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.313603937625885,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.01772385835647583,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.8575339913368225,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.007667127996683121,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.7694051265716553,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.012477626651525497,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.19115960597991943,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.014969517290592194,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.5240126252174377,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.011565862596035004,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.29221007227897644,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.007554464042186737,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.6604180932044983,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.021175915002822877,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.24356646835803986,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.00589771680533886,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.6210055351257324,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.006590907275676727,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.8084,
+ "eval_generator_rouge1": 99.8307,
+ "eval_generator_rouge2": 99.7138,
+ "eval_generator_rougeL": 99.5563,
+ "eval_loss": 0.00618594279512763,
+ "eval_runtime": 615.6447,
+ "eval_samples_per_second": 5.545,
+ "eval_selector_accuracy": 97.9556,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.174,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.10265772044658661,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.006059937924146652,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.5004086494445801,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.012070311605930329,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.07480980455875397,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0035718731582164765,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.01576077565550804,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.004927867650985717,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.0444757305085659,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.00403532050549984,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.09332595765590668,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.004731206595897675,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.0037110887933522463,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0022928833961486815,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.011130036786198616,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.005582543462514878,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.10506990551948547,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.006190977245569229,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.23082585632801056,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.010367032140493393,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.5436015725135803,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.0035324115306138992,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.36652716994285583,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.004156689718365669,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.11949720978736877,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.003846340999007225,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.18794448673725128,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.005602637305855751,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.30812641978263855,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.006678573787212372,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.09404585510492325,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0035119958221912382,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.8528,
+ "eval_generator_rouge1": 99.8679,
+ "eval_generator_rouge2": 99.7892,
+ "eval_generator_rougeL": 99.6975,
+ "eval_loss": 0.0053189839236438274,
+ "eval_runtime": 604.6245,
+ "eval_samples_per_second": 5.646,
+ "eval_selector_accuracy": 97.8972,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.177,
+ "step": 942
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 3.524157743353037e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/rkl/checkpoint-942/training_args.bin b/qwen3/rkl/checkpoint-942/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3862019231d51094f64737a17b9e90c9702be5bb
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc7dbd9317362495edd0aabf350a46bb54254e0f793b789083e43c4c49777755
+size 7185
diff --git a/qwen3/rkl/checkpoint-942/zero_to_fp32.py b/qwen3/rkl/checkpoint-942/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/rkl/checkpoint-942/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/rkl/eval_results.json b/qwen3/rkl/eval_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..f32de21e3bd604066e002026f2dea417a3d1e4d7
--- /dev/null
+++ b/qwen3/rkl/eval_results.json
@@ -0,0 +1,14 @@
+{
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.1449,
+ "eval_generator_rouge1": 99.9473,
+ "eval_generator_rouge2": 99.8921,
+ "eval_generator_rougeL": 99.7553,
+ "eval_loss": 0.003887052647769451,
+ "eval_runtime": 585.9552,
+ "eval_samples_per_second": 5.826,
+ "eval_selector_accuracy": 98.7734,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183
+}
\ No newline at end of file
diff --git a/qwen3/rkl/tokenizer.json b/qwen3/rkl/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/rkl/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/rkl/tokenizer_config.json b/qwen3/rkl/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/rkl/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/rkl/train_results.json b/qwen3/rkl/train_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..aab7291c8cc604b56a0d9190e91f66ae47abf778
--- /dev/null
+++ b/qwen3/rkl/train_results.json
@@ -0,0 +1,8 @@
+{
+ "epoch": 5.0,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.050996783147655,
+ "train_runtime": 7746.3372,
+ "train_samples_per_second": 6.472,
+ "train_steps_per_second": 0.203
+}
\ No newline at end of file
diff --git a/qwen3/rkl/trainer_state.json b/qwen3/rkl/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..02188852a23b2fe351dace2fbeb19a8f91290bf9
--- /dev/null
+++ b/qwen3/rkl/trainer_state.json
@@ -0,0 +1,664 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 5.0,
+ "eval_steps": 500,
+ "global_step": 1570,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 6.2327961921691895,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 1.962670135498047,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 4.827701568603516,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.43457612991333006,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 2.8958382606506348,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.29139797687530516,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 1.0482853651046753,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.1546297550201416,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 1.5923899412155151,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.1172574520111084,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.9633629322052002,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.08894596099853516,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 1.1507108211517334,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.07953760027885437,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 2.0309667587280273,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.08130698204040528,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 1.1275790929794312,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.054671406745910645,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 1.0041522979736328,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.08292654752731324,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 1.9121218919754028,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.05164262652397156,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.26428329944610596,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.032167059183120725,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.4897637665271759,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.02957485318183899,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.6934449672698975,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.049537676572799685,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.21298079192638397,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.035175830125808716,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 82.8855,
+ "eval_generator_rouge1": 99.1271,
+ "eval_generator_rouge2": 98.6581,
+ "eval_generator_rougeL": 98.5916,
+ "eval_loss": 0.01617370918393135,
+ "eval_runtime": 611.3283,
+ "eval_samples_per_second": 5.585,
+ "eval_selector_accuracy": 95.8528,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 8.917404174804688,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.0453259289264679,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.24316377937793732,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.03384425640106201,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.1466674506664276,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.028050321340560912,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.4751643240451813,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.014143100380897522,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.45425429940223694,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.011174042522907258,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.36087778210639954,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.021321842074394227,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.20291069149971008,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.01833752542734146,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.313603937625885,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.01772385835647583,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.8575339913368225,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.007667127996683121,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.7694051265716553,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.012477626651525497,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.19115960597991943,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.014969517290592194,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.5240126252174377,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.011565862596035004,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.29221007227897644,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.007554464042186737,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.6604180932044983,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.021175915002822877,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.24356646835803986,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.00589771680533886,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.6210055351257324,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.006590907275676727,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.8084,
+ "eval_generator_rouge1": 99.8307,
+ "eval_generator_rouge2": 99.7138,
+ "eval_generator_rougeL": 99.5563,
+ "eval_loss": 0.00618594279512763,
+ "eval_runtime": 615.6447,
+ "eval_samples_per_second": 5.545,
+ "eval_selector_accuracy": 97.9556,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.174,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.10265772044658661,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.006059937924146652,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.5004086494445801,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.012070311605930329,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.07480980455875397,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0035718731582164765,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.01576077565550804,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.004927867650985717,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.0444757305085659,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.00403532050549984,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.09332595765590668,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.004731206595897675,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.0037110887933522463,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0022928833961486815,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.011130036786198616,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.005582543462514878,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.10506990551948547,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.006190977245569229,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.23082585632801056,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.010367032140493393,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.5436015725135803,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.0035324115306138992,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.36652716994285583,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.004156689718365669,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.11949720978736877,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.003846340999007225,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.18794448673725128,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.005602637305855751,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.30812641978263855,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.006678573787212372,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.09404585510492325,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0035119958221912382,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.8528,
+ "eval_generator_rouge1": 99.8679,
+ "eval_generator_rouge2": 99.7892,
+ "eval_generator_rougeL": 99.6975,
+ "eval_loss": 0.0053189839236438274,
+ "eval_runtime": 604.6245,
+ "eval_samples_per_second": 5.646,
+ "eval_selector_accuracy": 97.8972,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.177,
+ "step": 942
+ },
+ {
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.16364775598049164,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.004304204136133194,
+ "step": 960
+ },
+ {
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.042379967868328094,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.001880204677581787,
+ "step": 980
+ },
+ {
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.007348992861807346,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.0023815888911485673,
+ "step": 1000
+ },
+ {
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.08003639429807663,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.006686699390411377,
+ "step": 1020
+ },
+ {
+ "epoch": 3.3127243717590744,
+ "grad_norm": 0.3456681966781616,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.00394720807671547,
+ "step": 1040
+ },
+ {
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.006578488275408745,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.004488777369260788,
+ "step": 1060
+ },
+ {
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.08876754343509674,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.006795570254325867,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.03748416155576706,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0018583754077553748,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.048245642334222794,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.0009692028164863586,
+ "step": 1120
+ },
+ {
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.3362617790699005,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0013890210539102554,
+ "step": 1140
+ },
+ {
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.06560458987951279,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.006552443653345108,
+ "step": 1160
+ },
+ {
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.04897458106279373,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0024676153436303137,
+ "step": 1180
+ },
+ {
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.03441966325044632,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.002887801453471184,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8871160749900278,
+ "grad_norm": 0.1817827820777893,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.005654159188270569,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.01794292777776718,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0008344464935362339,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.0864,
+ "eval_generator_rouge1": 99.9527,
+ "eval_generator_rouge2": 99.8894,
+ "eval_generator_rougeL": 99.7552,
+ "eval_loss": 0.003953293431550264,
+ "eval_runtime": 713.0036,
+ "eval_samples_per_second": 4.788,
+ "eval_selector_accuracy": 98.7734,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.15,
+ "step": 1256
+ },
+ {
+ "epoch": 4.0127642600717985,
+ "grad_norm": 0.6850450038909912,
+ "learning_rate": 9.464081721960592e-06,
+ "loss": 0.0026287056505680086,
+ "step": 1260
+ },
+ {
+ "epoch": 4.076585560430794,
+ "grad_norm": 0.16716192662715912,
+ "learning_rate": 8.331628936475818e-06,
+ "loss": 0.0009219101630151272,
+ "step": 1280
+ },
+ {
+ "epoch": 4.140406860789788,
+ "grad_norm": 0.0726974830031395,
+ "learning_rate": 7.2659843568748385e-06,
+ "loss": 0.0009689806029200554,
+ "step": 1300
+ },
+ {
+ "epoch": 4.204228161148784,
+ "grad_norm": 0.014396958984434605,
+ "learning_rate": 6.268854516770008e-06,
+ "loss": 0.0010982549749314785,
+ "step": 1320
+ },
+ {
+ "epoch": 4.268049461507778,
+ "grad_norm": 0.035876017063856125,
+ "learning_rate": 5.341836229602331e-06,
+ "loss": 0.00099984435364604,
+ "step": 1340
+ },
+ {
+ "epoch": 4.331870761866773,
+ "grad_norm": 2.9912006855010986,
+ "learning_rate": 4.4864140314888216e-06,
+ "loss": 0.006877808272838593,
+ "step": 1360
+ },
+ {
+ "epoch": 4.395692062225768,
+ "grad_norm": 0.09273593127727509,
+ "learning_rate": 3.703957803871932e-06,
+ "loss": 0.00488009825348854,
+ "step": 1380
+ },
+ {
+ "epoch": 4.4595133625847625,
+ "grad_norm": 0.030779356136918068,
+ "learning_rate": 2.9957205797780495e-06,
+ "loss": 0.0020341061055660246,
+ "step": 1400
+ },
+ {
+ "epoch": 4.523334662943758,
+ "grad_norm": 0.14085310697555542,
+ "learning_rate": 2.362836537198199e-06,
+ "loss": 0.0019017530605196953,
+ "step": 1420
+ },
+ {
+ "epoch": 4.587155963302752,
+ "grad_norm": 0.013417008332908154,
+ "learning_rate": 1.8063191828043703e-06,
+ "loss": 0.0017165429890155792,
+ "step": 1440
+ },
+ {
+ "epoch": 4.650977263661747,
+ "grad_norm": 0.015544545836746693,
+ "learning_rate": 1.3270597289100437e-06,
+ "loss": 0.0006548282690346241,
+ "step": 1460
+ },
+ {
+ "epoch": 4.714798564020742,
+ "grad_norm": 0.019597982987761497,
+ "learning_rate": 9.258256662741205e-07,
+ "loss": 0.0006419078446924687,
+ "step": 1480
+ },
+ {
+ "epoch": 4.778619864379737,
+ "grad_norm": 0.018851926550269127,
+ "learning_rate": 6.032595350337341e-07,
+ "loss": 0.001545669510960579,
+ "step": 1500
+ },
+ {
+ "epoch": 4.842441164738732,
+ "grad_norm": 7.279566287994385,
+ "learning_rate": 3.5987789573421356e-07,
+ "loss": 0.009023332595825195,
+ "step": 1520
+ },
+ {
+ "epoch": 4.9062624650977265,
+ "grad_norm": 0.08475331962108612,
+ "learning_rate": 1.9607050210401192e-07,
+ "loss": 0.0006238400470465422,
+ "step": 1540
+ },
+ {
+ "epoch": 4.970083765456721,
+ "grad_norm": 0.04514538496732712,
+ "learning_rate": 1.1209967689925832e-07,
+ "loss": 0.0023126199841499327,
+ "step": 1560
+ },
+ {
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.2617,
+ "eval_generator_rouge1": 99.9585,
+ "eval_generator_rouge2": 99.8979,
+ "eval_generator_rougeL": 99.795,
+ "eval_loss": 0.003887052647769451,
+ "eval_runtime": 600.6059,
+ "eval_samples_per_second": 5.684,
+ "eval_selector_accuracy": 98.6565,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.178,
+ "step": 1570
+ },
+ {
+ "epoch": 5.0,
+ "step": 1570,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.050996783147655,
+ "train_runtime": 7746.3372,
+ "train_samples_per_second": 6.472,
+ "train_steps_per_second": 0.203
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 5.871444840310374e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/rkl/training_args.bin b/qwen3/rkl/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3862019231d51094f64737a17b9e90c9702be5bb
--- /dev/null
+++ b/qwen3/rkl/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc7dbd9317362495edd0aabf350a46bb54254e0f793b789083e43c4c49777755
+size 7185
diff --git a/qwen3/run_all_logs/20260822_031516/amid.log b/qwen3/run_all_logs/20260822_031516/amid.log
new file mode 100644
index 0000000000000000000000000000000000000000..a6d3c3476031f14ce36d4df23ee3551f3ec7d023
--- /dev/null
+++ b/qwen3/run_all_logs/20260822_031516/amid.log
@@ -0,0 +1,2100 @@
+W0823 00:23:43.881000 1404762 torch/distributed/run.py:803]
+W0823 00:23:43.881000 1404762 torch/distributed/run.py:803] *****************************************
+W0823 00:23:43.881000 1404762 torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+W0823 00:23:43.881000 1404762 torch/distributed/run.py:803] *****************************************
+[INFO|2026-08-23 00:23:50] llamafactory.hparams.parser:651 >> Process rank: 1, world size: 2, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16
+[INFO|2026-08-23 00:23:50] llamafactory.hparams.parser:144 >> Set `ddp_find_unused_parameters` to False in DDP training since LoRA is enabled.
+[INFO|2026-08-23 00:23:50] llamafactory.hparams.parser:651 >> Process rank: 0, world size: 2, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16
+Starting amid with LlamaFactory and DeepSpeed=True.
+[INFO|configuration_utils.py:780] 2026-08-23 00:23:51,961 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 00:23:51,967 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-23 00:23:55,043 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 00:23:55,044 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-23 00:23:55,281 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 00:23:55,283 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-23 00:23:56] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_train.jsonl...
+[INFO|2026-08-23 00:23:57] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_eval.jsonl...
+training example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 1947, 19029, 10592, 311, 35592, 304, 279, 17328, 1947, 1707, 429, 9482, 1573, 279, 1042, 220, 16, 21, 15, 15, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 9286, 38050, 756, 414, 330, 13193, 788, 341, 286, 330, 408, 14645, 788, 330, 68689, 756, 286, 330, 2468, 14645, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 37525, 756, 414, 330, 13193, 788, 5613, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 18098, 287, 756, 414, 330, 13193, 788, 341, 286, 330, 11141, 3575, 34043, 788, 330, 16468, 756, 286, 330, 37375, 14645, 788, 330, 68689, 698, 414, 456, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 9286, 38050, 756, 414, 330, 1313, 788, 330, 49087, 2354, 698, 262, 1153, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 37525, 756, 414, 330, 1313, 788, 330, 4648, 37525, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of art movements linked to paintings in the genre art style that ended before the year 1600?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "ArtMovement",
+ "properties": {
+ "end_year": "INTEGER",
+ "start_year": "INTEGER"
+ }
+ },
+ {
+ "label": "Genre",
+ "properties": {}
+ },
+ {
+ "label": "Painting",
+ "properties": {
+ "country_of_origin": "STRING",
+ "creation_year": "INTEGER"
+ }
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "ArtMovement",
+ "type": "associatedWith"
+ },
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "Genre",
+ "type": "hasGenre"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+eval example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 9893, 1483, 304, 2987, 279, 220, 17, 15, 17, 16, 55218, 18144, 73379, 476, 279, 23007, 24847, 7755, 9257, 36331, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 13193, 788, 341, 286, 330, 1028, 788, 330, 7097, 756, 286, 330, 31309, 788, 330, 22852, 756, 286, 330, 4082, 3575, 2259, 26286, 788, 330, 68689, 756, 286, 330, 4082, 3575, 1243, 89827, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 28214, 756, 414, 330, 13193, 788, 5613, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 5657, 788, 330, 28214, 756, 414, 330, 1313, 788, 330, 30210, 1047, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of weapons used in either the 2021 Baghdad suicide bombings or the Botroseya Church bombing?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "TerroristAttack",
+ "properties": {
+ "date": "DATE",
+ "locations": "LIST",
+ "number_of_deaths": "INTEGER",
+ "number_of_injuries": "INTEGER"
+ }
+ },
+ {
+ "label": "Weapon",
+ "properties": {}
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "TerroristAttack",
+ "target": "Weapon",
+ "type": "employs"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+[INFO|configuration_utils.py:780] 2026-08-23 00:24:01,172 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 00:24:01,174 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-23 00:24:01] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training.
+[INFO|modeling_utils.py:769] 2026-08-23 00:24:01,887 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/model.safetensors
+[INFO|modeling_utils.py:839] 2026-08-23 00:24:01,887 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-23 00:24:01,888 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": false
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-23 00:24:02,581 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95
+}
+
+[INFO|2026-08-23 00:24:02] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled.
+[INFO|2026-08-23 00:24:02] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-23 00:24:02] llamafactory.model.adapter:144 >> Upcasting trainable params to float32.
+[INFO|2026-08-23 00:24:02] llamafactory.model.adapter:144 >> Fine-tuning method: LoRA
+[INFO|2026-08-23 00:24:02] llamafactory.model.model_utils.misc:144 >> Found linear modules: v_proj,q_proj,o_proj,gate_proj,up_proj,k_proj,down_proj
+[INFO|2026-08-23 00:24:03] llamafactory.model.loader:144 >> trainable params: 20,185,088 || all params: 616,235,008 || trainable%: 3.2756
+[INFO|configuration_utils.py:780] 2026-08-23 00:24:03,917 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 00:24:03,920 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 00:24:06,878 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-23 00:24:07,117 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 00:24:07,120 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-23 00:24:09,300 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 00:24:09,303 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-23 00:24:09] llamafactory.model.model_utils.kv_cache:144 >> KV cache is enabled for faster generation.
+[INFO|modeling_utils.py:769] 2026-08-23 00:24:09,539 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model.safetensors.index.json
+[INFO|modeling_utils.py:839] 2026-08-23 00:24:09,541 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-23 00:24:09,543 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": true
+}
+
+
Loading weights: 0%| | 0/398 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-23 00:24:11,586 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.7,
+ "top_k": 20,
+ "top_p": 0.8
+}
+
+[INFO|2026-08-23 00:24:11] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-23 00:24:12] llamafactory.model.adapter:144 >> Merged 1 adapter(s).
+[INFO|2026-08-23 00:24:12] llamafactory.model.adapter:144 >> Loaded adapter(s): results/qwen3/teacher_lora
+[INFO|2026-08-23 00:24:12] llamafactory.model.loader:144 >> all params: 4,022,468,096
+[INFO|2026-08-23 00:24:12] llamafactory.train.trainer_utils:144 >> Created reference model from Qwen/Qwen3-4B-Instruct-2507
+
Loading weights: 0%| | 0/398 [00:00> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}.
+Gradient accumulation steps mismatch: GradientAccumulationPlugin has 1, DeepSpeed config has 8. Using DeepSpeed's value.
+[INFO|trainer.py:1469] 2026-08-23 00:24:17,627 >> ***** Running training *****
+[INFO|trainer.py:1470] 2026-08-23 00:24:17,627 >> Num examples = 10,027
+[INFO|trainer.py:1471] 2026-08-23 00:24:17,627 >> Num Epochs = 5
+[INFO|trainer.py:1472] 2026-08-23 00:24:17,627 >> Num update steps per epoch = 314
+[INFO|trainer.py:1473] 2026-08-23 00:24:17,627 >> Instantaneous batch size per device = 2
+[INFO|trainer.py:1476] 2026-08-23 00:24:17,627 >> Total train batch size (w. parallel, distributed & accumulation) = 32
+[INFO|trainer.py:1477] 2026-08-23 00:24:17,627 >> Gradient Accumulation steps = 8
+[INFO|trainer.py:1478] 2026-08-23 00:24:17,627 >> Total optimization steps = 1,570
+[INFO|trainer.py:1479] 2026-08-23 00:24:17,631 >> Number of trainable parameters = 20,185,088
+
0%| | 0/1570 [00:00>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 00:39:50,795 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 00:39:50,795 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/amid/checkpoint-314
+[INFO|configuration_utils.py:780] 2026-08-23 00:49:44,193 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 00:49:44,195 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 00:49:44,229 >> chat template saved in results/qwen3/amid/checkpoint-314/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 00:49:44,229 >> tokenizer config file saved in results/qwen3/amid/checkpoint-314/tokenizer_config.json
+
20%|██ | 315/1570 [25:30<63:02:56, 180.86s/it]
20%|██ | 316/1570 [25:33<44:24:26, 127.49s/it]
20%|██ | 317/1570 [25:36<31:22:13, 90.13s/it]
20%|██ | 318/1570 [25:39<22:15:19, 63.99s/it]
20%|██ | 319/1570 [25:42<15:52:32, 45.69s/it]
20%|██ | 320/1570 [25:45<11:24:18, 32.85s/it]
{'loss': '0.009849', 'grad_norm': '0.273', 'learning_rate': '9.016e-05', 'epoch': '1.019'}
+
20%|██ | 320/1570 [25:45<11:24:18, 32.85s/it]
20%|██ | 321/1570 [25:48<8:16:55, 23.87s/it]
21%|██ | 322/1570 [25:50<6:05:13, 17.56s/it]
21%|██ | 323/1570 [25:53<4:33:43, 13.17s/it]
21%|██ | 324/1570 [25:56<3:29:49, 10.10s/it]
21%|██ | 325/1570 [25:59<2:45:15, 7.96s/it]
21%|██ | 326/1570 [26:02<2:13:33, 6.44s/it]
21%|██ | 327/1570 [26:05<1:51:43, 5.39s/it]
21%|██ | 328/1570 [26:08<1:36:26, 4.66s/it]
21%|██ | 329/1570 [26:11<1:25:46, 4.15s/it]
21%|██ | 330/1570 [26:14<1:18:32, 3.80s/it]
21%|██ | 331/1570 [26:17<1:13:08, 3.54s/it]
21%|██ | 332/1570 [26:20<1:09:06, 3.35s/it]
21%|██ | 333/1570 [26:23<1:06:47, 3.24s/it]
21%|██▏ | 334/1570 [26:26<1:05:16, 3.17s/it]
21%|██▏ | 335/1570 [26:29<1:03:58, 3.11s/it]
21%|██▏ | 336/1570 [26:32<1:03:02, 3.06s/it]
21%|██▏ | 337/1570 [26:35<1:01:59, 3.02s/it]
22%|██▏ | 338/1570 [26:38<1:01:18, 2.99s/it]
22%|██▏ | 339/1570 [26:40<1:00:50, 2.97s/it]
22%|██▏ | 340/1570 [26:43<1:00:57, 2.97s/it]
{'loss': '0.005673', 'grad_norm': '0.06083', 'learning_rate': '8.894e-05', 'epoch': '1.083'}
+
22%|██▏ | 340/1570 [26:43<1:00:57, 2.97s/it]
22%|██▏ | 341/1570 [26:46<1:00:51, 2.97s/it]
22%|██▏ | 342/1570 [26:49<1:00:49, 2.97s/it]
22%|██▏ | 343/1570 [26:52<1:00:27, 2.96s/it]
22%|██▏ | 344/1570 [26:55<1:00:24, 2.96s/it]
22%|██▏ | 345/1570 [26:58<1:00:23, 2.96s/it]
22%|██▏ | 346/1570 [27:01<1:00:16, 2.95s/it]
22%|██▏ | 347/1570 [27:04<1:00:28, 2.97s/it]
22%|██▏ | 348/1570 [27:07<1:00:30, 2.97s/it]
22%|██▏ | 349/1570 [27:10<1:00:18, 2.96s/it]
22%|██▏ | 350/1570 [27:13<1:00:06, 2.96s/it]
22%|██▏ | 351/1570 [27:16<1:00:07, 2.96s/it]
22%|██▏ | 352/1570 [27:19<1:00:06, 2.96s/it]
22%|██▏ | 353/1570 [27:22<1:00:12, 2.97s/it]
23%|██▎ | 354/1570 [27:25<1:00:21, 2.98s/it]
23%|██▎ | 355/1570 [27:28<1:00:20, 2.98s/it]
23%|██▎ | 356/1570 [27:31<1:00:14, 2.98s/it]
23%|██▎ | 357/1570 [27:34<1:00:02, 2.97s/it]
23%|██▎ | 358/1570 [27:37<59:57, 2.97s/it]
23%|██▎ | 359/1570 [27:40<59:55, 2.97s/it]
23%|██▎ | 360/1570 [27:43<59:26, 2.95s/it]
{'loss': '0.005298', 'grad_norm': '0.04863', 'learning_rate': '8.766e-05', 'epoch': '1.147'}
+
23%|██▎ | 360/1570 [27:43<59:26, 2.95s/it]
23%|██▎ | 361/1570 [27:46<59:38, 2.96s/it]
23%|██▎ | 362/1570 [27:49<59:43, 2.97s/it]
23%|██▎ | 363/1570 [27:52<59:17, 2.95s/it]
23%|██▎ | 364/1570 [27:55<59:20, 2.95s/it]
23%|██▎ | 365/1570 [27:57<59:04, 2.94s/it]
23%|██▎ | 366/1570 [28:00<59:10, 2.95s/it]
23%|██▎ | 367/1570 [28:03<59:01, 2.94s/it]
23%|██▎ | 368/1570 [28:06<58:51, 2.94s/it]
24%|██▎ | 369/1570 [28:09<58:58, 2.95s/it]
24%|██▎ | 370/1570 [28:12<58:59, 2.95s/it]
24%|██▎ | 371/1570 [28:15<58:45, 2.94s/it]
24%|██▎ | 372/1570 [28:18<58:40, 2.94s/it]
24%|██▍ | 373/1570 [28:21<58:52, 2.95s/it]
24%|██▍ | 374/1570 [28:24<58:40, 2.94s/it]
24%|██▍ | 375/1570 [28:27<59:09, 2.97s/it]
24%|██▍ | 376/1570 [28:30<59:11, 2.97s/it]
24%|██▍ | 377/1570 [28:33<58:55, 2.96s/it]
24%|██▍ | 378/1570 [28:36<58:54, 2.97s/it]
24%|██▍ | 379/1570 [28:39<59:06, 2.98s/it]
24%|██▍ | 380/1570 [28:42<58:36, 2.95s/it]
{'loss': '0.002137', 'grad_norm': '0.05065', 'learning_rate': '8.631e-05', 'epoch': '1.211'}
+
24%|██▍ | 380/1570 [28:42<58:36, 2.95s/it]
24%|██▍ | 381/1570 [28:45<58:40, 2.96s/it]
24%|██▍ | 382/1570 [28:48<58:27, 2.95s/it]
24%|██▍ | 383/1570 [28:51<58:20, 2.95s/it]
24%|██▍ | 384/1570 [28:54<58:26, 2.96s/it]
25%|██▍ | 385/1570 [28:57<58:16, 2.95s/it]
25%|██▍ | 386/1570 [28:59<58:00, 2.94s/it]
25%|██▍ | 387/1570 [29:02<58:18, 2.96s/it]
25%|██▍ | 388/1570 [29:05<58:06, 2.95s/it]
25%|██▍ | 389/1570 [29:08<57:57, 2.94s/it]
25%|██▍ | 390/1570 [29:11<58:02, 2.95s/it]
25%|██▍ | 391/1570 [29:14<58:05, 2.96s/it]
25%|██▍ | 392/1570 [29:17<57:59, 2.95s/it]
25%|██▌ | 393/1570 [29:20<57:43, 2.94s/it]
25%|██▌ | 394/1570 [29:23<58:09, 2.97s/it]
25%|██▌ | 395/1570 [29:26<58:17, 2.98s/it]
25%|██▌ | 396/1570 [29:29<58:06, 2.97s/it]
25%|██▌ | 397/1570 [29:32<58:12, 2.98s/it]
25%|██▌ | 398/1570 [29:35<58:04, 2.97s/it]
25%|██▌ | 399/1570 [29:38<57:44, 2.96s/it]
25%|██▌ | 400/1570 [29:41<57:41, 2.96s/it]
{'loss': '0.004293', 'grad_norm': '0.05359', 'learning_rate': '8.491e-05', 'epoch': '1.274'}
+
25%|██▌ | 400/1570 [29:41<57:41, 2.96s/it]
26%|██▌ | 401/1570 [29:44<58:01, 2.98s/it]
26%|██▌ | 402/1570 [29:47<57:36, 2.96s/it]
26%|██▌ | 403/1570 [29:50<57:44, 2.97s/it]
26%|██▌ | 404/1570 [29:53<57:31, 2.96s/it]
26%|██▌ | 405/1570 [29:56<57:34, 2.97s/it]
26%|██▌ | 406/1570 [29:59<57:49, 2.98s/it]
26%|██▌ | 407/1570 [30:02<57:59, 2.99s/it]
26%|██▌ | 408/1570 [30:05<57:30, 2.97s/it]
26%|██▌ | 409/1570 [30:08<57:46, 2.99s/it]
26%|██▌ | 410/1570 [30:11<57:32, 2.98s/it]
26%|██▌ | 411/1570 [30:14<57:21, 2.97s/it]
26%|██▌ | 412/1570 [30:17<57:10, 2.96s/it]
26%|██▋ | 413/1570 [30:20<57:05, 2.96s/it]
26%|██▋ | 414/1570 [30:23<57:33, 2.99s/it]
26%|██▋ | 415/1570 [30:26<57:11, 2.97s/it]
26%|██▋ | 416/1570 [30:29<57:17, 2.98s/it]
27%|██▋ | 417/1570 [30:31<57:09, 2.97s/it]
27%|██▋ | 418/1570 [30:34<57:05, 2.97s/it]
27%|██▋ | 419/1570 [30:37<56:39, 2.95s/it]
27%|██▋ | 420/1570 [30:40<56:43, 2.96s/it]
{'loss': '0.003451', 'grad_norm': '0.1137', 'learning_rate': '8.345e-05', 'epoch': '1.338'}
+
27%|██▋ | 420/1570 [30:40<56:43, 2.96s/it]
27%|██▋ | 421/1570 [30:43<57:00, 2.98s/it]
27%|██▋ | 422/1570 [30:46<56:26, 2.95s/it]
27%|██▋ | 423/1570 [30:49<56:17, 2.94s/it]
27%|██▋ | 424/1570 [30:52<56:13, 2.94s/it]
27%|██▋ | 425/1570 [30:55<56:19, 2.95s/it]
27%|██▋ | 426/1570 [30:58<56:03, 2.94s/it]
27%|██▋ | 427/1570 [31:01<56:14, 2.95s/it]
27%|██▋ | 428/1570 [31:04<55:54, 2.94s/it]
27%|██▋ | 429/1570 [31:07<55:56, 2.94s/it]
27%|██▋ | 430/1570 [31:10<56:06, 2.95s/it]
27%|██▋ | 431/1570 [31:13<56:03, 2.95s/it]
28%|██▊ | 432/1570 [31:16<56:13, 2.96s/it]
28%|██▊ | 433/1570 [31:19<56:15, 2.97s/it]
28%|██▊ | 434/1570 [31:22<55:51, 2.95s/it]
28%|██▊ | 435/1570 [31:25<55:56, 2.96s/it]
28%|██▊ | 436/1570 [31:28<55:46, 2.95s/it]
28%|██▊ | 437/1570 [31:31<56:01, 2.97s/it]
28%|██▊ | 438/1570 [31:34<56:06, 2.97s/it]
28%|██▊ | 439/1570 [31:37<56:00, 2.97s/it]
28%|██▊ | 440/1570 [31:39<55:58, 2.97s/it]
{'loss': '0.002835', 'grad_norm': '0.01605', 'learning_rate': '8.194e-05', 'epoch': '1.402'}
+
28%|██▊ | 440/1570 [31:39<55:58, 2.97s/it]
28%|██▊ | 441/1570 [31:42<55:44, 2.96s/it]
28%|██▊ | 442/1570 [31:45<55:42, 2.96s/it]
28%|██▊ | 443/1570 [31:48<55:22, 2.95s/it]
28%|██▊ | 444/1570 [31:51<55:15, 2.94s/it]
28%|██▊ | 445/1570 [31:54<55:04, 2.94s/it]
28%|██▊ | 446/1570 [31:57<55:13, 2.95s/it]
28%|██▊ | 447/1570 [32:00<55:16, 2.95s/it]
29%|██▊ | 448/1570 [32:03<55:09, 2.95s/it]
29%|██▊ | 449/1570 [32:06<55:33, 2.97s/it]
29%|██▊ | 450/1570 [32:09<55:36, 2.98s/it]
29%|██▊ | 451/1570 [32:12<55:40, 2.99s/it]
29%|██▉ | 452/1570 [32:15<55:32, 2.98s/it]
29%|██▉ | 453/1570 [32:18<55:18, 2.97s/it]
29%|██▉ | 454/1570 [32:21<55:18, 2.97s/it]
29%|██▉ | 455/1570 [32:24<55:29, 2.99s/it]
29%|██▉ | 456/1570 [32:27<55:35, 2.99s/it]
29%|██▉ | 457/1570 [32:30<55:16, 2.98s/it]
29%|██▉ | 458/1570 [32:33<55:11, 2.98s/it]
29%|██▉ | 459/1570 [32:36<55:10, 2.98s/it]
29%|██▉ | 460/1570 [32:39<54:36, 2.95s/it]
{'loss': '0.005546', 'grad_norm': '0.154', 'learning_rate': '8.037e-05', 'epoch': '1.466'}
+
29%|██▉ | 460/1570 [32:39<54:36, 2.95s/it]
29%|██▉ | 461/1570 [32:42<54:27, 2.95s/it]
29%|██▉ | 462/1570 [32:45<54:30, 2.95s/it]
29%|██▉ | 463/1570 [32:48<54:49, 2.97s/it]
30%|██▉ | 464/1570 [32:51<55:00, 2.98s/it]
30%|██▉ | 465/1570 [32:54<54:49, 2.98s/it]
30%|██▉ | 466/1570 [32:57<54:16, 2.95s/it]
30%|██▉ | 467/1570 [33:00<54:20, 2.96s/it]
30%|██▉ | 468/1570 [33:02<54:13, 2.95s/it]
30%|██▉ | 469/1570 [33:05<54:17, 2.96s/it]
30%|██▉ | 470/1570 [33:08<54:02, 2.95s/it]
30%|███ | 471/1570 [33:11<53:51, 2.94s/it]
30%|███ | 472/1570 [33:14<53:47, 2.94s/it]
30%|███ | 473/1570 [33:17<53:58, 2.95s/it]
30%|███ | 474/1570 [33:20<53:45, 2.94s/it]
30%|███ | 475/1570 [33:23<53:39, 2.94s/it]
30%|███ | 476/1570 [33:26<53:42, 2.95s/it]
30%|███ | 477/1570 [33:29<53:41, 2.95s/it]
30%|███ | 478/1570 [33:32<53:30, 2.94s/it]
31%|███ | 479/1570 [33:35<53:37, 2.95s/it]
31%|███ | 480/1570 [33:38<53:34, 2.95s/it]
{'loss': '0.004453', 'grad_norm': '0.5877', 'learning_rate': '7.876e-05', 'epoch': '1.53'}
+
31%|███ | 480/1570 [33:38<53:34, 2.95s/it]
31%|███ | 481/1570 [33:41<53:40, 2.96s/it]
31%|███ | 482/1570 [33:44<53:27, 2.95s/it]
31%|███ | 483/1570 [33:47<53:28, 2.95s/it]
31%|███ | 484/1570 [33:50<53:28, 2.95s/it]
31%|███ | 485/1570 [33:53<53:25, 2.95s/it]
31%|███ | 486/1570 [33:56<53:31, 2.96s/it]
31%|███ | 487/1570 [33:59<53:16, 2.95s/it]
31%|███ | 488/1570 [34:01<53:16, 2.95s/it]
31%|███ | 489/1570 [34:04<53:02, 2.94s/it]
31%|███ | 490/1570 [34:07<52:57, 2.94s/it]
31%|███▏ | 491/1570 [34:10<53:12, 2.96s/it]
31%|███▏ | 492/1570 [34:13<53:07, 2.96s/it]
31%|███▏ | 493/1570 [34:16<53:12, 2.96s/it]
31%|███▏ | 494/1570 [34:19<52:39, 2.94s/it]
32%|███▏ | 495/1570 [34:22<52:37, 2.94s/it]
32%|███▏ | 496/1570 [34:25<52:26, 2.93s/it]
32%|███▏ | 497/1570 [34:28<52:27, 2.93s/it]
32%|███▏ | 498/1570 [34:31<52:41, 2.95s/it]
32%|███▏ | 499/1570 [34:34<52:37, 2.95s/it]
32%|███▏ | 500/1570 [34:37<52:23, 2.94s/it]
{'loss': '0.005636', 'grad_norm': '0.1025', 'learning_rate': '7.71e-05', 'epoch': '1.594'}
+
32%|███▏ | 500/1570 [34:37<52:23, 2.94s/it]
32%|███▏ | 501/1570 [34:40<52:19, 2.94s/it]
32%|███▏ | 502/1570 [34:43<52:22, 2.94s/it]
32%|███▏ | 503/1570 [34:46<52:23, 2.95s/it]
32%|███▏ | 504/1570 [34:49<52:40, 2.96s/it]
32%|███▏ | 505/1570 [34:52<52:40, 2.97s/it]
32%|███▏ | 506/1570 [34:55<52:19, 2.95s/it]
32%|███▏ | 507/1570 [34:57<52:20, 2.95s/it]
32%|███▏ | 508/1570 [35:00<52:16, 2.95s/it]
32%|███▏ | 509/1570 [35:03<52:07, 2.95s/it]
32%|███▏ | 510/1570 [35:06<52:15, 2.96s/it]
33%|███▎ | 511/1570 [35:09<52:08, 2.95s/it]
33%|███▎ | 512/1570 [35:12<52:09, 2.96s/it]
33%|███▎ | 513/1570 [35:15<52:22, 2.97s/it]
33%|███▎ | 514/1570 [35:18<52:17, 2.97s/it]
33%|███▎ | 515/1570 [35:21<52:11, 2.97s/it]
33%|███▎ | 516/1570 [35:24<52:02, 2.96s/it]
33%|███▎ | 517/1570 [35:27<51:39, 2.94s/it]
33%|███▎ | 518/1570 [35:30<51:29, 2.94s/it]
33%|███▎ | 519/1570 [35:33<51:59, 2.97s/it]
33%|███▎ | 520/1570 [35:36<52:01, 2.97s/it]
{'loss': '0.003258', 'grad_norm': '0.04325', 'learning_rate': '7.54e-05', 'epoch': '1.657'}
+
33%|███▎ | 520/1570 [35:36<52:01, 2.97s/it]
33%|███▎ | 521/1570 [35:39<52:02, 2.98s/it]
33%|███▎ | 522/1570 [35:42<52:10, 2.99s/it]
33%|███▎ | 523/1570 [35:45<51:55, 2.98s/it]
33%|███▎ | 524/1570 [35:48<52:00, 2.98s/it]
33%|███▎ | 525/1570 [35:51<51:58, 2.98s/it]
34%|███▎ | 526/1570 [35:54<52:02, 2.99s/it]
34%|███▎ | 527/1570 [35:57<51:58, 2.99s/it]
34%|███▎ | 528/1570 [36:00<51:28, 2.96s/it]
34%|███▎ | 529/1570 [36:03<51:22, 2.96s/it]
34%|███▍ | 530/1570 [36:06<51:17, 2.96s/it]
34%|███▍ | 531/1570 [36:09<51:26, 2.97s/it]
34%|███▍ | 532/1570 [36:12<51:30, 2.98s/it]
34%|███▍ | 533/1570 [36:15<51:35, 2.99s/it]
34%|███▍ | 534/1570 [36:18<51:35, 2.99s/it]
34%|███▍ | 535/1570 [36:21<51:30, 2.99s/it]
34%|███▍ | 536/1570 [36:24<51:31, 2.99s/it]
34%|███▍ | 537/1570 [36:27<51:10, 2.97s/it]
34%|███▍ | 538/1570 [36:30<51:07, 2.97s/it]
34%|███▍ | 539/1570 [36:32<50:41, 2.95s/it]
34%|███▍ | 540/1570 [36:35<50:46, 2.96s/it]
{'loss': '0.004207', 'grad_norm': '0.2089', 'learning_rate': '7.366e-05', 'epoch': '1.721'}
+
34%|███▍ | 540/1570 [36:35<50:46, 2.96s/it]
34%|███▍ | 541/1570 [36:38<51:04, 2.98s/it]
35%|███▍ | 542/1570 [36:41<51:06, 2.98s/it]
35%|███▍ | 543/1570 [36:44<50:50, 2.97s/it]
35%|███▍ | 544/1570 [36:47<50:37, 2.96s/it]
35%|███▍ | 545/1570 [36:50<50:48, 2.97s/it]
35%|███▍ | 546/1570 [36:53<50:43, 2.97s/it]
35%|███▍ | 547/1570 [36:56<50:33, 2.96s/it]
35%|███▍ | 548/1570 [36:59<50:20, 2.96s/it]
35%|███▍ | 549/1570 [37:02<49:55, 2.93s/it]
35%|███▌ | 550/1570 [37:05<49:57, 2.94s/it]
35%|███▌ | 551/1570 [37:08<50:28, 2.97s/it]
35%|███▌ | 552/1570 [37:11<50:18, 2.96s/it]
35%|███▌ | 553/1570 [37:14<50:06, 2.96s/it]
35%|███▌ | 554/1570 [37:17<50:10, 2.96s/it]
35%|███▌ | 555/1570 [37:20<50:13, 2.97s/it]
35%|███▌ | 556/1570 [37:23<50:08, 2.97s/it]
35%|███▌ | 557/1570 [37:26<50:10, 2.97s/it]
36%|███▌ | 558/1570 [37:29<50:13, 2.98s/it]
36%|███▌ | 559/1570 [37:32<50:05, 2.97s/it]
36%|███▌ | 560/1570 [37:35<49:53, 2.96s/it]
{'loss': '0.001915', 'grad_norm': '0.1404', 'learning_rate': '7.188e-05', 'epoch': '1.785'}
+
36%|███▌ | 560/1570 [37:35<49:53, 2.96s/it]
36%|███▌ | 561/1570 [37:38<49:54, 2.97s/it]
36%|███▌ | 562/1570 [37:41<49:42, 2.96s/it]
36%|███▌ | 563/1570 [37:44<49:26, 2.95s/it]
36%|███▌ | 564/1570 [37:47<49:22, 2.95s/it]
36%|███▌ | 565/1570 [37:50<49:47, 2.97s/it]
36%|███▌ | 566/1570 [37:53<49:32, 2.96s/it]
36%|███▌ | 567/1570 [37:55<49:23, 2.95s/it]
36%|███▌ | 568/1570 [37:58<49:15, 2.95s/it]
36%|███▌ | 569/1570 [38:01<49:18, 2.96s/it]
36%|███▋ | 570/1570 [38:04<49:13, 2.95s/it]
36%|███▋ | 571/1570 [38:07<48:55, 2.94s/it]
36%|███▋ | 572/1570 [38:10<48:45, 2.93s/it]
36%|███▋ | 573/1570 [38:13<49:03, 2.95s/it]
37%|███▋ | 574/1570 [38:16<48:50, 2.94s/it]
37%|███▋ | 575/1570 [38:19<48:54, 2.95s/it]
37%|███▋ | 576/1570 [38:22<48:34, 2.93s/it]
37%|███▋ | 577/1570 [38:25<48:50, 2.95s/it]
37%|███▋ | 578/1570 [38:28<48:40, 2.94s/it]
37%|███▋ | 579/1570 [38:31<48:41, 2.95s/it]
37%|███▋ | 580/1570 [38:34<49:08, 2.98s/it]
{'loss': '0.004428', 'grad_norm': '0.2286', 'learning_rate': '7.006e-05', 'epoch': '1.849'}
+
37%|███▋ | 580/1570 [38:34<49:08, 2.98s/it]
37%|███▋ | 581/1570 [38:37<48:46, 2.96s/it]
37%|███▋ | 582/1570 [38:40<48:42, 2.96s/it]
37%|███▋ | 583/1570 [38:43<48:40, 2.96s/it]
37%|███▋ | 584/1570 [38:46<48:50, 2.97s/it]
37%|███▋ | 585/1570 [38:49<48:53, 2.98s/it]
37%|███▋ | 586/1570 [38:52<48:44, 2.97s/it]
37%|███▋ | 587/1570 [38:55<48:28, 2.96s/it]
37%|███▋ | 588/1570 [38:57<47:56, 2.93s/it]
38%|███▊ | 589/1570 [39:00<48:16, 2.95s/it]
38%|███▊ | 590/1570 [39:03<48:18, 2.96s/it]
38%|███▊ | 591/1570 [39:06<48:24, 2.97s/it]
38%|███▊ | 592/1570 [39:09<48:31, 2.98s/it]
38%|███▊ | 593/1570 [39:12<48:17, 2.97s/it]
38%|███▊ | 594/1570 [39:15<48:01, 2.95s/it]
38%|███▊ | 595/1570 [39:18<47:54, 2.95s/it]
38%|███▊ | 596/1570 [39:21<48:10, 2.97s/it]
38%|███▊ | 597/1570 [39:24<48:10, 2.97s/it]
38%|███▊ | 598/1570 [39:27<48:13, 2.98s/it]
38%|███▊ | 599/1570 [39:30<48:15, 2.98s/it]
38%|███▊ | 600/1570 [39:33<47:49, 2.96s/it]
{'loss': '0.001783', 'grad_norm': '0.03684', 'learning_rate': '6.821e-05', 'epoch': '1.913'}
+
38%|███▊ | 600/1570 [39:33<47:49, 2.96s/it]
38%|███▊ | 601/1570 [39:36<47:47, 2.96s/it]
38%|███▊ | 602/1570 [39:39<47:40, 2.95s/it]
38%|███▊ | 603/1570 [39:42<47:41, 2.96s/it]
38%|███▊ | 604/1570 [39:45<47:45, 2.97s/it]
39%|███▊ | 605/1570 [39:48<47:37, 2.96s/it]
39%|███▊ | 606/1570 [39:51<47:53, 2.98s/it]
39%|███▊ | 607/1570 [39:54<47:52, 2.98s/it]
39%|███▊ | 608/1570 [39:57<47:43, 2.98s/it]
39%|███▉ | 609/1570 [40:00<47:15, 2.95s/it]
39%|███▉ | 610/1570 [40:03<47:15, 2.95s/it]
39%|███▉ | 611/1570 [40:06<47:20, 2.96s/it]
39%|███▉ | 612/1570 [40:09<47:17, 2.96s/it]
39%|███▉ | 613/1570 [40:12<47:03, 2.95s/it]
39%|███▉ | 614/1570 [40:14<46:54, 2.94s/it]
39%|███▉ | 615/1570 [40:17<47:00, 2.95s/it]
39%|███▉ | 616/1570 [40:20<47:09, 2.97s/it]
39%|███▉ | 617/1570 [40:23<47:22, 2.98s/it]
39%|███▉ | 618/1570 [40:26<47:17, 2.98s/it]
39%|███▉ | 619/1570 [40:29<47:30, 3.00s/it]
39%|███▉ | 620/1570 [40:33<47:31, 3.00s/it]
{'loss': '0.003587', 'grad_norm': '0.06729', 'learning_rate': '6.634e-05', 'epoch': '1.976'}
+
39%|███▉ | 620/1570 [40:33<47:31, 3.00s/it]
40%|███▉ | 621/1570 [40:36<47:36, 3.01s/it]
40%|███▉ | 622/1570 [40:38<46:59, 2.97s/it]
40%|███▉ | 623/1570 [40:41<46:53, 2.97s/it]
40%|███▉ | 624/1570 [40:44<46:49, 2.97s/it]
40%|███▉ | 625/1570 [40:47<46:58, 2.98s/it]
40%|███▉ | 626/1570 [40:50<46:36, 2.96s/it]
40%|███▉ | 627/1570 [40:53<46:32, 2.96s/it]
40%|████ | 628/1570 [40:54<38:01, 2.42s/it][INFO|trainer.py:2668] 2026-08-23 01:05:12,553 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 01:05:12,553 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 01:05:12,553 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/amid/checkpoint-628
+[INFO|configuration_utils.py:780] 2026-08-23 01:15:05,335 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 01:15:05,337 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 01:15:05,372 >> chat template saved in results/qwen3/amid/checkpoint-628/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 01:15:05,372 >> tokenizer config file saved in results/qwen3/amid/checkpoint-628/tokenizer_config.json
+
40%|████ | 629/1570 [50:51<47:14:02, 180.70s/it]
40%|████ | 630/1570 [50:54<33:15:38, 127.38s/it]
40%|████ | 631/1570 [50:57<23:29:17, 90.05s/it]
40%|████ | 632/1570 [51:00<16:39:07, 63.91s/it]
40%|████ | 633/1570 [51:03<11:52:30, 45.63s/it]
40%|████ | 634/1570 [51:06<8:32:01, 32.82s/it]
40%|████ | 635/1570 [51:09<6:12:18, 23.89s/it]
41%|████ | 636/1570 [51:12<4:34:07, 17.61s/it]
41%|████ | 637/1570 [51:15<3:25:21, 13.21s/it]
41%|████ | 638/1570 [51:18<2:37:16, 10.12s/it]
41%|████ | 639/1570 [51:21<2:03:35, 7.96s/it]
41%|████ | 640/1570 [51:24<1:40:26, 6.48s/it]
{'loss': '0.002082', 'grad_norm': '0.02913', 'learning_rate': '6.444e-05', 'epoch': '2.038'}
+
41%|████ | 640/1570 [51:24<1:40:26, 6.48s/it]
41%|████ | 641/1570 [51:27<1:24:00, 5.43s/it]
41%|████ | 642/1570 [51:30<1:12:35, 4.69s/it]
41%|████ | 643/1570 [51:33<1:04:39, 4.18s/it]
41%|████ | 644/1570 [51:36<58:57, 3.82s/it]
41%|████ | 645/1570 [51:39<55:11, 3.58s/it]
41%|████ | 646/1570 [51:42<52:23, 3.40s/it]
41%|████ | 647/1570 [51:45<50:29, 3.28s/it]
41%|████▏ | 648/1570 [51:48<48:56, 3.19s/it]
41%|████▏ | 649/1570 [51:51<47:54, 3.12s/it]
41%|████▏ | 650/1570 [51:53<47:04, 3.07s/it]
41%|████▏ | 651/1570 [51:56<46:44, 3.05s/it]
42%|████▏ | 652/1570 [51:59<46:08, 3.02s/it]
42%|████▏ | 653/1570 [52:02<46:16, 3.03s/it]
42%|████▏ | 654/1570 [52:05<45:28, 2.98s/it]
42%|████▏ | 655/1570 [52:08<45:19, 2.97s/it]
42%|████▏ | 656/1570 [52:11<45:07, 2.96s/it]
42%|████▏ | 657/1570 [52:14<45:00, 2.96s/it]
42%|████▏ | 658/1570 [52:17<44:48, 2.95s/it]
42%|████▏ | 659/1570 [52:20<44:41, 2.94s/it]
42%|████▏ | 660/1570 [52:23<44:43, 2.95s/it]
{'loss': '0.001914', 'grad_norm': '0.1737', 'learning_rate': '6.251e-05', 'epoch': '2.102'}
+
42%|████▏ | 660/1570 [52:23<44:43, 2.95s/it]
42%|████▏ | 661/1570 [52:26<44:56, 2.97s/it]
42%|████▏ | 662/1570 [52:29<44:49, 2.96s/it]
42%|████▏ | 663/1570 [52:32<44:47, 2.96s/it]
42%|████▏ | 664/1570 [52:35<44:32, 2.95s/it]
42%|████▏ | 665/1570 [52:38<44:20, 2.94s/it]
42%|████▏ | 666/1570 [52:41<44:24, 2.95s/it]
42%|████▏ | 667/1570 [52:44<44:33, 2.96s/it]
43%|████▎ | 668/1570 [52:47<44:37, 2.97s/it]
43%|████▎ | 669/1570 [52:50<44:20, 2.95s/it]
43%|████▎ | 670/1570 [52:53<44:21, 2.96s/it]
43%|████▎ | 671/1570 [52:56<44:19, 2.96s/it]
43%|████▎ | 672/1570 [52:58<44:15, 2.96s/it]
43%|████▎ | 673/1570 [53:01<43:55, 2.94s/it]
43%|████▎ | 674/1570 [53:04<44:17, 2.97s/it]
43%|████▎ | 675/1570 [53:07<44:22, 2.97s/it]
43%|████▎ | 676/1570 [53:10<44:07, 2.96s/it]
43%|████▎ | 677/1570 [53:13<44:05, 2.96s/it]
43%|████▎ | 678/1570 [53:16<43:57, 2.96s/it]
43%|████▎ | 679/1570 [53:19<44:03, 2.97s/it]
43%|████▎ | 680/1570 [53:22<44:12, 2.98s/it]
{'loss': '0.0009251', 'grad_norm': '0.01645', 'learning_rate': '6.057e-05', 'epoch': '2.166'}
+
43%|████▎ | 680/1570 [53:22<44:12, 2.98s/it]
43%|████▎ | 681/1570 [53:25<44:13, 2.98s/it]
43%|████▎ | 682/1570 [53:28<43:59, 2.97s/it]
44%|████▎ | 683/1570 [53:31<43:46, 2.96s/it]
44%|████▎ | 684/1570 [53:34<43:53, 2.97s/it]
44%|████▎ | 685/1570 [53:37<43:52, 2.97s/it]
44%|████▎ | 686/1570 [53:40<43:59, 2.99s/it]
44%|████▍ | 687/1570 [53:43<43:54, 2.98s/it]
44%|████▍ | 688/1570 [53:46<44:02, 3.00s/it]
44%|████▍ | 689/1570 [53:49<43:41, 2.98s/it]
44%|████▍ | 690/1570 [53:52<43:34, 2.97s/it]
44%|████▍ | 691/1570 [53:55<43:31, 2.97s/it]
44%|████▍ | 692/1570 [53:58<43:32, 2.98s/it]
44%|████▍ | 693/1570 [54:01<43:37, 2.98s/it]
44%|████▍ | 694/1570 [54:04<43:45, 3.00s/it]
44%|████▍ | 695/1570 [54:07<43:31, 2.98s/it]
44%|████▍ | 696/1570 [54:10<43:10, 2.96s/it]
44%|████▍ | 697/1570 [54:13<42:43, 2.94s/it]
44%|████▍ | 698/1570 [54:16<42:33, 2.93s/it]
45%|████▍ | 699/1570 [54:19<42:48, 2.95s/it]
45%|████▍ | 700/1570 [54:22<42:41, 2.94s/it]
{'loss': '0.001781', 'grad_norm': '0.01557', 'learning_rate': '5.86e-05', 'epoch': '2.23'}
+
45%|████▍ | 700/1570 [54:22<42:41, 2.94s/it]
45%|████▍ | 701/1570 [54:25<42:46, 2.95s/it]
45%|████▍ | 702/1570 [54:28<42:57, 2.97s/it]
45%|████▍ | 703/1570 [54:30<42:30, 2.94s/it]
45%|████▍ | 704/1570 [54:33<42:38, 2.95s/it]
45%|████▍ | 705/1570 [54:36<42:40, 2.96s/it]
45%|████▍ | 706/1570 [54:39<42:50, 2.98s/it]
45%|████▌ | 707/1570 [54:42<42:40, 2.97s/it]
45%|████▌ | 708/1570 [54:45<42:32, 2.96s/it]
45%|████▌ | 709/1570 [54:48<42:39, 2.97s/it]
45%|████▌ | 710/1570 [54:51<42:24, 2.96s/it]
45%|████▌ | 711/1570 [54:54<42:29, 2.97s/it]
45%|████▌ | 712/1570 [54:57<42:27, 2.97s/it]
45%|████▌ | 713/1570 [55:00<42:29, 2.97s/it]
45%|████▌ | 714/1570 [55:03<42:26, 2.98s/it]
46%|████▌ | 715/1570 [55:06<42:26, 2.98s/it]
46%|████▌ | 716/1570 [55:09<42:28, 2.98s/it]
46%|████▌ | 717/1570 [55:12<42:29, 2.99s/it]
46%|████▌ | 718/1570 [55:15<42:30, 2.99s/it]
46%|████▌ | 719/1570 [55:18<42:31, 3.00s/it]
46%|████▌ | 720/1570 [55:21<42:14, 2.98s/it]
{'loss': '0.001054', 'grad_norm': '0.01029', 'learning_rate': '5.663e-05', 'epoch': '2.294'}
+
46%|████▌ | 720/1570 [55:21<42:14, 2.98s/it]
46%|████▌ | 721/1570 [55:24<41:48, 2.95s/it]
46%|████▌ | 722/1570 [55:27<41:54, 2.96s/it]
46%|████▌ | 723/1570 [55:30<41:48, 2.96s/it]
46%|████▌ | 724/1570 [55:33<41:49, 2.97s/it]
46%|████▌ | 725/1570 [55:36<42:10, 2.99s/it]
46%|████▌ | 726/1570 [55:39<41:55, 2.98s/it]
46%|████▋ | 727/1570 [55:42<41:52, 2.98s/it]
46%|████▋ | 728/1570 [55:45<42:03, 3.00s/it]
46%|████▋ | 729/1570 [55:48<41:37, 2.97s/it]
46%|████▋ | 730/1570 [55:51<41:26, 2.96s/it]
47%|████▋ | 731/1570 [55:54<41:16, 2.95s/it]
47%|████▋ | 732/1570 [55:57<41:15, 2.95s/it]
47%|████▋ | 733/1570 [56:00<41:17, 2.96s/it]
47%|████▋ | 734/1570 [56:03<41:02, 2.95s/it]
47%|████▋ | 735/1570 [56:05<41:00, 2.95s/it]
47%|████▋ | 736/1570 [56:08<41:04, 2.95s/it]
47%|████▋ | 737/1570 [56:11<40:54, 2.95s/it]
47%|████▋ | 738/1570 [56:14<40:47, 2.94s/it]
47%|████▋ | 739/1570 [56:17<40:29, 2.92s/it]
47%|████▋ | 740/1570 [56:20<40:29, 2.93s/it]
{'loss': '0.0009572', 'grad_norm': '0.01287', 'learning_rate': '5.464e-05', 'epoch': '2.357'}
+
47%|████▋ | 740/1570 [56:20<40:29, 2.93s/it]
47%|████▋ | 741/1570 [56:23<40:42, 2.95s/it]
47%|████▋ | 742/1570 [56:26<41:00, 2.97s/it]
47%|████▋ | 743/1570 [56:29<40:59, 2.97s/it]
47%|████▋ | 744/1570 [56:32<40:55, 2.97s/it]
47%|████▋ | 745/1570 [56:35<40:56, 2.98s/it]
48%|████▊ | 746/1570 [56:38<40:41, 2.96s/it]
48%|████▊ | 747/1570 [56:41<40:40, 2.96s/it]
48%|████▊ | 748/1570 [56:44<40:37, 2.97s/it]
48%|████▊ | 749/1570 [56:47<40:36, 2.97s/it]
48%|████▊ | 750/1570 [56:50<40:01, 2.93s/it]
48%|████▊ | 751/1570 [56:53<40:06, 2.94s/it]
48%|████▊ | 752/1570 [56:56<40:15, 2.95s/it]
48%|████▊ | 753/1570 [56:59<40:08, 2.95s/it]
48%|████▊ | 754/1570 [57:02<40:18, 2.96s/it]
48%|████▊ | 755/1570 [57:05<40:13, 2.96s/it]
48%|████▊ | 756/1570 [57:08<40:01, 2.95s/it]
48%|████▊ | 757/1570 [57:10<39:47, 2.94s/it]
48%|████▊ | 758/1570 [57:13<39:58, 2.95s/it]
48%|████▊ | 759/1570 [57:16<40:00, 2.96s/it]
48%|████▊ | 760/1570 [57:19<40:03, 2.97s/it]
{'loss': '0.0007099', 'grad_norm': '0.009928', 'learning_rate': '5.265e-05', 'epoch': '2.421'}
+
48%|████▊ | 760/1570 [57:19<40:03, 2.97s/it]
48%|████▊ | 761/1570 [57:22<40:10, 2.98s/it]
49%|████▊ | 762/1570 [57:25<40:16, 2.99s/it]
49%|████▊ | 763/1570 [57:28<40:02, 2.98s/it]
49%|████▊ | 764/1570 [57:31<39:58, 2.98s/it]
49%|████▊ | 765/1570 [57:34<39:48, 2.97s/it]
49%|████▉ | 766/1570 [57:37<39:48, 2.97s/it]
49%|████▉ | 767/1570 [57:40<39:45, 2.97s/it]
49%|████▉ | 768/1570 [57:43<39:46, 2.98s/it]
49%|████▉ | 769/1570 [57:46<39:32, 2.96s/it]
49%|████▉ | 770/1570 [57:49<39:42, 2.98s/it]
49%|████▉ | 771/1570 [57:52<39:41, 2.98s/it]
49%|████▉ | 772/1570 [57:55<39:34, 2.98s/it]
49%|████▉ | 773/1570 [57:58<39:27, 2.97s/it]
49%|████▉ | 774/1570 [58:01<39:16, 2.96s/it]
49%|████▉ | 775/1570 [58:04<39:20, 2.97s/it]
49%|████▉ | 776/1570 [58:07<39:12, 2.96s/it]
49%|████▉ | 777/1570 [58:10<39:09, 2.96s/it]
50%|████▉ | 778/1570 [58:13<39:09, 2.97s/it]
50%|████▉ | 779/1570 [58:16<39:00, 2.96s/it]
50%|████▉ | 780/1570 [58:19<38:47, 2.95s/it]
{'loss': '0.0007611', 'grad_norm': '0.0143', 'learning_rate': '5.065e-05', 'epoch': '2.485'}
+
50%|████▉ | 780/1570 [58:19<38:47, 2.95s/it]
50%|████▉ | 781/1570 [58:22<38:45, 2.95s/it]
50%|████▉ | 782/1570 [58:25<38:58, 2.97s/it]
50%|████▉ | 783/1570 [58:28<39:00, 2.97s/it]
50%|████▉ | 784/1570 [58:31<38:59, 2.98s/it]
50%|█████ | 785/1570 [58:34<38:42, 2.96s/it]
50%|█████ | 786/1570 [58:37<38:35, 2.95s/it]
50%|█████ | 787/1570 [58:40<38:33, 2.95s/it]
50%|█████ | 788/1570 [58:42<38:22, 2.94s/it]
50%|█████ | 789/1570 [58:45<38:22, 2.95s/it]
50%|█████ | 790/1570 [58:48<38:29, 2.96s/it]
50%|█████ | 791/1570 [58:51<38:25, 2.96s/it]
50%|█████ | 792/1570 [58:54<38:27, 2.97s/it]
51%|█████ | 793/1570 [58:57<38:22, 2.96s/it]
51%|█████ | 794/1570 [59:00<38:08, 2.95s/it]
51%|█████ | 795/1570 [59:03<38:03, 2.95s/it]
51%|█████ | 796/1570 [59:06<38:13, 2.96s/it]
51%|█████ | 797/1570 [59:09<38:16, 2.97s/it]
51%|█████ | 798/1570 [59:12<38:08, 2.96s/it]
51%|█████ | 799/1570 [59:15<37:53, 2.95s/it]
51%|█████ | 800/1570 [59:18<38:04, 2.97s/it]
{'loss': '0.002759', 'grad_norm': '0.03284', 'learning_rate': '4.865e-05', 'epoch': '2.549'}
+
51%|█████ | 800/1570 [59:18<38:04, 2.97s/it]
51%|█████ | 801/1570 [59:21<38:10, 2.98s/it]
51%|█████ | 802/1570 [59:24<38:01, 2.97s/it]
51%|█████ | 803/1570 [59:27<38:06, 2.98s/it]
51%|█████ | 804/1570 [59:30<37:59, 2.98s/it]
51%|█████▏ | 805/1570 [59:33<37:43, 2.96s/it]
51%|█████▏ | 806/1570 [59:36<37:44, 2.96s/it]
51%|█████▏ | 807/1570 [59:39<37:30, 2.95s/it]
51%|█████▏ | 808/1570 [59:42<37:18, 2.94s/it]
52%|█████▏ | 809/1570 [59:45<37:23, 2.95s/it]
52%|█████▏ | 810/1570 [59:48<37:20, 2.95s/it]
52%|█████▏ | 811/1570 [59:51<37:17, 2.95s/it]
52%|█████▏ | 812/1570 [59:53<37:21, 2.96s/it]
52%|█████▏ | 813/1570 [59:56<37:23, 2.96s/it]
52%|█████▏ | 814/1570 [59:59<37:20, 2.96s/it]
52%|█████▏ | 815/1570 [1:00:02<37:17, 2.96s/it]
52%|█████▏ | 816/1570 [1:00:05<37:35, 2.99s/it]
52%|█████▏ | 817/1570 [1:00:08<37:18, 2.97s/it]
52%|█████▏ | 818/1570 [1:00:11<37:08, 2.96s/it]
52%|█████▏ | 819/1570 [1:00:14<37:10, 2.97s/it]
52%|█████▏ | 820/1570 [1:00:17<37:13, 2.98s/it]
{'loss': '0.001119', 'grad_norm': '0.1909', 'learning_rate': '4.665e-05', 'epoch': '2.613'}
+
52%|█████▏ | 820/1570 [1:00:17<37:13, 2.98s/it]
52%|█████▏ | 821/1570 [1:00:20<37:11, 2.98s/it]
52%|█████▏ | 822/1570 [1:00:23<36:52, 2.96s/it]
52%|█████▏ | 823/1570 [1:00:26<36:41, 2.95s/it]
52%|█████▏ | 824/1570 [1:00:29<36:37, 2.95s/it]
53%|█████▎ | 825/1570 [1:00:32<36:22, 2.93s/it]
53%|█████▎ | 826/1570 [1:00:35<36:28, 2.94s/it]
53%|█████▎ | 827/1570 [1:00:38<36:32, 2.95s/it]
53%|█████▎ | 828/1570 [1:00:41<36:30, 2.95s/it]
53%|█████▎ | 829/1570 [1:00:44<36:21, 2.94s/it]
53%|█████▎ | 830/1570 [1:00:47<36:29, 2.96s/it]
53%|█████▎ | 831/1570 [1:00:50<36:23, 2.95s/it]
53%|█████▎ | 832/1570 [1:00:53<36:08, 2.94s/it]
53%|█████▎ | 833/1570 [1:00:56<36:21, 2.96s/it]
53%|█████▎ | 834/1570 [1:00:59<36:12, 2.95s/it]
53%|█████▎ | 835/1570 [1:01:02<36:16, 2.96s/it]
53%|█████▎ | 836/1570 [1:01:04<36:08, 2.95s/it]
53%|█████▎ | 837/1570 [1:01:07<36:04, 2.95s/it]
53%|█████▎ | 838/1570 [1:01:10<35:56, 2.95s/it]
53%|█████▎ | 839/1570 [1:01:13<35:55, 2.95s/it]
54%|█████▎ | 840/1570 [1:01:16<35:39, 2.93s/it]
{'loss': '0.001494', 'grad_norm': '0.02937', 'learning_rate': '4.466e-05', 'epoch': '2.677'}
+
54%|█████▎ | 840/1570 [1:01:16<35:39, 2.93s/it]
54%|█████▎ | 841/1570 [1:01:19<35:33, 2.93s/it]
54%|█████▎ | 842/1570 [1:01:22<35:31, 2.93s/it]
54%|█████▎ | 843/1570 [1:01:25<35:30, 2.93s/it]
54%|█████▍ | 844/1570 [1:01:28<35:32, 2.94s/it]
54%|█████▍ | 845/1570 [1:01:31<35:34, 2.94s/it]
54%|█████▍ | 846/1570 [1:01:34<35:51, 2.97s/it]
54%|█████▍ | 847/1570 [1:01:37<35:37, 2.96s/it]
54%|█████▍ | 848/1570 [1:01:40<35:31, 2.95s/it]
54%|█████▍ | 849/1570 [1:01:43<35:26, 2.95s/it]
54%|█████▍ | 850/1570 [1:01:46<35:34, 2.96s/it]
54%|█████▍ | 851/1570 [1:01:49<35:26, 2.96s/it]
54%|█████▍ | 852/1570 [1:01:52<35:27, 2.96s/it]
54%|█████▍ | 853/1570 [1:01:55<35:27, 2.97s/it]
54%|█████▍ | 854/1570 [1:01:58<35:24, 2.97s/it]
54%|█████▍ | 855/1570 [1:02:01<35:24, 2.97s/it]
55%|█████▍ | 856/1570 [1:02:04<35:12, 2.96s/it]
55%|█████▍ | 857/1570 [1:02:06<35:13, 2.96s/it]
55%|█████▍ | 858/1570 [1:02:09<35:13, 2.97s/it]
55%|█████▍ | 859/1570 [1:02:12<35:04, 2.96s/it]
55%|█████▍ | 860/1570 [1:02:15<35:17, 2.98s/it]
{'loss': '0.002777', 'grad_norm': '0.03121', 'learning_rate': '4.268e-05', 'epoch': '2.74'}
+
55%|█████▍ | 860/1570 [1:02:15<35:17, 2.98s/it]
55%|█████▍ | 861/1570 [1:02:18<35:01, 2.96s/it]
55%|█████▍ | 862/1570 [1:02:21<35:01, 2.97s/it]
55%|█████▍ | 863/1570 [1:02:24<34:55, 2.96s/it]
55%|█████▌ | 864/1570 [1:02:27<34:48, 2.96s/it]
55%|█████▌ | 865/1570 [1:02:30<34:50, 2.97s/it]
55%|█████▌ | 866/1570 [1:02:33<34:35, 2.95s/it]
55%|█████▌ | 867/1570 [1:02:36<34:27, 2.94s/it]
55%|█████▌ | 868/1570 [1:02:39<34:17, 2.93s/it]
55%|█████▌ | 869/1570 [1:02:42<34:27, 2.95s/it]
55%|█████▌ | 870/1570 [1:02:45<34:14, 2.93s/it]
55%|█████▌ | 871/1570 [1:02:48<34:24, 2.95s/it]
56%|█████▌ | 872/1570 [1:02:51<34:21, 2.95s/it]
56%|█████▌ | 873/1570 [1:02:54<34:21, 2.96s/it]
56%|█████▌ | 874/1570 [1:02:57<34:23, 2.96s/it]
56%|█████▌ | 875/1570 [1:03:00<34:25, 2.97s/it]
56%|█████▌ | 876/1570 [1:03:03<34:16, 2.96s/it]
56%|█████▌ | 877/1570 [1:03:06<34:10, 2.96s/it]
56%|█████▌ | 878/1570 [1:03:09<34:10, 2.96s/it]
56%|█████▌ | 879/1570 [1:03:12<34:22, 2.99s/it]
56%|█████▌ | 880/1570 [1:03:15<34:19, 2.99s/it]
{'loss': '0.0019', 'grad_norm': '0.01913', 'learning_rate': '4.071e-05', 'epoch': '2.804'}
+
56%|█████▌ | 880/1570 [1:03:15<34:19, 2.99s/it]
56%|█████▌ | 881/1570 [1:03:18<34:06, 2.97s/it]
56%|█████▌ | 882/1570 [1:03:20<33:50, 2.95s/it]
56%|█████▌ | 883/1570 [1:03:23<33:51, 2.96s/it]
56%|█████▋ | 884/1570 [1:03:26<33:50, 2.96s/it]
56%|█████▋ | 885/1570 [1:03:29<33:33, 2.94s/it]
56%|█████▋ | 886/1570 [1:03:32<33:32, 2.94s/it]
56%|█████▋ | 887/1570 [1:03:35<33:32, 2.95s/it]
57%|█████▋ | 888/1570 [1:03:38<33:42, 2.97s/it]
57%|█████▋ | 889/1570 [1:03:41<33:40, 2.97s/it]
57%|█████▋ | 890/1570 [1:03:44<33:31, 2.96s/it]
57%|█████▋ | 891/1570 [1:03:47<33:32, 2.96s/it]
57%|█████▋ | 892/1570 [1:03:50<33:35, 2.97s/it]
57%|█████▋ | 893/1570 [1:03:53<33:30, 2.97s/it]
57%|█████▋ | 894/1570 [1:03:56<33:42, 2.99s/it]
57%|█████▋ | 895/1570 [1:03:59<33:29, 2.98s/it]
57%|█████▋ | 896/1570 [1:04:02<33:20, 2.97s/it]
57%|█████▋ | 897/1570 [1:04:05<33:27, 2.98s/it]
57%|█████▋ | 898/1570 [1:04:08<33:36, 3.00s/it]
57%|█████▋ | 899/1570 [1:04:11<33:14, 2.97s/it]
57%|█████▋ | 900/1570 [1:04:14<33:15, 2.98s/it]
{'loss': '0.001008', 'grad_norm': '0.2538', 'learning_rate': '3.875e-05', 'epoch': '2.868'}
+
57%|█████▋ | 900/1570 [1:04:14<33:15, 2.98s/it]
57%|█████▋ | 901/1570 [1:04:17<33:10, 2.98s/it]
57%|█████▋ | 902/1570 [1:04:20<33:14, 2.99s/it]
58%|█████▊ | 903/1570 [1:04:23<32:52, 2.96s/it]
58%|█████▊ | 904/1570 [1:04:26<32:49, 2.96s/it]
58%|█████▊ | 905/1570 [1:04:29<32:50, 2.96s/it]
58%|█████▊ | 906/1570 [1:04:32<32:36, 2.95s/it]
58%|█████▊ | 907/1570 [1:04:35<32:35, 2.95s/it]
58%|█████▊ | 908/1570 [1:04:38<32:37, 2.96s/it]
58%|█████▊ | 909/1570 [1:04:41<32:36, 2.96s/it]
58%|█████▊ | 910/1570 [1:04:44<32:34, 2.96s/it]
58%|█████▊ | 911/1570 [1:04:46<32:30, 2.96s/it]
58%|█████▊ | 912/1570 [1:04:49<32:33, 2.97s/it]
58%|█████▊ | 913/1570 [1:04:52<32:18, 2.95s/it]
58%|█████▊ | 914/1570 [1:04:55<32:22, 2.96s/it]
58%|█████▊ | 915/1570 [1:04:58<32:15, 2.95s/it]
58%|█████▊ | 916/1570 [1:05:01<32:12, 2.95s/it]
58%|█████▊ | 917/1570 [1:05:04<31:55, 2.93s/it]
58%|█████▊ | 918/1570 [1:05:07<31:53, 2.93s/it]
59%|█████▊ | 919/1570 [1:05:10<31:48, 2.93s/it]
59%|█████▊ | 920/1570 [1:05:13<32:00, 2.95s/it]
{'loss': '0.001208', 'grad_norm': '0.001713', 'learning_rate': '3.682e-05', 'epoch': '2.932'}
+
59%|█████▊ | 920/1570 [1:05:13<32:00, 2.95s/it]
59%|█████▊ | 921/1570 [1:05:16<31:57, 2.95s/it]
59%|█████▊ | 922/1570 [1:05:19<31:58, 2.96s/it]
59%|█████▉ | 923/1570 [1:05:22<31:47, 2.95s/it]
59%|█████▉ | 924/1570 [1:05:25<31:46, 2.95s/it]
59%|█████▉ | 925/1570 [1:05:28<31:48, 2.96s/it]
59%|█████▉ | 926/1570 [1:05:31<31:38, 2.95s/it]
59%|█████▉ | 927/1570 [1:05:34<31:38, 2.95s/it]
59%|█████▉ | 928/1570 [1:05:37<31:32, 2.95s/it]
59%|█████▉ | 929/1570 [1:05:40<31:44, 2.97s/it]
59%|█████▉ | 930/1570 [1:05:43<31:50, 2.98s/it]
59%|█████▉ | 931/1570 [1:05:46<31:31, 2.96s/it]
59%|█████▉ | 932/1570 [1:05:49<31:39, 2.98s/it]
59%|█████▉ | 933/1570 [1:05:52<31:36, 2.98s/it]
59%|█████▉ | 934/1570 [1:05:55<31:39, 2.99s/it]
60%|█████▉ | 935/1570 [1:05:58<31:37, 2.99s/it]
60%|█████▉ | 936/1570 [1:06:01<31:39, 3.00s/it]
60%|█████▉ | 937/1570 [1:06:04<31:27, 2.98s/it]
60%|█████▉ | 938/1570 [1:06:06<31:14, 2.97s/it]
60%|█████▉ | 939/1570 [1:06:09<31:19, 2.98s/it]
60%|█████▉ | 940/1570 [1:06:12<31:18, 2.98s/it]
{'loss': '0.0006427', 'grad_norm': '0.03926', 'learning_rate': '3.49e-05', 'epoch': '2.996'}
+
60%|█████▉ | 940/1570 [1:06:12<31:18, 2.98s/it]
60%|█████▉ | 941/1570 [1:06:15<31:06, 2.97s/it]
60%|██████ | 942/1570 [1:06:17<25:30, 2.44s/it][INFO|trainer.py:2668] 2026-08-23 01:30:34,713 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 01:30:34,713 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 01:30:34,713 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/amid/checkpoint-942
+[INFO|configuration_utils.py:780] 2026-08-23 01:40:26,163 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 01:40:26,166 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 01:40:26,202 >> chat template saved in results/qwen3/amid/checkpoint-942/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 01:40:26,202 >> tokenizer config file saved in results/qwen3/amid/checkpoint-942/tokenizer_config.json
+
60%|██████ | 943/1570 [1:16:12<31:24:18, 180.32s/it]
60%|██████ | 944/1570 [1:16:15<22:06:16, 127.12s/it]
60%|██████ | 945/1570 [1:16:18<15:36:14, 89.88s/it]
60%|██████ | 946/1570 [1:16:21<11:03:36, 63.81s/it]
60%|██████ | 947/1570 [1:16:24<7:53:01, 45.56s/it]
60%|██████ | 948/1570 [1:16:27<5:39:50, 32.78s/it]
60%|██████ | 949/1570 [1:16:30<4:06:46, 23.84s/it]
61%|██████ | 950/1570 [1:16:33<3:01:45, 17.59s/it]
61%|██████ | 951/1570 [1:16:36<2:16:11, 13.20s/it]
61%|██████ | 952/1570 [1:16:39<1:44:25, 10.14s/it]
61%|██████ | 953/1570 [1:16:42<1:22:15, 8.00s/it]
61%|██████ | 954/1570 [1:16:45<1:06:58, 6.52s/it]
61%|██████ | 955/1570 [1:16:48<55:53, 5.45s/it]
61%|██████ | 956/1570 [1:16:51<48:23, 4.73s/it]
61%|██████ | 957/1570 [1:16:54<43:09, 4.22s/it]
61%|██████ | 958/1570 [1:16:57<39:12, 3.84s/it]
61%|██████ | 959/1570 [1:17:00<36:46, 3.61s/it]
61%|██████ | 960/1570 [1:17:03<34:46, 3.42s/it]
{'loss': '0.001276', 'grad_norm': '0.00369', 'learning_rate': '3.301e-05', 'epoch': '3.057'}
+
61%|██████ | 960/1570 [1:17:03<34:46, 3.42s/it]
61%|██████ | 961/1570 [1:17:06<33:11, 3.27s/it]
61%|██████▏ | 962/1570 [1:17:09<32:05, 3.17s/it]
61%|██████▏ | 963/1570 [1:17:12<31:42, 3.13s/it]
61%|██████▏ | 964/1570 [1:17:15<31:14, 3.09s/it]
61%|██████▏ | 965/1570 [1:17:18<31:02, 3.08s/it]
62%|██████▏ | 966/1570 [1:17:21<30:41, 3.05s/it]
62%|██████▏ | 967/1570 [1:17:24<30:21, 3.02s/it]
62%|██████▏ | 968/1570 [1:17:27<31:25, 3.13s/it]
62%|██████▏ | 969/1570 [1:17:30<30:58, 3.09s/it]
62%|██████▏ | 970/1570 [1:17:33<30:40, 3.07s/it]
62%|██████▏ | 971/1570 [1:17:36<30:11, 3.02s/it]
62%|██████▏ | 972/1570 [1:17:39<29:49, 2.99s/it]
62%|██████▏ | 973/1570 [1:17:42<29:43, 2.99s/it]
62%|██████▏ | 974/1570 [1:17:45<29:38, 2.98s/it]
62%|██████▏ | 975/1570 [1:17:48<29:33, 2.98s/it]
62%|██████▏ | 976/1570 [1:17:51<29:26, 2.97s/it]
62%|██████▏ | 977/1570 [1:17:54<29:24, 2.98s/it]
62%|██████▏ | 978/1570 [1:17:57<29:21, 2.98s/it]
62%|██████▏ | 979/1570 [1:18:00<29:13, 2.97s/it]
62%|██████▏ | 980/1570 [1:18:03<29:27, 3.00s/it]
{'loss': '0.0006768', 'grad_norm': '0.02855', 'learning_rate': '3.114e-05', 'epoch': '3.121'}
+
62%|██████▏ | 980/1570 [1:18:03<29:27, 3.00s/it]
62%|██████▏ | 981/1570 [1:18:06<29:28, 3.00s/it]
63%|██████▎ | 982/1570 [1:18:09<29:24, 3.00s/it]
63%|██████▎ | 983/1570 [1:18:12<29:25, 3.01s/it]
63%|██████▎ | 984/1570 [1:18:15<29:08, 2.98s/it]
63%|██████▎ | 985/1570 [1:18:18<28:58, 2.97s/it]
63%|██████▎ | 986/1570 [1:18:21<28:53, 2.97s/it]
63%|██████▎ | 987/1570 [1:18:24<28:48, 2.97s/it]
63%|██████▎ | 988/1570 [1:18:27<28:51, 2.97s/it]
63%|██████▎ | 989/1570 [1:18:30<28:54, 2.99s/it]
63%|██████▎ | 990/1570 [1:18:33<28:37, 2.96s/it]
63%|██████▎ | 991/1570 [1:18:36<28:41, 2.97s/it]
63%|██████▎ | 992/1570 [1:18:39<28:40, 2.98s/it]
63%|██████▎ | 993/1570 [1:18:42<28:31, 2.97s/it]
63%|██████▎ | 994/1570 [1:18:44<28:23, 2.96s/it]
63%|██████▎ | 995/1570 [1:18:47<28:21, 2.96s/it]
63%|██████▎ | 996/1570 [1:18:50<28:14, 2.95s/it]
64%|██████▎ | 997/1570 [1:18:53<28:06, 2.94s/it]
64%|██████▎ | 998/1570 [1:18:56<28:15, 2.96s/it]
64%|██████▎ | 999/1570 [1:18:59<28:18, 2.98s/it]
64%|██████▎ | 1000/1570 [1:19:02<28:37, 3.01s/it]
{'loss': '0.001208', 'grad_norm': '0.007299', 'learning_rate': '2.931e-05', 'epoch': '3.185'}
+
64%|██████▎ | 1000/1570 [1:19:02<28:37, 3.01s/it]
64%|██████▍ | 1001/1570 [1:19:05<28:29, 3.00s/it]
64%|██████▍ | 1002/1570 [1:19:08<28:07, 2.97s/it]
64%|██████▍ | 1003/1570 [1:19:11<28:13, 2.99s/it]
64%|██████▍ | 1004/1570 [1:19:14<28:00, 2.97s/it]
64%|██████▍ | 1005/1570 [1:19:17<28:04, 2.98s/it]
64%|██████▍ | 1006/1570 [1:19:20<28:06, 2.99s/it]
64%|██████▍ | 1007/1570 [1:19:23<28:00, 2.99s/it]
64%|██████▍ | 1008/1570 [1:19:26<27:48, 2.97s/it]
64%|██████▍ | 1009/1570 [1:19:29<27:43, 2.96s/it]
64%|██████▍ | 1010/1570 [1:19:32<27:32, 2.95s/it]
64%|██████▍ | 1011/1570 [1:19:35<28:10, 3.02s/it]
64%|██████▍ | 1012/1570 [1:19:38<27:59, 3.01s/it]
65%|██████▍ | 1013/1570 [1:19:41<27:49, 3.00s/it]
65%|██████▍ | 1014/1570 [1:19:44<27:42, 2.99s/it]
65%|██████▍ | 1015/1570 [1:19:47<27:37, 2.99s/it]
65%|██████▍ | 1016/1570 [1:19:50<27:33, 2.99s/it]
65%|██████▍ | 1017/1570 [1:19:53<27:36, 3.00s/it]
65%|██████▍ | 1018/1570 [1:19:56<27:18, 2.97s/it]
65%|██████▍ | 1019/1570 [1:19:59<27:20, 2.98s/it]
65%|██████▍ | 1020/1570 [1:20:02<27:13, 2.97s/it]
{'loss': '0.001417', 'grad_norm': '0.03703', 'learning_rate': '2.751e-05', 'epoch': '3.249'}
+
65%|██████▍ | 1020/1570 [1:20:02<27:13, 2.97s/it]
65%|██████▌ | 1021/1570 [1:20:05<27:12, 2.97s/it]
65%|██████▌ | 1022/1570 [1:20:08<27:02, 2.96s/it]
65%|██████▌ | 1023/1570 [1:20:11<26:56, 2.96s/it]
65%|██████▌ | 1024/1570 [1:20:14<26:53, 2.96s/it]
65%|██████▌ | 1025/1570 [1:20:17<26:46, 2.95s/it]
65%|██████▌ | 1026/1570 [1:20:20<26:55, 2.97s/it]
65%|██████▌ | 1027/1570 [1:20:23<27:01, 2.99s/it]
65%|██████▌ | 1028/1570 [1:20:26<26:53, 2.98s/it]
66%|██████▌ | 1029/1570 [1:20:29<26:44, 2.97s/it]
66%|██████▌ | 1030/1570 [1:20:32<26:36, 2.96s/it]
66%|██████▌ | 1031/1570 [1:20:35<26:31, 2.95s/it]
66%|██████▌ | 1032/1570 [1:20:38<27:07, 3.03s/it]
66%|██████▌ | 1033/1570 [1:20:41<27:06, 3.03s/it]
66%|██████▌ | 1034/1570 [1:20:44<26:55, 3.01s/it]
66%|██████▌ | 1035/1570 [1:20:47<26:43, 3.00s/it]
66%|██████▌ | 1036/1570 [1:20:50<26:36, 2.99s/it]
66%|██████▌ | 1037/1570 [1:20:53<26:43, 3.01s/it]
66%|██████▌ | 1038/1570 [1:20:56<26:36, 3.00s/it]
66%|██████▌ | 1039/1570 [1:20:59<26:37, 3.01s/it]
66%|██████▌ | 1040/1570 [1:21:02<26:25, 2.99s/it]
{'loss': '0.001242', 'grad_norm': '0.2127', 'learning_rate': '2.574e-05', 'epoch': '3.313'}
+
66%|██████▌ | 1040/1570 [1:21:02<26:25, 2.99s/it]
66%|██████▋ | 1041/1570 [1:21:05<26:00, 2.95s/it]
66%|██████▋ | 1042/1570 [1:21:08<26:02, 2.96s/it]
66%|██████▋ | 1043/1570 [1:21:11<26:00, 2.96s/it]
66%|██████▋ | 1044/1570 [1:21:13<25:56, 2.96s/it]
67%|██████▋ | 1045/1570 [1:21:16<25:51, 2.96s/it]
67%|██████▋ | 1046/1570 [1:21:20<26:15, 3.01s/it]
67%|██████▋ | 1047/1570 [1:21:22<26:03, 2.99s/it]
67%|██████▋ | 1048/1570 [1:21:25<25:54, 2.98s/it]
67%|██████▋ | 1049/1570 [1:21:29<26:16, 3.03s/it]
67%|██████▋ | 1050/1570 [1:21:32<26:06, 3.01s/it]
67%|██████▋ | 1051/1570 [1:21:35<25:56, 3.00s/it]
67%|██████▋ | 1052/1570 [1:21:37<25:49, 2.99s/it]
67%|██████▋ | 1053/1570 [1:21:40<25:42, 2.98s/it]
67%|██████▋ | 1054/1570 [1:21:43<25:33, 2.97s/it]
67%|██████▋ | 1055/1570 [1:21:46<25:18, 2.95s/it]
67%|██████▋ | 1056/1570 [1:21:49<25:31, 2.98s/it]
67%|██████▋ | 1057/1570 [1:21:52<25:20, 2.96s/it]
67%|██████▋ | 1058/1570 [1:21:55<25:19, 2.97s/it]
67%|██████▋ | 1059/1570 [1:21:58<25:19, 2.97s/it]
68%|██████▊ | 1060/1570 [1:22:01<25:27, 2.99s/it]
{'loss': '0.0009986', 'grad_norm': '0.003384', 'learning_rate': '2.402e-05', 'epoch': '3.377'}
+
68%|██████▊ | 1060/1570 [1:22:01<25:27, 2.99s/it]
68%|██████▊ | 1061/1570 [1:22:05<26:09, 3.08s/it]
68%|██████▊ | 1062/1570 [1:22:08<25:52, 3.06s/it]
68%|██████▊ | 1063/1570 [1:22:10<25:28, 3.01s/it]
68%|██████▊ | 1064/1570 [1:22:14<25:39, 3.04s/it]
68%|██████▊ | 1065/1570 [1:22:17<25:54, 3.08s/it]
68%|██████▊ | 1066/1570 [1:22:20<25:38, 3.05s/it]
68%|██████▊ | 1067/1570 [1:22:23<25:14, 3.01s/it]
68%|██████▊ | 1068/1570 [1:22:26<25:11, 3.01s/it]
68%|██████▊ | 1069/1570 [1:22:29<25:08, 3.01s/it]
68%|██████▊ | 1070/1570 [1:22:32<24:53, 2.99s/it]
68%|██████▊ | 1071/1570 [1:22:35<26:11, 3.15s/it]
68%|██████▊ | 1072/1570 [1:22:38<25:46, 3.10s/it]
68%|██████▊ | 1073/1570 [1:22:41<25:22, 3.06s/it]
68%|██████▊ | 1074/1570 [1:22:44<25:06, 3.04s/it]
68%|██████▊ | 1075/1570 [1:22:47<24:48, 3.01s/it]
69%|██████▊ | 1076/1570 [1:22:50<24:34, 2.99s/it]
69%|██████▊ | 1077/1570 [1:22:53<24:24, 2.97s/it]
69%|██████▊ | 1078/1570 [1:22:56<24:25, 2.98s/it]
69%|██████▊ | 1079/1570 [1:22:59<24:14, 2.96s/it]
69%|██████▉ | 1080/1570 [1:23:02<24:13, 2.97s/it]
{'loss': '0.00114', 'grad_norm': '0.02066', 'learning_rate': '2.233e-05', 'epoch': '3.44'}
+
69%|██████▉ | 1080/1570 [1:23:02<24:13, 2.97s/it]
69%|██████▉ | 1081/1570 [1:23:05<24:13, 2.97s/it]
69%|██████▉ | 1082/1570 [1:23:08<24:11, 2.98s/it]
69%|██████▉ | 1083/1570 [1:23:11<24:07, 2.97s/it]
69%|██████▉ | 1084/1570 [1:23:14<24:38, 3.04s/it]
69%|██████▉ | 1085/1570 [1:23:17<24:23, 3.02s/it]
69%|██████▉ | 1086/1570 [1:23:20<24:09, 3.00s/it]
69%|██████▉ | 1087/1570 [1:23:23<24:04, 2.99s/it]
69%|██████▉ | 1088/1570 [1:23:26<23:55, 2.98s/it]
69%|██████▉ | 1089/1570 [1:23:29<23:51, 2.98s/it]
69%|██████▉ | 1090/1570 [1:23:32<23:38, 2.95s/it]
69%|██████▉ | 1091/1570 [1:23:35<23:36, 2.96s/it]
70%|██████▉ | 1092/1570 [1:23:38<23:43, 2.98s/it]
70%|██████▉ | 1093/1570 [1:23:41<23:41, 2.98s/it]
70%|██████▉ | 1094/1570 [1:23:44<23:35, 2.97s/it]
70%|██████▉ | 1095/1570 [1:23:47<23:30, 2.97s/it]
70%|██████▉ | 1096/1570 [1:23:50<23:41, 3.00s/it]
70%|██████▉ | 1097/1570 [1:23:53<23:34, 2.99s/it]
70%|██████▉ | 1098/1570 [1:23:56<23:50, 3.03s/it]
70%|███████ | 1099/1570 [1:23:59<23:48, 3.03s/it]
70%|███████ | 1100/1570 [1:24:02<23:34, 3.01s/it]
{'loss': '0.001631', 'grad_norm': '0.001546', 'learning_rate': '2.069e-05', 'epoch': '3.504'}
+
70%|███████ | 1100/1570 [1:24:02<23:34, 3.01s/it]
70%|███████ | 1101/1570 [1:24:05<23:55, 3.06s/it]
70%|███████ | 1102/1570 [1:24:08<23:39, 3.03s/it]
70%|███████ | 1103/1570 [1:24:11<23:25, 3.01s/it]
70%|███████ | 1104/1570 [1:24:14<23:11, 2.99s/it]
70%|███████ | 1105/1570 [1:24:17<23:04, 2.98s/it]
70%|███████ | 1106/1570 [1:24:20<23:31, 3.04s/it]
71%|███████ | 1107/1570 [1:24:23<23:34, 3.06s/it]
71%|███████ | 1108/1570 [1:24:26<23:13, 3.02s/it]
71%|███████ | 1109/1570 [1:24:29<23:02, 3.00s/it]
71%|███████ | 1110/1570 [1:24:32<22:51, 2.98s/it]
71%|███████ | 1111/1570 [1:24:35<22:40, 2.96s/it]
71%|███████ | 1112/1570 [1:24:38<22:37, 2.96s/it]
71%|███████ | 1113/1570 [1:24:41<22:36, 2.97s/it]
71%|███████ | 1114/1570 [1:24:44<22:26, 2.95s/it]
71%|███████ | 1115/1570 [1:24:47<22:28, 2.96s/it]
71%|███████ | 1116/1570 [1:24:49<22:22, 2.96s/it]
71%|███████ | 1117/1570 [1:24:52<22:16, 2.95s/it]
71%|███████ | 1118/1570 [1:24:55<22:18, 2.96s/it]
71%|███████▏ | 1119/1570 [1:24:58<22:21, 2.98s/it]
71%|███████▏ | 1120/1570 [1:25:01<22:14, 2.97s/it]
{'loss': '0.001068', 'grad_norm': '0.01176', 'learning_rate': '1.91e-05', 'epoch': '3.568'}
+
71%|███████▏ | 1120/1570 [1:25:01<22:14, 2.97s/it]
71%|███████▏ | 1121/1570 [1:25:04<22:16, 2.98s/it]
71%|███████▏ | 1122/1570 [1:25:07<22:13, 2.98s/it]
72%|███████▏ | 1123/1570 [1:25:10<22:07, 2.97s/it]
72%|███████▏ | 1124/1570 [1:25:13<21:57, 2.95s/it]
72%|███████▏ | 1125/1570 [1:25:16<21:52, 2.95s/it]
72%|███████▏ | 1126/1570 [1:25:19<21:50, 2.95s/it]
72%|███████▏ | 1127/1570 [1:25:22<21:49, 2.96s/it]
72%|███████▏ | 1128/1570 [1:25:25<21:50, 2.96s/it]
72%|███████▏ | 1129/1570 [1:25:28<21:47, 2.96s/it]
72%|███████▏ | 1130/1570 [1:25:31<21:43, 2.96s/it]
72%|███████▏ | 1131/1570 [1:25:34<21:43, 2.97s/it]
72%|███████▏ | 1132/1570 [1:25:37<21:39, 2.97s/it]
72%|███████▏ | 1133/1570 [1:25:40<21:51, 3.00s/it]
72%|███████▏ | 1134/1570 [1:25:43<21:40, 2.98s/it]
72%|███████▏ | 1135/1570 [1:25:46<21:57, 3.03s/it]
72%|███████▏ | 1136/1570 [1:25:49<21:45, 3.01s/it]
72%|███████▏ | 1137/1570 [1:25:52<21:33, 2.99s/it]
72%|███████▏ | 1138/1570 [1:25:55<21:28, 2.98s/it]
73%|███████▎ | 1139/1570 [1:25:58<21:21, 2.97s/it]
73%|███████▎ | 1140/1570 [1:26:01<21:26, 2.99s/it]
{'loss': '0.0006688', 'grad_norm': '0.007704', 'learning_rate': '1.755e-05', 'epoch': '3.632'}
+
73%|███████▎ | 1140/1570 [1:26:01<21:26, 2.99s/it]
73%|███████▎ | 1141/1570 [1:26:04<21:23, 2.99s/it]
73%|███████▎ | 1142/1570 [1:26:07<21:17, 2.99s/it]
73%|███████▎ | 1143/1570 [1:26:10<21:10, 2.97s/it]
73%|███████▎ | 1144/1570 [1:26:13<21:09, 2.98s/it]
73%|███████▎ | 1145/1570 [1:26:16<21:11, 2.99s/it]
73%|███████▎ | 1146/1570 [1:26:19<21:05, 2.98s/it]
73%|███████▎ | 1147/1570 [1:26:22<21:00, 2.98s/it]
73%|███████▎ | 1148/1570 [1:26:25<20:56, 2.98s/it]
73%|███████▎ | 1149/1570 [1:26:28<20:54, 2.98s/it]
73%|███████▎ | 1150/1570 [1:26:31<20:46, 2.97s/it]
73%|███████▎ | 1151/1570 [1:26:34<20:41, 2.96s/it]
73%|███████▎ | 1152/1570 [1:26:37<20:35, 2.96s/it]
73%|███████▎ | 1153/1570 [1:26:40<20:30, 2.95s/it]
74%|███████▎ | 1154/1570 [1:26:42<20:24, 2.94s/it]
74%|███████▎ | 1155/1570 [1:26:45<20:27, 2.96s/it]
74%|███████▎ | 1156/1570 [1:26:48<20:15, 2.94s/it]
74%|███████▎ | 1157/1570 [1:26:51<20:10, 2.93s/it]
74%|███████▍ | 1158/1570 [1:26:54<20:17, 2.95s/it]
74%|███████▍ | 1159/1570 [1:26:57<20:13, 2.95s/it]
74%|███████▍ | 1160/1570 [1:27:00<20:20, 2.98s/it]
{'loss': '0.001656', 'grad_norm': '0.04896', 'learning_rate': '1.606e-05', 'epoch': '3.696'}
+
74%|███████▍ | 1160/1570 [1:27:00<20:20, 2.98s/it]
74%|███████▍ | 1161/1570 [1:27:03<20:12, 2.96s/it]
74%|███████▍ | 1162/1570 [1:27:06<20:12, 2.97s/it]
74%|███████▍ | 1163/1570 [1:27:09<20:09, 2.97s/it]
74%|███████▍ | 1164/1570 [1:27:12<20:02, 2.96s/it]
74%|███████▍ | 1165/1570 [1:27:15<20:02, 2.97s/it]
74%|███████▍ | 1166/1570 [1:27:18<19:55, 2.96s/it]
74%|███████▍ | 1167/1570 [1:27:21<19:42, 2.93s/it]
74%|███████▍ | 1168/1570 [1:27:24<19:37, 2.93s/it]
74%|███████▍ | 1169/1570 [1:27:27<19:43, 2.95s/it]
75%|███████▍ | 1170/1570 [1:27:30<19:56, 2.99s/it]
75%|███████▍ | 1171/1570 [1:27:33<19:46, 2.97s/it]
75%|███████▍ | 1172/1570 [1:27:36<19:41, 2.97s/it]
75%|███████▍ | 1173/1570 [1:27:39<19:48, 2.99s/it]
75%|███████▍ | 1174/1570 [1:27:42<20:04, 3.04s/it]
75%|███████▍ | 1175/1570 [1:27:45<19:50, 3.01s/it]
75%|███████▍ | 1176/1570 [1:27:48<19:50, 3.02s/it]
75%|███████▍ | 1177/1570 [1:27:51<19:51, 3.03s/it]
75%|███████▌ | 1178/1570 [1:27:54<19:49, 3.03s/it]
75%|███████▌ | 1179/1570 [1:27:57<19:44, 3.03s/it]
75%|███████▌ | 1180/1570 [1:28:00<19:32, 3.01s/it]
{'loss': '0.0006196', 'grad_norm': '0.04278', 'learning_rate': '1.462e-05', 'epoch': '3.759'}
+
75%|███████▌ | 1180/1570 [1:28:00<19:32, 3.01s/it]
75%|███████▌ | 1181/1570 [1:28:03<19:43, 3.04s/it]
75%|███████▌ | 1182/1570 [1:28:06<19:27, 3.01s/it]
75%|███████▌ | 1183/1570 [1:28:09<19:19, 3.00s/it]
75%|███████▌ | 1184/1570 [1:28:12<19:16, 3.00s/it]
75%|███████▌ | 1185/1570 [1:28:15<19:09, 2.99s/it]
76%|███████▌ | 1186/1570 [1:28:18<19:10, 3.00s/it]
76%|███████▌ | 1187/1570 [1:28:21<19:05, 2.99s/it]
76%|███████▌ | 1188/1570 [1:28:24<19:03, 2.99s/it]
76%|███████▌ | 1189/1570 [1:28:27<19:00, 2.99s/it]
76%|███████▌ | 1190/1570 [1:28:30<18:57, 2.99s/it]
76%|███████▌ | 1191/1570 [1:28:33<18:52, 2.99s/it]
76%|███████▌ | 1192/1570 [1:28:36<18:48, 2.99s/it]
76%|███████▌ | 1193/1570 [1:28:39<18:49, 3.00s/it]
76%|███████▌ | 1194/1570 [1:28:42<18:46, 2.99s/it]
76%|███████▌ | 1195/1570 [1:28:45<18:41, 2.99s/it]
76%|███████▌ | 1196/1570 [1:28:48<18:35, 2.98s/it]
76%|███████▌ | 1197/1570 [1:28:51<18:32, 2.98s/it]
76%|███████▋ | 1198/1570 [1:28:54<18:35, 3.00s/it]
76%|███████▋ | 1199/1570 [1:28:57<18:58, 3.07s/it]
76%|███████▋ | 1200/1570 [1:29:00<18:56, 3.07s/it]
{'loss': '0.0009099', 'grad_norm': '0.0041', 'learning_rate': '1.324e-05', 'epoch': '3.823'}
+
76%|███████▋ | 1200/1570 [1:29:00<18:56, 3.07s/it]
76%|███████▋ | 1201/1570 [1:29:03<18:45, 3.05s/it]
77%|███████▋ | 1202/1570 [1:29:06<18:38, 3.04s/it]
77%|███████▋ | 1203/1570 [1:29:09<18:33, 3.03s/it]
77%|███████▋ | 1204/1570 [1:29:12<18:46, 3.08s/it]
77%|███████▋ | 1205/1570 [1:29:15<18:26, 3.03s/it]
77%|███████▋ | 1206/1570 [1:29:18<18:24, 3.03s/it]
77%|███████▋ | 1207/1570 [1:29:21<18:15, 3.02s/it]
77%|███████▋ | 1208/1570 [1:29:24<18:05, 3.00s/it]
77%|███████▋ | 1209/1570 [1:29:27<18:03, 3.00s/it]
77%|███████▋ | 1210/1570 [1:29:30<18:02, 3.01s/it]
77%|███████▋ | 1211/1570 [1:29:34<18:19, 3.06s/it]
77%|███████▋ | 1212/1570 [1:29:37<18:06, 3.03s/it]
77%|███████▋ | 1213/1570 [1:29:39<17:52, 3.00s/it]
77%|███████▋ | 1214/1570 [1:29:42<17:47, 3.00s/it]
77%|███████▋ | 1215/1570 [1:29:45<17:41, 2.99s/it]
77%|███████▋ | 1216/1570 [1:29:48<17:38, 2.99s/it]
78%|███████▊ | 1217/1570 [1:29:51<17:31, 2.98s/it]
78%|███████▊ | 1218/1570 [1:29:54<17:34, 2.99s/it]
78%|███████▊ | 1219/1570 [1:29:58<17:45, 3.04s/it]
78%|███████▊ | 1220/1570 [1:30:00<17:29, 3.00s/it]
{'loss': '0.0012', 'grad_norm': '0.06637', 'learning_rate': '1.192e-05', 'epoch': '3.887'}
+
78%|███████▊ | 1220/1570 [1:30:00<17:29, 3.00s/it]
78%|███████▊ | 1221/1570 [1:30:03<17:22, 2.99s/it]
78%|███████▊ | 1222/1570 [1:30:06<17:14, 2.97s/it]
78%|███████▊ | 1223/1570 [1:30:09<17:09, 2.97s/it]
78%|███████▊ | 1224/1570 [1:30:12<17:05, 2.96s/it]
78%|███████▊ | 1225/1570 [1:30:15<17:05, 2.97s/it]
78%|███████▊ | 1226/1570 [1:30:18<17:03, 2.98s/it]
78%|███████▊ | 1227/1570 [1:30:21<17:06, 2.99s/it]
78%|███████▊ | 1228/1570 [1:30:24<17:03, 2.99s/it]
78%|███████▊ | 1229/1570 [1:30:27<16:57, 2.98s/it]
78%|███████▊ | 1230/1570 [1:30:30<17:04, 3.01s/it]
78%|███████▊ | 1231/1570 [1:30:33<16:58, 3.00s/it]
78%|███████▊ | 1232/1570 [1:30:36<16:53, 3.00s/it]
79%|███████▊ | 1233/1570 [1:30:39<16:42, 2.97s/it]
79%|███████▊ | 1234/1570 [1:30:42<16:42, 2.98s/it]
79%|███████▊ | 1235/1570 [1:30:45<16:34, 2.97s/it]
79%|███████▊ | 1236/1570 [1:30:48<16:30, 2.97s/it]
79%|███████▉ | 1237/1570 [1:30:51<16:22, 2.95s/it]
79%|███████▉ | 1238/1570 [1:30:54<16:19, 2.95s/it]
79%|███████▉ | 1239/1570 [1:30:57<16:17, 2.95s/it]
79%|███████▉ | 1240/1570 [1:31:00<16:20, 2.97s/it]
{'loss': '0.0003461', 'grad_norm': '0.005992', 'learning_rate': '1.066e-05', 'epoch': '3.951'}
+
79%|███████▉ | 1240/1570 [1:31:00<16:20, 2.97s/it]
79%|███████▉ | 1241/1570 [1:31:03<16:25, 3.00s/it]
79%|███████▉ | 1242/1570 [1:31:06<16:12, 2.96s/it]
79%|███████▉ | 1243/1570 [1:31:09<16:12, 2.97s/it]
79%|███████▉ | 1244/1570 [1:31:12<16:06, 2.97s/it]
79%|███████▉ | 1245/1570 [1:31:15<16:01, 2.96s/it]
79%|███████▉ | 1246/1570 [1:31:18<16:03, 2.97s/it]
79%|███████▉ | 1247/1570 [1:31:21<16:01, 2.98s/it]
79%|███████▉ | 1248/1570 [1:31:24<15:59, 2.98s/it]
80%|███████▉ | 1249/1570 [1:31:27<15:57, 2.98s/it]
80%|███████▉ | 1250/1570 [1:31:30<15:55, 2.99s/it]
80%|███████▉ | 1251/1570 [1:31:33<16:02, 3.02s/it]
80%|███████▉ | 1252/1570 [1:31:36<15:51, 2.99s/it]
80%|███████▉ | 1253/1570 [1:31:39<15:47, 2.99s/it]
80%|███████▉ | 1254/1570 [1:31:42<15:40, 2.98s/it]
80%|███████▉ | 1255/1570 [1:31:45<15:51, 3.02s/it]
80%|████████ | 1256/1570 [1:31:46<12:54, 2.47s/it][INFO|trainer.py:2668] 2026-08-23 01:56:04,114 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 01:56:04,114 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 01:56:04,114 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/amid/checkpoint-1256
+[INFO|configuration_utils.py:780] 2026-08-23 02:05:52,795 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 02:05:52,797 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 02:05:52,832 >> chat template saved in results/qwen3/amid/checkpoint-1256/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 02:05:52,832 >> tokenizer config file saved in results/qwen3/amid/checkpoint-1256/tokenizer_config.json
+
80%|████████ | 1257/1570 [1:41:39<15:36:28, 179.52s/it]
80%|████████ | 1258/1570 [1:41:42<10:58:01, 126.54s/it]
80%|████████ | 1259/1570 [1:41:44<7:43:43, 89.46s/it]
80%|████████ | 1260/1570 [1:41:47<5:28:06, 63.50s/it]
{'loss': '0.0005225', 'grad_norm': '0.124', 'learning_rate': '9.464e-06', 'epoch': '4.013'}
+
80%|████████ | 1260/1570 [1:41:47<5:28:06, 63.50s/it]
80%|████████ | 1261/1570 [1:41:50<3:53:27, 45.33s/it]
80%|████████ | 1262/1570 [1:41:53<2:47:29, 32.63s/it]
80%|████████ | 1263/1570 [1:41:56<2:01:26, 23.73s/it]
81%|████████ | 1264/1570 [1:41:59<1:29:19, 17.52s/it]
81%|████████ | 1265/1570 [1:42:02<1:06:51, 13.15s/it]
81%|████████ | 1266/1570 [1:42:05<51:04, 10.08s/it]
81%|████████ | 1267/1570 [1:42:08<40:11, 7.96s/it]
81%|████████ | 1268/1570 [1:42:11<32:33, 6.47s/it]
81%|████████ | 1269/1570 [1:42:14<27:11, 5.42s/it]
81%|████████ | 1270/1570 [1:42:17<23:27, 4.69s/it]
81%|████████ | 1271/1570 [1:42:20<20:46, 4.17s/it]
81%|████████ | 1272/1570 [1:42:23<19:01, 3.83s/it]
81%|████████ | 1273/1570 [1:42:26<17:41, 3.57s/it]
81%|████████ | 1274/1570 [1:42:29<16:42, 3.39s/it]
81%|████████ | 1275/1570 [1:42:32<16:11, 3.29s/it]
81%|████████▏ | 1276/1570 [1:42:35<15:42, 3.20s/it]
81%|████████▏ | 1277/1570 [1:42:38<15:20, 3.14s/it]
81%|████████▏ | 1278/1570 [1:42:41<15:02, 3.09s/it]
81%|████████▏ | 1279/1570 [1:42:44<14:49, 3.06s/it]
82%|████████▏ | 1280/1570 [1:42:47<14:42, 3.04s/it]
{'loss': '0.000229', 'grad_norm': '0.03299', 'learning_rate': '8.332e-06', 'epoch': '4.077'}
+
82%|████████▏ | 1280/1570 [1:42:47<14:42, 3.04s/it]
82%|████████▏ | 1281/1570 [1:42:50<14:35, 3.03s/it]
82%|████████▏ | 1282/1570 [1:42:53<14:24, 3.00s/it]
82%|████████▏ | 1283/1570 [1:42:56<14:19, 2.99s/it]
82%|████████▏ | 1284/1570 [1:42:59<14:12, 2.98s/it]
82%|████████▏ | 1285/1570 [1:43:02<14:07, 2.97s/it]
82%|████████▏ | 1286/1570 [1:43:05<14:04, 2.98s/it]
82%|████████▏ | 1287/1570 [1:43:08<14:05, 2.99s/it]
82%|████████▏ | 1288/1570 [1:43:11<14:01, 2.98s/it]
82%|████████▏ | 1289/1570 [1:43:14<13:54, 2.97s/it]
82%|████████▏ | 1290/1570 [1:43:17<13:49, 2.96s/it]
82%|████████▏ | 1291/1570 [1:43:20<13:47, 2.97s/it]
82%|████████▏ | 1292/1570 [1:43:23<13:40, 2.95s/it]
82%|████████▏ | 1293/1570 [1:43:26<13:39, 2.96s/it]
82%|████████▏ | 1294/1570 [1:43:29<13:40, 2.97s/it]
82%|████████▏ | 1295/1570 [1:43:32<13:38, 2.98s/it]
83%|████████▎ | 1296/1570 [1:43:35<13:31, 2.96s/it]
83%|████████▎ | 1297/1570 [1:43:38<13:32, 2.97s/it]
83%|████████▎ | 1298/1570 [1:43:40<13:19, 2.94s/it]
83%|████████▎ | 1299/1570 [1:43:43<13:22, 2.96s/it]
83%|████████▎ | 1300/1570 [1:43:46<13:20, 2.97s/it]
{'loss': '0.0002809', 'grad_norm': '0.002514', 'learning_rate': '7.266e-06', 'epoch': '4.14'}
+
83%|████████▎ | 1300/1570 [1:43:46<13:20, 2.97s/it]
83%|████████▎ | 1301/1570 [1:43:49<13:19, 2.97s/it]
83%|████████▎ | 1302/1570 [1:43:52<13:18, 2.98s/it]
83%|████████▎ | 1303/1570 [1:43:55<13:12, 2.97s/it]
83%|████████▎ | 1304/1570 [1:43:58<13:11, 2.98s/it]
83%|████████▎ | 1305/1570 [1:44:01<13:05, 2.97s/it]
83%|████████▎ | 1306/1570 [1:44:04<13:05, 2.97s/it]
83%|████████▎ | 1307/1570 [1:44:07<13:10, 3.00s/it]
83%|████████▎ | 1308/1570 [1:44:10<13:04, 2.99s/it]
83%|████████▎ | 1309/1570 [1:44:13<12:56, 2.97s/it]
83%|████████▎ | 1310/1570 [1:44:16<12:57, 2.99s/it]
84%|████████▎ | 1311/1570 [1:44:19<12:54, 2.99s/it]
84%|████████▎ | 1312/1570 [1:44:22<12:48, 2.98s/it]
84%|████████▎ | 1313/1570 [1:44:25<12:43, 2.97s/it]
84%|████████▎ | 1314/1570 [1:44:28<12:40, 2.97s/it]
84%|████████▍ | 1315/1570 [1:44:31<12:35, 2.96s/it]
84%|████████▍ | 1316/1570 [1:44:34<12:34, 2.97s/it]
84%|████████▍ | 1317/1570 [1:44:37<12:31, 2.97s/it]
84%|████████▍ | 1318/1570 [1:44:40<12:28, 2.97s/it]
84%|████████▍ | 1319/1570 [1:44:43<12:26, 2.97s/it]
84%|████████▍ | 1320/1570 [1:44:46<12:21, 2.97s/it]
{'loss': '0.0007838', 'grad_norm': '0.01395', 'learning_rate': '6.269e-06', 'epoch': '4.204'}
+
84%|████████▍ | 1320/1570 [1:44:46<12:21, 2.97s/it]
84%|████████▍ | 1321/1570 [1:44:49<12:15, 2.96s/it]
84%|████████▍ | 1322/1570 [1:44:52<12:16, 2.97s/it]
84%|████████▍ | 1323/1570 [1:44:55<12:11, 2.96s/it]
84%|████████▍ | 1324/1570 [1:44:58<12:13, 2.98s/it]
84%|████████▍ | 1325/1570 [1:45:01<12:11, 2.99s/it]
84%|████████▍ | 1326/1570 [1:45:04<12:12, 3.00s/it]
85%|████████▍ | 1327/1570 [1:45:07<12:04, 2.98s/it]
85%|████████▍ | 1328/1570 [1:45:10<12:02, 2.98s/it]
85%|████████▍ | 1329/1570 [1:45:13<11:58, 2.98s/it]
85%|████████▍ | 1330/1570 [1:45:16<11:53, 2.97s/it]
85%|████████▍ | 1331/1570 [1:45:19<11:46, 2.96s/it]
85%|████████▍ | 1332/1570 [1:45:22<11:46, 2.97s/it]
85%|████████▍ | 1333/1570 [1:45:25<11:46, 2.98s/it]
85%|████████▍ | 1334/1570 [1:45:28<11:36, 2.95s/it]
85%|████████▌ | 1335/1570 [1:45:30<11:33, 2.95s/it]
85%|████████▌ | 1336/1570 [1:45:33<11:29, 2.95s/it]
85%|████████▌ | 1337/1570 [1:45:36<11:31, 2.97s/it]
85%|████████▌ | 1338/1570 [1:45:39<11:26, 2.96s/it]
85%|████████▌ | 1339/1570 [1:45:42<11:21, 2.95s/it]
85%|████████▌ | 1340/1570 [1:45:45<11:21, 2.96s/it]
{'loss': '0.001084', 'grad_norm': '0.005952', 'learning_rate': '5.342e-06', 'epoch': '4.268'}
+
85%|████████▌ | 1340/1570 [1:45:45<11:21, 2.96s/it]
85%|████████▌ | 1341/1570 [1:45:48<11:15, 2.95s/it]
85%|████████▌ | 1342/1570 [1:45:51<11:17, 2.97s/it]
86%|████████▌ | 1343/1570 [1:45:54<11:17, 2.98s/it]
86%|████████▌ | 1344/1570 [1:45:57<11:11, 2.97s/it]
86%|████████▌ | 1345/1570 [1:46:00<11:11, 2.99s/it]
86%|████████▌ | 1346/1570 [1:46:03<11:05, 2.97s/it]
86%|████████▌ | 1347/1570 [1:46:06<11:02, 2.97s/it]
86%|████████▌ | 1348/1570 [1:46:09<10:58, 2.97s/it]
86%|████████▌ | 1349/1570 [1:46:12<10:55, 2.97s/it]
86%|████████▌ | 1350/1570 [1:46:15<10:53, 2.97s/it]
86%|████████▌ | 1351/1570 [1:46:18<11:16, 3.09s/it]
86%|████████▌ | 1352/1570 [1:46:21<11:08, 3.06s/it]
86%|████████▌ | 1353/1570 [1:46:24<10:55, 3.02s/it]
86%|████████▌ | 1354/1570 [1:46:27<10:51, 3.02s/it]
86%|████████▋ | 1355/1570 [1:46:30<10:46, 3.00s/it]
86%|████████▋ | 1356/1570 [1:46:33<10:35, 2.97s/it]
86%|████████▋ | 1357/1570 [1:46:36<10:31, 2.96s/it]
86%|████████▋ | 1358/1570 [1:46:39<10:26, 2.96s/it]
87%|████████▋ | 1359/1570 [1:46:42<10:26, 2.97s/it]
87%|████████▋ | 1360/1570 [1:46:45<10:24, 2.97s/it]
{'loss': '0.001041', 'grad_norm': '0.2455', 'learning_rate': '4.486e-06', 'epoch': '4.332'}
+
87%|████████▋ | 1360/1570 [1:46:45<10:24, 2.97s/it]
87%|████████▋ | 1361/1570 [1:46:48<10:22, 2.98s/it]
87%|████████▋ | 1362/1570 [1:46:51<10:13, 2.95s/it]
87%|████████▋ | 1363/1570 [1:46:54<10:08, 2.94s/it]
87%|████████▋ | 1364/1570 [1:46:57<10:07, 2.95s/it]
87%|████████▋ | 1365/1570 [1:47:00<10:05, 2.95s/it]
87%|████████▋ | 1366/1570 [1:47:03<10:04, 2.96s/it]
87%|████████▋ | 1367/1570 [1:47:06<10:02, 2.97s/it]
87%|████████▋ | 1368/1570 [1:47:09<10:00, 2.97s/it]
87%|████████▋ | 1369/1570 [1:47:12<09:54, 2.96s/it]
87%|████████▋ | 1370/1570 [1:47:15<09:53, 2.97s/it]
87%|████████▋ | 1371/1570 [1:47:18<09:52, 2.98s/it]
87%|████████▋ | 1372/1570 [1:47:21<09:53, 3.00s/it]
87%|████████▋ | 1373/1570 [1:47:24<09:48, 2.99s/it]
88%|████████▊ | 1374/1570 [1:47:27<09:41, 2.97s/it]
88%|████████▊ | 1375/1570 [1:47:30<09:39, 2.97s/it]
88%|████████▊ | 1376/1570 [1:47:33<09:36, 2.97s/it]
88%|████████▊ | 1377/1570 [1:47:35<09:27, 2.94s/it]
88%|████████▊ | 1378/1570 [1:47:38<09:28, 2.96s/it]
88%|████████▊ | 1379/1570 [1:47:41<09:30, 2.99s/it]
88%|████████▊ | 1380/1570 [1:47:44<09:28, 2.99s/it]
{'loss': '0.000621', 'grad_norm': '0.009249', 'learning_rate': '3.704e-06', 'epoch': '4.396'}
+
88%|████████▊ | 1380/1570 [1:47:44<09:28, 2.99s/it]
88%|████████▊ | 1381/1570 [1:47:47<09:25, 2.99s/it]
88%|████████▊ | 1382/1570 [1:47:50<09:17, 2.97s/it]
88%|████████▊ | 1383/1570 [1:47:53<09:16, 2.98s/it]
88%|████████▊ | 1384/1570 [1:47:56<09:12, 2.97s/it]
88%|████████▊ | 1385/1570 [1:47:59<09:13, 2.99s/it]
88%|████████▊ | 1386/1570 [1:48:02<09:08, 2.98s/it]
88%|████████▊ | 1387/1570 [1:48:05<09:04, 2.98s/it]
88%|████████▊ | 1388/1570 [1:48:08<09:03, 2.98s/it]
88%|████████▊ | 1389/1570 [1:48:11<09:01, 2.99s/it]
89%|████████▊ | 1390/1570 [1:48:15<09:19, 3.11s/it]
89%|████████▊ | 1391/1570 [1:48:18<09:08, 3.06s/it]
89%|████████▊ | 1392/1570 [1:48:21<08:58, 3.03s/it]
89%|████████▊ | 1393/1570 [1:48:24<08:52, 3.01s/it]
89%|████████▉ | 1394/1570 [1:48:26<08:46, 2.99s/it]
89%|████████▉ | 1395/1570 [1:48:29<08:41, 2.98s/it]
89%|████████▉ | 1396/1570 [1:48:32<08:40, 2.99s/it]
89%|████████▉ | 1397/1570 [1:48:35<08:33, 2.97s/it]
89%|████████▉ | 1398/1570 [1:48:38<08:33, 2.99s/it]
89%|████████▉ | 1399/1570 [1:48:41<08:28, 2.97s/it]
89%|████████▉ | 1400/1570 [1:48:44<08:24, 2.97s/it]
{'loss': '0.0005129', 'grad_norm': '0.01025', 'learning_rate': '2.996e-06', 'epoch': '4.46'}
+
89%|████████▉ | 1400/1570 [1:48:44<08:24, 2.97s/it]
89%|████████▉ | 1401/1570 [1:48:47<08:20, 2.96s/it]
89%|████████▉ | 1402/1570 [1:48:50<08:17, 2.96s/it]
89%|████████▉ | 1403/1570 [1:48:53<08:12, 2.95s/it]
89%|████████▉ | 1404/1570 [1:48:56<08:16, 2.99s/it]
89%|████████▉ | 1405/1570 [1:48:59<08:14, 2.99s/it]
90%|████████▉ | 1406/1570 [1:49:02<08:12, 3.00s/it]
90%|████████▉ | 1407/1570 [1:49:05<08:07, 2.99s/it]
90%|████████▉ | 1408/1570 [1:49:08<08:05, 3.00s/it]
90%|████████▉ | 1409/1570 [1:49:11<08:02, 3.00s/it]
90%|████████▉ | 1410/1570 [1:49:14<08:01, 3.01s/it]
90%|████████▉ | 1411/1570 [1:49:17<07:54, 2.98s/it]
90%|████████▉ | 1412/1570 [1:49:20<07:51, 2.98s/it]
90%|█████████ | 1413/1570 [1:49:23<07:50, 2.99s/it]
90%|█████████ | 1414/1570 [1:49:26<07:47, 3.00s/it]
90%|█████████ | 1415/1570 [1:49:29<07:46, 3.01s/it]
90%|█████████ | 1416/1570 [1:49:32<07:41, 3.00s/it]
90%|█████████ | 1417/1570 [1:49:35<07:37, 2.99s/it]
90%|█████████ | 1418/1570 [1:49:38<07:33, 2.98s/it]
90%|█████████ | 1419/1570 [1:49:41<07:30, 2.99s/it]
90%|█████████ | 1420/1570 [1:49:44<07:30, 3.00s/it]
{'loss': '0.0004933', 'grad_norm': '0.02401', 'learning_rate': '2.363e-06', 'epoch': '4.523'}
+
90%|█████████ | 1420/1570 [1:49:44<07:30, 3.00s/it]
91%|█████████ | 1421/1570 [1:49:47<07:26, 3.00s/it]
91%|█████████ | 1422/1570 [1:49:50<07:21, 2.98s/it]
91%|█████████ | 1423/1570 [1:49:53<07:18, 2.98s/it]
91%|█████████ | 1424/1570 [1:49:56<07:17, 3.00s/it]
91%|█████████ | 1425/1570 [1:49:59<07:15, 3.00s/it]
91%|█████████ | 1426/1570 [1:50:02<07:12, 3.00s/it]
91%|█████████ | 1427/1570 [1:50:05<07:05, 2.98s/it]
91%|█████████ | 1428/1570 [1:50:08<07:01, 2.97s/it]
91%|█████████ | 1429/1570 [1:50:11<06:57, 2.96s/it]
91%|█████████ | 1430/1570 [1:50:14<06:54, 2.96s/it]
91%|█████████ | 1431/1570 [1:50:17<06:51, 2.96s/it]
91%|█████████ | 1432/1570 [1:50:20<06:52, 2.99s/it]
91%|█████████▏| 1433/1570 [1:50:23<06:49, 2.99s/it]
91%|█████████▏| 1434/1570 [1:50:26<06:49, 3.01s/it]
91%|█████████▏| 1435/1570 [1:50:29<06:46, 3.01s/it]
91%|█████████▏| 1436/1570 [1:50:32<06:42, 3.00s/it]
92%|█████████▏| 1437/1570 [1:50:35<06:39, 3.01s/it]
92%|█████████▏| 1438/1570 [1:50:38<06:38, 3.02s/it]
92%|█████████▏| 1439/1570 [1:50:41<06:33, 3.00s/it]
92%|█████████▏| 1440/1570 [1:50:44<06:29, 2.99s/it]
{'loss': '0.0005305', 'grad_norm': '0.02824', 'learning_rate': '1.806e-06', 'epoch': '4.587'}
+
92%|█████████▏| 1440/1570 [1:50:44<06:29, 2.99s/it]
92%|█████████▏| 1441/1570 [1:50:47<06:26, 3.00s/it]
92%|█████████▏| 1442/1570 [1:50:50<06:19, 2.97s/it]
92%|█████████▏| 1443/1570 [1:50:53<06:15, 2.96s/it]
92%|█████████▏| 1444/1570 [1:50:56<06:11, 2.95s/it]
92%|█████████▏| 1445/1570 [1:50:59<06:08, 2.95s/it]
92%|█████████▏| 1446/1570 [1:51:02<06:05, 2.95s/it]
92%|█████████▏| 1447/1570 [1:51:05<06:03, 2.96s/it]
92%|█████████▏| 1448/1570 [1:51:08<05:59, 2.95s/it]
92%|█████████▏| 1449/1570 [1:51:10<05:56, 2.95s/it]
92%|█████████▏| 1450/1570 [1:51:13<05:54, 2.95s/it]
92%|█████████▏| 1451/1570 [1:51:17<05:54, 2.98s/it]
92%|█████████▏| 1452/1570 [1:51:19<05:50, 2.97s/it]
93%|█████████▎| 1453/1570 [1:51:22<05:48, 2.98s/it]
93%|█████████▎| 1454/1570 [1:51:25<05:46, 2.99s/it]
93%|█████████▎| 1455/1570 [1:51:28<05:41, 2.97s/it]
93%|█████████▎| 1456/1570 [1:51:31<05:38, 2.97s/it]
93%|█████████▎| 1457/1570 [1:51:34<05:34, 2.96s/it]
93%|█████████▎| 1458/1570 [1:51:37<05:32, 2.96s/it]
93%|█████████▎| 1459/1570 [1:51:40<05:29, 2.97s/it]
93%|█████████▎| 1460/1570 [1:51:43<05:24, 2.95s/it]
{'loss': '0.0003377', 'grad_norm': '0.01864', 'learning_rate': '1.327e-06', 'epoch': '4.651'}
+
93%|█████████▎| 1460/1570 [1:51:43<05:24, 2.95s/it]
93%|█████████▎| 1461/1570 [1:51:46<05:20, 2.94s/it]
93%|█████████▎| 1462/1570 [1:51:49<05:18, 2.95s/it]
93%|█████████▎| 1463/1570 [1:51:52<05:17, 2.97s/it]
93%|█████████▎| 1464/1570 [1:51:55<05:13, 2.96s/it]
93%|█████████▎| 1465/1570 [1:51:58<05:11, 2.97s/it]
93%|█████████▎| 1466/1570 [1:52:01<05:08, 2.96s/it]
93%|█████████▎| 1467/1570 [1:52:04<05:04, 2.95s/it]
94%|█████████▎| 1468/1570 [1:52:07<05:03, 2.97s/it]
94%|█████████▎| 1469/1570 [1:52:10<04:59, 2.96s/it]
94%|█████████▎| 1470/1570 [1:52:13<04:57, 2.97s/it]
94%|█████████▎| 1471/1570 [1:52:16<04:53, 2.96s/it]
94%|█████████▍| 1472/1570 [1:52:19<04:48, 2.95s/it]
94%|█████████▍| 1473/1570 [1:52:22<04:45, 2.94s/it]
94%|█████████▍| 1474/1570 [1:52:25<04:42, 2.94s/it]
94%|█████████▍| 1475/1570 [1:52:28<04:40, 2.95s/it]
94%|█████████▍| 1476/1570 [1:52:31<04:38, 2.96s/it]
94%|█████████▍| 1477/1570 [1:52:33<04:34, 2.96s/it]
94%|█████████▍| 1478/1570 [1:52:36<04:33, 2.97s/it]
94%|█████████▍| 1479/1570 [1:52:39<04:27, 2.94s/it]
94%|█████████▍| 1480/1570 [1:52:42<04:25, 2.95s/it]
{'loss': '0.0002382', 'grad_norm': '0.003617', 'learning_rate': '9.258e-07', 'epoch': '4.715'}
+
94%|█████████▍| 1480/1570 [1:52:42<04:25, 2.95s/it]
94%|█████████▍| 1481/1570 [1:52:45<04:24, 2.97s/it]
94%|█████████▍| 1482/1570 [1:52:48<04:19, 2.95s/it]
94%|█████████▍| 1483/1570 [1:52:51<04:16, 2.95s/it]
95%|█████████▍| 1484/1570 [1:52:54<04:14, 2.96s/it]
95%|█████████▍| 1485/1570 [1:52:57<04:11, 2.95s/it]
95%|█████████▍| 1486/1570 [1:53:00<04:08, 2.96s/it]
95%|█████████▍| 1487/1570 [1:53:03<04:04, 2.95s/it]
95%|█████████▍| 1488/1570 [1:53:06<04:04, 2.98s/it]
95%|█████████▍| 1489/1570 [1:53:09<04:01, 2.98s/it]
95%|█████████▍| 1490/1570 [1:53:12<03:58, 2.98s/it]
95%|█████████▍| 1491/1570 [1:53:15<03:54, 2.97s/it]
95%|█████████▌| 1492/1570 [1:53:18<03:51, 2.96s/it]
95%|█████████▌| 1493/1570 [1:53:21<03:47, 2.95s/it]
95%|█████████▌| 1494/1570 [1:53:24<03:45, 2.96s/it]
95%|█████████▌| 1495/1570 [1:53:27<03:42, 2.96s/it]
95%|█████████▌| 1496/1570 [1:53:30<03:40, 2.98s/it]
95%|█████████▌| 1497/1570 [1:53:33<03:37, 2.97s/it]
95%|█████████▌| 1498/1570 [1:53:36<03:34, 2.98s/it]
95%|█████████▌| 1499/1570 [1:53:39<03:31, 2.98s/it]
96%|█████████▌| 1500/1570 [1:53:42<03:27, 2.97s/it]
{'loss': '0.0005346', 'grad_norm': '0.01665', 'learning_rate': '6.033e-07', 'epoch': '4.779'}
+
96%|█████████▌| 1500/1570 [1:53:42<03:27, 2.97s/it]
96%|█████████▌| 1501/1570 [1:53:45<03:24, 2.96s/it]
96%|█████████▌| 1502/1570 [1:53:48<03:21, 2.96s/it]
96%|█████████▌| 1503/1570 [1:53:51<03:18, 2.96s/it]
96%|█████████▌| 1504/1570 [1:53:54<03:16, 2.98s/it]
96%|█████████▌| 1505/1570 [1:53:57<03:13, 2.98s/it]
96%|█████████▌| 1506/1570 [1:54:00<03:10, 2.98s/it]
96%|█████████▌| 1507/1570 [1:54:02<03:07, 2.97s/it]
96%|█████████▌| 1508/1570 [1:54:05<03:04, 2.97s/it]
96%|█████████▌| 1509/1570 [1:54:08<03:00, 2.96s/it]
96%|█████████▌| 1510/1570 [1:54:11<02:57, 2.96s/it]
96%|█████████▌| 1511/1570 [1:54:14<02:55, 2.97s/it]
96%|█████████▋| 1512/1570 [1:54:17<02:51, 2.96s/it]
96%|█████████▋| 1513/1570 [1:54:20<02:47, 2.95s/it]
96%|█████████▋| 1514/1570 [1:54:23<02:45, 2.95s/it]
96%|█████████▋| 1515/1570 [1:54:26<02:43, 2.97s/it]
97%|█████████▋| 1516/1570 [1:54:29<02:39, 2.96s/it]
97%|█████████▋| 1517/1570 [1:54:32<02:38, 2.98s/it]
97%|█████████▋| 1518/1570 [1:54:35<02:34, 2.97s/it]
97%|█████████▋| 1519/1570 [1:54:38<02:32, 2.98s/it]
97%|█████████▋| 1520/1570 [1:54:41<02:27, 2.95s/it]
{'loss': '0.001527', 'grad_norm': '0.148', 'learning_rate': '3.599e-07', 'epoch': '4.842'}
+
97%|█████████▋| 1520/1570 [1:54:41<02:27, 2.95s/it]
97%|█████████▋| 1521/1570 [1:54:44<02:25, 2.97s/it]
97%|█████████▋| 1522/1570 [1:54:47<02:22, 2.97s/it]
97%|█████████▋| 1523/1570 [1:54:50<02:20, 2.98s/it]
97%|█████████▋| 1524/1570 [1:54:53<02:17, 3.00s/it]
97%|█████████▋| 1525/1570 [1:54:56<02:14, 2.98s/it]
97%|█████████▋| 1526/1570 [1:54:59<02:11, 2.99s/it]
97%|█████████▋| 1527/1570 [1:55:02<02:08, 3.00s/it]
97%|█████████▋| 1528/1570 [1:55:05<02:06, 3.00s/it]
97%|█████████▋| 1529/1570 [1:55:08<02:02, 2.99s/it]
97%|█████████▋| 1530/1570 [1:55:11<01:58, 2.97s/it]
98%|█████████▊| 1531/1570 [1:55:14<01:55, 2.97s/it]
98%|█████████▊| 1532/1570 [1:55:17<01:53, 2.98s/it]
98%|█████████▊| 1533/1570 [1:55:20<01:50, 2.98s/it]
98%|█████████▊| 1534/1570 [1:55:23<01:46, 2.97s/it]
98%|█████████▊| 1535/1570 [1:55:26<01:44, 2.97s/it]
98%|█████████▊| 1536/1570 [1:55:29<01:41, 2.99s/it]
98%|█████████▊| 1537/1570 [1:55:32<01:38, 3.00s/it]
98%|█████████▊| 1538/1570 [1:55:35<01:35, 2.99s/it]
98%|█████████▊| 1539/1570 [1:55:38<01:32, 2.99s/it]
98%|█████████▊| 1540/1570 [1:55:41<01:29, 2.99s/it]
{'loss': '0.0003546', 'grad_norm': '0.1594', 'learning_rate': '1.961e-07', 'epoch': '4.906'}
+
98%|█████████▊| 1540/1570 [1:55:41<01:29, 2.99s/it]
98%|█████████▊| 1541/1570 [1:55:44<01:26, 2.98s/it]
98%|█████████▊| 1542/1570 [1:55:47<01:22, 2.95s/it]
98%|█████████▊| 1543/1570 [1:55:50<01:20, 2.96s/it]
98%|█████████▊| 1544/1570 [1:55:53<01:17, 2.96s/it]
98%|█████████▊| 1545/1570 [1:55:56<01:14, 2.98s/it]
98%|█████████▊| 1546/1570 [1:55:58<01:10, 2.96s/it]
99%|█████████▊| 1547/1570 [1:56:01<01:07, 2.95s/it]
99%|█████████▊| 1548/1570 [1:56:04<01:05, 2.96s/it]
99%|█████████▊| 1549/1570 [1:56:07<01:02, 2.98s/it]
99%|█████████▊| 1550/1570 [1:56:10<00:59, 2.98s/it]
99%|█████████▉| 1551/1570 [1:56:13<00:56, 2.96s/it]
99%|█████████▉| 1552/1570 [1:56:16<00:53, 2.96s/it]
99%|█████████▉| 1553/1570 [1:56:19<00:50, 2.96s/it]
99%|█████████▉| 1554/1570 [1:56:22<00:47, 2.97s/it]
99%|█████████▉| 1555/1570 [1:56:25<00:44, 2.96s/it]
99%|█████████▉| 1556/1570 [1:56:28<00:41, 2.97s/it]
99%|█████████▉| 1557/1570 [1:56:31<00:38, 2.97s/it]
99%|█████████▉| 1558/1570 [1:56:34<00:35, 2.96s/it]
99%|█████████▉| 1559/1570 [1:56:37<00:32, 2.96s/it]
99%|█████████▉| 1560/1570 [1:56:40<00:29, 2.97s/it]
{'loss': '0.0004393', 'grad_norm': '0.01692', 'learning_rate': '1.121e-07', 'epoch': '4.97'}
+
99%|█████████▉| 1560/1570 [1:56:40<00:29, 2.97s/it]
99%|█████████▉| 1561/1570 [1:56:43<00:26, 2.99s/it]
99%|█████████▉| 1562/1570 [1:56:46<00:23, 2.97s/it]
100%|█████████▉| 1563/1570 [1:56:49<00:20, 2.96s/it]
100%|█████████▉| 1564/1570 [1:56:52<00:17, 2.97s/it]
100%|█████████▉| 1565/1570 [1:56:55<00:14, 2.98s/it]
100%|█████████▉| 1566/1570 [1:56:58<00:11, 2.99s/it]
100%|█████████▉| 1567/1570 [1:57:01<00:08, 3.00s/it]
100%|█████████▉| 1568/1570 [1:57:04<00:06, 3.00s/it]
100%|█████████▉| 1569/1570 [1:57:07<00:02, 2.98s/it]
100%|██████████| 1570/1570 [1:57:08<00:00, 2.44s/it][INFO|trainer.py:2668] 2026-08-23 02:21:26,151 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 02:21:26,151 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 02:21:26,151 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/amid/checkpoint-1570
+[INFO|configuration_utils.py:780] 2026-08-23 02:31:18,831 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 02:31:18,832 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 02:31:18,868 >> chat template saved in results/qwen3/amid/checkpoint-1570/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 02:31:18,868 >> tokenizer config file saved in results/qwen3/amid/checkpoint-1570/tokenizer_config.json
+[INFO|trainer.py:1823] 2026-08-23 02:31:19,487 >>
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
{'train_runtime': '7622', 'train_samples_per_second': '6.578', 'train_steps_per_second': '0.206', 'train_loss': '0.01864', 'epoch': '5'}
+
100%|██████████| 1570/1570 [2:07:01<00:00, 2.44s/it]
100%|██████████| 1570/1570 [2:07:01<00:00, 4.85s/it]
+[INFO|trainer.py:3815] 2026-08-23 02:31:19,943 >> Saving model checkpoint to results/qwen3/amid
+[INFO|configuration_utils.py:780] 2026-08-23 02:31:20,436 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 02:31:20,438 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 02:31:20,471 >> chat template saved in results/qwen3/amid/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 02:31:20,471 >> tokenizer config file saved in results/qwen3/amid/tokenizer_config.json
+[transformers] The following generation flags are not valid and may be ignored: ['temperature', 'top_p']. Set `TRANSFORMERS_VERBOSITY=info` for more details.
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+***** train metrics *****
+ epoch = 5.0
+ total_flos = 54682091GF
+ train_loss = 0.0186
+ train_runtime = 2:07:01.85
+ train_samples_per_second = 6.578
+ train_steps_per_second = 0.206
+[INFO|trainer.py:2668] 2026-08-23 02:31:20,582 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 02:31:20,582 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 02:31:20,582 >> Batch size = 16
+[WARNING|logging.py:340] 2026-08-23 02:31:20,701 >> The following generation flags are not valid and may be ignored: ['temperature', 'top_p'].
+[INFO|logging.py:355] 2026-08-23 02:31:20,702 >> - `temperature`: `do_sample` is not set to `True`. However, `temperature` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.
+- `top_p`: `do_sample` is not set to `True`. However, `top_p` is set to `0.7` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.
+If you're using a pretrained model, note that some of these attributes may be set through the model's `generation_config.json` file.
+[WARNING|utils.py:1642] 2026-08-23 02:31:20,704 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
0%| | 0/107 [00:00> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
2%|▏ | 2/107 [00:05<04:47, 2.74s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:31:31,757 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
3%|▎ | 3/107 [00:10<06:39, 3.84s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:31:37,154 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
4%|▎ | 4/107 [00:16<07:39, 4.46s/it][WARNING|utils.py:1642] 2026-08-23 02:31:42,641 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
5%|▍ | 5/107 [00:21<08:10, 4.81s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:31:48,110 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
6%|▌ | 6/107 [00:26<08:03, 4.79s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:31:52,856 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 7/107 [00:31<08:18, 4.98s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:31:58,241 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 8/107 [00:37<08:23, 5.09s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:32:03,562 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
8%|▊ | 9/107 [00:43<08:42, 5.34s/it][WARNING|utils.py:1642] 2026-08-23 02:32:09,448 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
9%|▉ | 10/107 [00:48<08:46, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:32:15,090 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
10%|█ | 11/107 [00:53<08:16, 5.18s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:32:19,690 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
11%|█ | 12/107 [00:59<08:26, 5.34s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:32:25,393 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
12%|█▏ | 13/107 [01:04<08:26, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:32:30,890 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
13%|█▎ | 14/107 [01:09<08:17, 5.35s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:32:36,167 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
14%|█▍ | 15/107 [01:15<08:16, 5.40s/it][WARNING|utils.py:1642] 2026-08-23 02:32:41,662 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
15%|█▍ | 16/107 [01:20<08:15, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:32:47,239 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
16%|█▌ | 17/107 [01:26<08:20, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:32:53,068 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
17%|█▋ | 18/107 [01:32<08:16, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:32:58,685 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
18%|█▊ | 19/107 [01:37<08:04, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:33:04,001 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
19%|█▊ | 20/107 [01:43<07:55, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:33:09,373 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
20%|█▉ | 21/107 [01:47<07:33, 5.27s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:33:14,203 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██ | 22/107 [01:53<07:37, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:33:19,852 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██▏ | 23/107 [01:59<07:34, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:33:25,306 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
22%|██▏ | 24/107 [02:04<07:27, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:33:30,673 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
23%|██▎ | 25/107 [02:10<07:45, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:33:37,019 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
24%|██▍ | 26/107 [02:16<07:41, 5.70s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:33:42,758 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
25%|██▌ | 27/107 [02:21<07:30, 5.63s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:33:48,238 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
26%|██▌ | 28/107 [02:27<07:24, 5.63s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:33:53,865 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
27%|██▋ | 29/107 [02:33<07:27, 5.74s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:33:59,869 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
28%|██▊ | 30/107 [02:38<07:13, 5.63s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:34:05,242 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
29%|██▉ | 31/107 [02:43<06:42, 5.30s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:34:09,770 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
30%|██▉ | 32/107 [02:48<06:39, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:34:15,170 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
31%|███ | 33/107 [02:54<06:38, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:34:20,672 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
32%|███▏ | 34/107 [03:00<06:51, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:34:26,904 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
33%|███▎ | 35/107 [03:06<06:41, 5.58s/it][WARNING|utils.py:1642] 2026-08-23 02:34:32,348 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
34%|███▎ | 36/107 [03:11<06:29, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:34:37,629 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
35%|███▍ | 37/107 [03:16<06:07, 5.25s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:34:42,335 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▌ | 38/107 [03:21<06:09, 5.36s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:34:47,941 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▋ | 39/107 [03:26<05:56, 5.24s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:34:52,886 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
37%|███▋ | 40/107 [03:31<05:51, 5.25s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:34:58,164 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
38%|███▊ | 41/107 [03:37<05:50, 5.32s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:35:03,636 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
39%|███▉ | 42/107 [03:43<05:57, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:35:09,579 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
40%|████ | 43/107 [03:49<05:57, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:35:15,380 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
41%|████ | 44/107 [03:54<05:46, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:35:20,665 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
42%|████▏ | 45/107 [03:59<05:31, 5.35s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:35:25,652 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
43%|████▎ | 46/107 [04:04<05:24, 5.32s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:35:30,901 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
44%|████▍ | 47/107 [04:10<05:27, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:35:36,703 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
45%|████▍ | 48/107 [04:16<05:25, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:35:42,352 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
46%|████▌ | 49/107 [04:21<05:16, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:35:47,673 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
47%|████▋ | 50/107 [04:26<05:09, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:35:53,020 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
48%|████▊ | 51/107 [04:32<05:06, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:35:58,627 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
49%|████▊ | 52/107 [04:37<04:55, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:36:03,776 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|████▉ | 53/107 [04:43<04:58, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:36:09,656 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|█████ | 54/107 [04:48<04:46, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:36:14,799 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
51%|█████▏ | 55/107 [04:54<04:44, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:36:20,409 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
52%|█████▏ | 56/107 [04:59<04:43, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:36:26,140 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
53%|█████▎ | 57/107 [05:05<04:39, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:36:31,806 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
54%|█████▍ | 58/107 [05:10<04:30, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:36:37,143 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
55%|█████▌ | 59/107 [05:16<04:22, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:36:42,509 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
56%|█████▌ | 60/107 [05:21<04:15, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:36:47,862 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
57%|█████▋ | 61/107 [05:26<04:00, 5.24s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:36:52,641 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
58%|█████▊ | 62/107 [05:31<03:56, 5.24s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:36:57,904 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
59%|█████▉ | 63/107 [05:37<03:53, 5.31s/it][WARNING|utils.py:1642] 2026-08-23 02:37:03,358 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
60%|█████▉ | 64/107 [05:42<03:46, 5.28s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:37:08,563 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
61%|██████ | 65/107 [05:47<03:40, 5.25s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:37:13,758 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
62%|██████▏ | 66/107 [05:52<03:36, 5.27s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:37:19,080 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
63%|██████▎ | 67/107 [05:58<03:31, 5.30s/it][WARNING|utils.py:1642] 2026-08-23 02:37:24,427 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▎ | 68/107 [06:02<03:20, 5.15s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:37:29,232 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▍ | 69/107 [06:08<03:15, 5.13s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:37:34,330 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
65%|██████▌ | 70/107 [06:13<03:18, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:37:40,274 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
66%|██████▋ | 71/107 [06:19<03:17, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:37:45,992 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
67%|██████▋ | 72/107 [06:24<03:09, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:37:51,264 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
68%|██████▊ | 73/107 [06:30<03:04, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:37:56,752 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
69%|██████▉ | 74/107 [06:35<02:58, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:38:02,100 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
70%|███████ | 75/107 [06:41<02:51, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:38:07,366 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
71%|███████ | 76/107 [06:46<02:47, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:38:12,809 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
72%|███████▏ | 77/107 [06:51<02:42, 5.40s/it][WARNING|utils.py:1642] 2026-08-23 02:38:18,242 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
73%|███████▎ | 78/107 [06:57<02:37, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:38:23,755 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
74%|███████▍ | 79/107 [07:02<02:31, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:38:29,141 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
75%|███████▍ | 80/107 [07:08<02:28, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:38:34,867 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
76%|███████▌ | 81/107 [07:13<02:22, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:38:40,227 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
77%|███████▋ | 82/107 [07:19<02:15, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:38:45,472 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
78%|███████▊ | 83/107 [07:24<02:07, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:38:50,638 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▊ | 84/107 [07:29<02:01, 5.27s/it][WARNING|utils.py:1642] 2026-08-23 02:38:55,760 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▉ | 85/107 [07:34<01:52, 5.13s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:39:00,571 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
80%|████████ | 86/107 [07:39<01:50, 5.25s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:39:06,109 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
81%|████████▏ | 87/107 [07:44<01:40, 5.04s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:39:10,658 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
82%|████████▏ | 88/107 [07:49<01:37, 5.13s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:39:15,987 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
83%|████████▎ | 89/107 [07:56<01:38, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:39:22,350 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
84%|████████▍ | 90/107 [08:01<01:32, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:39:27,680 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
85%|████████▌ | 91/107 [08:06<01:25, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:39:32,729 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
86%|████████▌ | 92/107 [08:11<01:18, 5.25s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:39:37,782 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
87%|████████▋ | 93/107 [08:16<01:13, 5.26s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:39:43,063 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
88%|████████▊ | 94/107 [08:22<01:09, 5.34s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:39:48,602 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
89%|████████▉ | 95/107 [08:27<01:03, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:39:53,907 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
90%|████████▉ | 96/107 [08:32<00:58, 5.28s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:39:59,072 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
91%|█████████ | 97/107 [08:38<00:53, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:40:04,466 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
92%|█████████▏| 98/107 [08:43<00:48, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:40:09,955 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 99/107 [08:49<00:43, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:40:15,358 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 100/107 [08:54<00:37, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:40:20,497 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
94%|█████████▍| 101/107 [08:59<00:32, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:40:26,202 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
95%|█████████▌| 102/107 [09:05<00:27, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:40:31,695 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
96%|█████████▋| 103/107 [09:10<00:21, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:40:36,896 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
97%|█████████▋| 104/107 [09:16<00:16, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:40:42,327 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
98%|█████████▊| 105/107 [09:21<00:10, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:40:47,985 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
99%|█████████▉| 106/107 [09:27<00:05, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 02:40:53,559 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
100%|██████████| 107/107 [09:32<00:00, 5.52s/it]
100%|██████████| 107/107 [09:33<00:00, 5.36s/it]
+***** eval metrics *****
+ epoch = 5.0
+ eval_generator_count = 1712.0
+ eval_generator_exact_match = 96.028
+ eval_generator_rouge1 = 99.8671
+ eval_generator_rouge2 = 99.8011
+ eval_generator_rougeL = 99.6739
+ eval_loss = 0.004
+ eval_runtime = 0:09:39.64
+ eval_samples_per_second = 5.89
+ eval_selector_accuracy = 98.6565
+ eval_selector_count = 1712.0
+ eval_steps_per_second = 0.185
+[INFO|modelcard.py:263] 2026-08-23 02:41:00,227 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
diff --git a/qwen3/run_all_logs/20260822_031516/csd.log b/qwen3/run_all_logs/20260822_031516/csd.log
new file mode 100644
index 0000000000000000000000000000000000000000..fac0ea6d3998aa60734135e124f6b949c7326f61
--- /dev/null
+++ b/qwen3/run_all_logs/20260822_031516/csd.log
@@ -0,0 +1,2100 @@
+W0822 19:34:52.807000 1400844 torch/distributed/run.py:803]
+W0822 19:34:52.807000 1400844 torch/distributed/run.py:803] *****************************************
+W0822 19:34:52.807000 1400844 torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+W0822 19:34:52.807000 1400844 torch/distributed/run.py:803] *****************************************
+[INFO|2026-08-22 19:34:59] llamafactory.hparams.parser:651 >> Process rank: 1, world size: 2, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16
+[INFO|2026-08-22 19:34:59] llamafactory.hparams.parser:144 >> Set `ddp_find_unused_parameters` to False in DDP training since LoRA is enabled.
+[INFO|2026-08-22 19:34:59] llamafactory.hparams.parser:651 >> Process rank: 0, world size: 2, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16
+Starting csd with LlamaFactory and DeepSpeed=True.
+[INFO|configuration_utils.py:780] 2026-08-22 19:35:00,891 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 19:35:00,897 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 19:35:03,979 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 19:35:03,981 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 19:35:04,218 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 19:35:04,220 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 19:35:05] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_train.jsonl...
+[INFO|2026-08-22 19:35:06] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_eval.jsonl...
+training example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 1947, 19029, 10592, 311, 35592, 304, 279, 17328, 1947, 1707, 429, 9482, 1573, 279, 1042, 220, 16, 21, 15, 15, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 9286, 38050, 756, 414, 330, 13193, 788, 341, 286, 330, 408, 14645, 788, 330, 68689, 756, 286, 330, 2468, 14645, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 37525, 756, 414, 330, 13193, 788, 5613, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 18098, 287, 756, 414, 330, 13193, 788, 341, 286, 330, 11141, 3575, 34043, 788, 330, 16468, 756, 286, 330, 37375, 14645, 788, 330, 68689, 698, 414, 456, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 9286, 38050, 756, 414, 330, 1313, 788, 330, 49087, 2354, 698, 262, 1153, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 37525, 756, 414, 330, 1313, 788, 330, 4648, 37525, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of art movements linked to paintings in the genre art style that ended before the year 1600?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "ArtMovement",
+ "properties": {
+ "end_year": "INTEGER",
+ "start_year": "INTEGER"
+ }
+ },
+ {
+ "label": "Genre",
+ "properties": {}
+ },
+ {
+ "label": "Painting",
+ "properties": {
+ "country_of_origin": "STRING",
+ "creation_year": "INTEGER"
+ }
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "ArtMovement",
+ "type": "associatedWith"
+ },
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "Genre",
+ "type": "hasGenre"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+eval example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 9893, 1483, 304, 2987, 279, 220, 17, 15, 17, 16, 55218, 18144, 73379, 476, 279, 23007, 24847, 7755, 9257, 36331, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 13193, 788, 341, 286, 330, 1028, 788, 330, 7097, 756, 286, 330, 31309, 788, 330, 22852, 756, 286, 330, 4082, 3575, 2259, 26286, 788, 330, 68689, 756, 286, 330, 4082, 3575, 1243, 89827, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 28214, 756, 414, 330, 13193, 788, 5613, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 5657, 788, 330, 28214, 756, 414, 330, 1313, 788, 330, 30210, 1047, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of weapons used in either the 2021 Baghdad suicide bombings or the Botroseya Church bombing?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "TerroristAttack",
+ "properties": {
+ "date": "DATE",
+ "locations": "LIST",
+ "number_of_deaths": "INTEGER",
+ "number_of_injuries": "INTEGER"
+ }
+ },
+ {
+ "label": "Weapon",
+ "properties": {}
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "TerroristAttack",
+ "target": "Weapon",
+ "type": "employs"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+[INFO|configuration_utils.py:780] 2026-08-22 19:35:10,140 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 19:35:10,143 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 19:35:10] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training.
+[INFO|modeling_utils.py:769] 2026-08-22 19:35:10,858 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/model.safetensors
+[INFO|modeling_utils.py:839] 2026-08-22 19:35:10,859 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 19:35:10,862 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": false
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 19:35:11,488 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95
+}
+
+[INFO|2026-08-22 19:35:11] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled.
+[INFO|2026-08-22 19:35:11] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-22 19:35:11] llamafactory.model.adapter:144 >> Upcasting trainable params to float32.
+[INFO|2026-08-22 19:35:11] llamafactory.model.adapter:144 >> Fine-tuning method: LoRA
+[INFO|2026-08-22 19:35:11] llamafactory.model.model_utils.misc:144 >> Found linear modules: down_proj,up_proj,v_proj,q_proj,gate_proj,o_proj,k_proj
+[INFO|2026-08-22 19:35:12] llamafactory.model.loader:144 >> trainable params: 20,185,088 || all params: 616,235,008 || trainable%: 3.2756
+[INFO|configuration_utils.py:780] 2026-08-22 19:35:12,400 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 19:35:12,405 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 19:35:15,174 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 19:35:15,458 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 19:35:15,460 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 19:35:17,467 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 19:35:17,469 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 19:35:17] llamafactory.model.model_utils.kv_cache:144 >> KV cache is enabled for faster generation.
+[INFO|modeling_utils.py:769] 2026-08-22 19:35:17,706 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model.safetensors.index.json
+[INFO|modeling_utils.py:839] 2026-08-22 19:35:17,707 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 19:35:17,708 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": true
+}
+
+
Loading weights: 0%| | 0/398 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 19:35:19,678 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.7,
+ "top_k": 20,
+ "top_p": 0.8
+}
+
+[INFO|2026-08-22 19:35:19] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-22 19:35:20] llamafactory.model.adapter:144 >> Merged 1 adapter(s).
+[INFO|2026-08-22 19:35:20] llamafactory.model.adapter:144 >> Loaded adapter(s): results/qwen3/teacher_lora
+[INFO|2026-08-22 19:35:20] llamafactory.model.loader:144 >> all params: 4,022,468,096
+[INFO|2026-08-22 19:35:20] llamafactory.train.trainer_utils:144 >> Created reference model from Qwen/Qwen3-4B-Instruct-2507
+
Loading weights: 0%| | 0/398 [00:00> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}.
+Gradient accumulation steps mismatch: GradientAccumulationPlugin has 1, DeepSpeed config has 8. Using DeepSpeed's value.
+[INFO|trainer.py:1469] 2026-08-22 19:35:25,914 >> ***** Running training *****
+[INFO|trainer.py:1470] 2026-08-22 19:35:25,915 >> Num examples = 10,027
+[INFO|trainer.py:1471] 2026-08-22 19:35:25,915 >> Num Epochs = 5
+[INFO|trainer.py:1472] 2026-08-22 19:35:25,915 >> Num update steps per epoch = 314
+[INFO|trainer.py:1473] 2026-08-22 19:35:25,915 >> Instantaneous batch size per device = 2
+[INFO|trainer.py:1476] 2026-08-22 19:35:25,915 >> Total train batch size (w. parallel, distributed & accumulation) = 32
+[INFO|trainer.py:1477] 2026-08-22 19:35:25,915 >> Gradient Accumulation steps = 8
+[INFO|trainer.py:1478] 2026-08-22 19:35:25,915 >> Total optimization steps = 1,570
+[INFO|trainer.py:1479] 2026-08-22 19:35:25,918 >> Number of trainable parameters = 20,185,088
+
0%| | 0/1570 [00:00>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 19:50:29,052 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 19:50:29,052 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/csd/checkpoint-314
+[INFO|configuration_utils.py:780] 2026-08-22 20:00:33,520 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 20:00:33,522 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 20:00:33,555 >> chat template saved in results/qwen3/csd/checkpoint-314/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 20:00:33,556 >> tokenizer config file saved in results/qwen3/csd/checkpoint-314/tokenizer_config.json
+
20%|██ | 315/1570 [25:11<64:09:18, 184.03s/it]
20%|██ | 316/1570 [25:14<45:10:00, 129.67s/it]
20%|██ | 317/1570 [25:16<31:52:55, 91.60s/it]
20%|██ | 318/1570 [25:19<22:35:46, 64.97s/it]
20%|██ | 319/1570 [25:22<16:05:43, 46.32s/it]
20%|██ | 320/1570 [25:25<11:32:47, 33.25s/it]
{'loss': '0.008805', 'grad_norm': '0.3481', 'learning_rate': '9.016e-05', 'epoch': '1.019'}
+
20%|██ | 320/1570 [25:25<11:32:47, 33.25s/it]
20%|██ | 321/1570 [25:28<8:21:59, 24.11s/it]
21%|██ | 322/1570 [25:30<6:08:05, 17.70s/it]
21%|██ | 323/1570 [25:33<4:34:51, 13.22s/it]
21%|██ | 324/1570 [25:36<3:29:40, 10.10s/it]
21%|██ | 325/1570 [25:39<2:44:07, 7.91s/it]
21%|██ | 326/1570 [25:41<2:11:51, 6.36s/it]
21%|██ | 327/1570 [25:44<1:49:30, 5.29s/it]
21%|██ | 328/1570 [25:47<1:33:54, 4.54s/it]
21%|██ | 329/1570 [25:50<1:23:05, 4.02s/it]
21%|██ | 330/1570 [25:53<1:15:34, 3.66s/it]
21%|██ | 331/1570 [25:55<1:10:10, 3.40s/it]
21%|██ | 332/1570 [25:58<1:06:10, 3.21s/it]
21%|██ | 333/1570 [26:01<1:03:30, 3.08s/it]
21%|██▏ | 334/1570 [26:04<1:01:38, 2.99s/it]
21%|██▏ | 335/1570 [26:06<1:00:24, 2.93s/it]
21%|██▏ | 336/1570 [26:09<59:18, 2.88s/it]
21%|██▏ | 337/1570 [26:12<58:40, 2.86s/it]
22%|██▏ | 338/1570 [26:15<58:03, 2.83s/it]
22%|██▏ | 339/1570 [26:18<57:47, 2.82s/it]
22%|██▏ | 340/1570 [26:20<57:51, 2.82s/it]
{'loss': '0.001391', 'grad_norm': '0.1989', 'learning_rate': '8.894e-05', 'epoch': '1.083'}
+
22%|██▏ | 340/1570 [26:20<57:51, 2.82s/it]
22%|██▏ | 341/1570 [26:23<57:47, 2.82s/it]
22%|██▏ | 342/1570 [26:26<57:44, 2.82s/it]
22%|██▏ | 343/1570 [26:29<57:24, 2.81s/it]
22%|██▏ | 344/1570 [26:32<57:02, 2.79s/it]
22%|██▏ | 345/1570 [26:34<56:54, 2.79s/it]
22%|██▏ | 346/1570 [26:37<56:37, 2.78s/it]
22%|██▏ | 347/1570 [26:40<56:47, 2.79s/it]
22%|██▏ | 348/1570 [26:43<56:38, 2.78s/it]
22%|██▏ | 349/1570 [26:45<56:36, 2.78s/it]
22%|██▏ | 350/1570 [26:48<56:20, 2.77s/it]
22%|██▏ | 351/1570 [26:51<56:10, 2.76s/it]
22%|██▏ | 352/1570 [26:54<56:01, 2.76s/it]
22%|██▏ | 353/1570 [26:57<56:07, 2.77s/it]
23%|██▎ | 354/1570 [26:59<56:22, 2.78s/it]
23%|██▎ | 355/1570 [27:02<56:24, 2.79s/it]
23%|██▎ | 356/1570 [27:05<56:25, 2.79s/it]
23%|██▎ | 357/1570 [27:08<56:25, 2.79s/it]
23%|██▎ | 358/1570 [27:11<56:26, 2.79s/it]
23%|██▎ | 359/1570 [27:13<56:25, 2.80s/it]
23%|██▎ | 360/1570 [27:16<56:06, 2.78s/it]
{'loss': '0.0005901', 'grad_norm': '1.051', 'learning_rate': '8.766e-05', 'epoch': '1.147'}
+
23%|██▎ | 360/1570 [27:16<56:06, 2.78s/it]
23%|██▎ | 361/1570 [27:19<56:11, 2.79s/it]
23%|██▎ | 362/1570 [27:22<56:07, 2.79s/it]
23%|██▎ | 363/1570 [27:24<55:55, 2.78s/it]
23%|██▎ | 364/1570 [27:27<55:59, 2.79s/it]
23%|██▎ | 365/1570 [27:30<55:56, 2.79s/it]
23%|██▎ | 366/1570 [27:33<55:58, 2.79s/it]
23%|██▎ | 367/1570 [27:36<55:43, 2.78s/it]
23%|██▎ | 368/1570 [27:38<55:43, 2.78s/it]
24%|██▎ | 369/1570 [27:41<55:48, 2.79s/it]
24%|██▎ | 370/1570 [27:44<55:48, 2.79s/it]
24%|██▎ | 371/1570 [27:47<55:33, 2.78s/it]
24%|██▎ | 372/1570 [27:49<55:27, 2.78s/it]
24%|██▍ | 373/1570 [27:52<55:26, 2.78s/it]
24%|██▍ | 374/1570 [27:55<55:17, 2.77s/it]
24%|██▍ | 375/1570 [27:58<55:36, 2.79s/it]
24%|██▍ | 376/1570 [28:01<55:43, 2.80s/it]
24%|██▍ | 377/1570 [28:03<55:35, 2.80s/it]
24%|██▍ | 378/1570 [28:06<55:31, 2.80s/it]
24%|██▍ | 379/1570 [28:09<55:32, 2.80s/it]
24%|██▍ | 380/1570 [28:12<55:10, 2.78s/it]
{'loss': '-0.0003093', 'grad_norm': '2.363', 'learning_rate': '8.631e-05', 'epoch': '1.211'}
+
24%|██▍ | 380/1570 [28:12<55:10, 2.78s/it]
24%|██▍ | 381/1570 [28:15<55:04, 2.78s/it]
24%|██▍ | 382/1570 [28:17<54:59, 2.78s/it]
24%|██▍ | 383/1570 [28:20<55:06, 2.79s/it]
24%|██▍ | 384/1570 [28:23<55:01, 2.78s/it]
25%|██▍ | 385/1570 [28:26<54:55, 2.78s/it]
25%|██▍ | 386/1570 [28:28<54:50, 2.78s/it]
25%|██▍ | 387/1570 [28:31<54:59, 2.79s/it]
25%|██▍ | 388/1570 [28:34<54:55, 2.79s/it]
25%|██▍ | 389/1570 [28:37<54:49, 2.79s/it]
25%|██▍ | 390/1570 [28:40<54:43, 2.78s/it]
25%|██▍ | 391/1570 [28:42<54:46, 2.79s/it]
25%|██▍ | 392/1570 [28:45<54:43, 2.79s/it]
25%|██▌ | 393/1570 [28:48<54:35, 2.78s/it]
25%|██▌ | 394/1570 [28:51<54:50, 2.80s/it]
25%|██▌ | 395/1570 [28:54<54:51, 2.80s/it]
25%|██▌ | 396/1570 [28:56<54:47, 2.80s/it]
25%|██▌ | 397/1570 [28:59<54:47, 2.80s/it]
25%|██▌ | 398/1570 [29:02<54:38, 2.80s/it]
25%|██▌ | 399/1570 [29:05<54:33, 2.80s/it]
25%|██▌ | 400/1570 [29:08<54:28, 2.79s/it]
{'loss': '-0.002663', 'grad_norm': '0.07664', 'learning_rate': '8.491e-05', 'epoch': '1.274'}
+
25%|██▌ | 400/1570 [29:08<54:28, 2.79s/it]
26%|██▌ | 401/1570 [29:10<54:35, 2.80s/it]
26%|██▌ | 402/1570 [29:13<54:22, 2.79s/it]
26%|██▌ | 403/1570 [29:16<54:30, 2.80s/it]
26%|██▌ | 404/1570 [29:19<58:24, 3.01s/it]
26%|██▌ | 405/1570 [29:23<1:01:56, 3.19s/it]
26%|██▌ | 406/1570 [29:27<1:04:03, 3.30s/it]
26%|██▌ | 407/1570 [29:30<1:01:17, 3.16s/it]
26%|██▌ | 408/1570 [29:32<58:55, 3.04s/it]
26%|██▌ | 409/1570 [29:35<57:29, 2.97s/it]
26%|██▌ | 410/1570 [29:38<56:10, 2.91s/it]
26%|██▌ | 411/1570 [29:41<55:13, 2.86s/it]
26%|██▌ | 412/1570 [29:43<54:37, 2.83s/it]
26%|██▋ | 413/1570 [29:46<54:17, 2.82s/it]
26%|██▋ | 414/1570 [29:49<54:16, 2.82s/it]
26%|██▋ | 415/1570 [29:52<53:43, 2.79s/it]
26%|██▋ | 416/1570 [29:54<53:41, 2.79s/it]
27%|██▋ | 417/1570 [29:57<53:24, 2.78s/it]
27%|██▋ | 418/1570 [30:01<57:48, 3.01s/it]
27%|██▋ | 419/1570 [30:04<1:00:56, 3.18s/it]
27%|██▋ | 420/1570 [30:08<1:03:15, 3.30s/it]
{'loss': '0.002673', 'grad_norm': '0.1551', 'learning_rate': '8.345e-05', 'epoch': '1.338'}
+
27%|██▋ | 420/1570 [30:08<1:03:15, 3.30s/it]
27%|██▋ | 421/1570 [30:12<1:04:53, 3.39s/it]
27%|██▋ | 422/1570 [30:14<1:01:11, 3.20s/it]
27%|██▋ | 423/1570 [30:17<58:43, 3.07s/it]
27%|██▋ | 424/1570 [30:20<57:05, 2.99s/it]
27%|██▋ | 425/1570 [30:23<56:00, 2.94s/it]
27%|██▋ | 426/1570 [30:25<55:01, 2.89s/it]
27%|██▋ | 427/1570 [30:28<54:32, 2.86s/it]
27%|██▋ | 428/1570 [30:31<53:56, 2.83s/it]
27%|██▋ | 429/1570 [30:34<53:39, 2.82s/it]
27%|██▋ | 430/1570 [30:37<53:33, 2.82s/it]
27%|██▋ | 431/1570 [30:39<53:18, 2.81s/it]
28%|██▊ | 432/1570 [30:42<53:14, 2.81s/it]
28%|██▊ | 433/1570 [30:45<53:10, 2.81s/it]
28%|██▊ | 434/1570 [30:48<52:46, 2.79s/it]
28%|██▊ | 435/1570 [30:51<52:49, 2.79s/it]
28%|██▊ | 436/1570 [30:53<52:40, 2.79s/it]
28%|██▊ | 437/1570 [30:56<52:50, 2.80s/it]
28%|██▊ | 438/1570 [30:59<52:57, 2.81s/it]
28%|██▊ | 439/1570 [31:02<52:51, 2.80s/it]
28%|██▊ | 440/1570 [31:05<52:56, 2.81s/it]
{'loss': '-0.001171', 'grad_norm': '0.7774', 'learning_rate': '8.194e-05', 'epoch': '1.402'}
+
28%|██▊ | 440/1570 [31:05<52:56, 2.81s/it]
28%|██▊ | 441/1570 [31:07<52:41, 2.80s/it]
28%|██▊ | 442/1570 [31:10<52:44, 2.80s/it]
28%|██▊ | 443/1570 [31:13<52:29, 2.79s/it]
28%|██▊ | 444/1570 [31:16<52:22, 2.79s/it]
28%|██▊ | 445/1570 [31:19<52:13, 2.79s/it]
28%|██▊ | 446/1570 [31:21<52:10, 2.78s/it]
28%|██▊ | 447/1570 [31:24<52:07, 2.78s/it]
29%|██▊ | 448/1570 [31:27<52:09, 2.79s/it]
29%|██▊ | 449/1570 [31:30<52:22, 2.80s/it]
29%|██▊ | 450/1570 [31:33<52:21, 2.81s/it]
29%|██▊ | 451/1570 [31:35<52:29, 2.81s/it]
29%|██▉ | 452/1570 [31:38<52:21, 2.81s/it]
29%|██▉ | 453/1570 [31:41<52:12, 2.80s/it]
29%|██▉ | 454/1570 [31:44<52:16, 2.81s/it]
29%|██▉ | 455/1570 [31:47<52:23, 2.82s/it]
29%|██▉ | 456/1570 [31:49<52:25, 2.82s/it]
29%|██▉ | 457/1570 [31:52<52:07, 2.81s/it]
29%|██▉ | 458/1570 [31:55<52:05, 2.81s/it]
29%|██▉ | 459/1570 [31:58<52:02, 2.81s/it]
29%|██▉ | 460/1570 [32:01<51:33, 2.79s/it]
{'loss': '-0.001906', 'grad_norm': '0.4202', 'learning_rate': '8.037e-05', 'epoch': '1.466'}
+
29%|██▉ | 460/1570 [32:01<51:33, 2.79s/it]
29%|██▉ | 461/1570 [32:03<51:31, 2.79s/it]
29%|██▉ | 462/1570 [32:06<51:28, 2.79s/it]
29%|██▉ | 463/1570 [32:09<51:47, 2.81s/it]
30%|██▉ | 464/1570 [32:12<51:48, 2.81s/it]
30%|██▉ | 465/1570 [32:15<51:42, 2.81s/it]
30%|██▉ | 466/1570 [32:17<51:22, 2.79s/it]
30%|██▉ | 467/1570 [32:20<51:14, 2.79s/it]
30%|██▉ | 468/1570 [32:23<51:13, 2.79s/it]
30%|██▉ | 469/1570 [32:26<51:13, 2.79s/it]
30%|██▉ | 470/1570 [32:29<51:05, 2.79s/it]
30%|███ | 471/1570 [32:31<50:54, 2.78s/it]
30%|███ | 472/1570 [32:34<50:45, 2.77s/it]
30%|███ | 473/1570 [32:37<50:55, 2.79s/it]
30%|███ | 474/1570 [32:40<50:45, 2.78s/it]
30%|███ | 475/1570 [32:42<50:44, 2.78s/it]
30%|███ | 476/1570 [32:45<50:47, 2.79s/it]
30%|███ | 477/1570 [32:48<50:38, 2.78s/it]
30%|███ | 478/1570 [32:51<50:39, 2.78s/it]
31%|███ | 479/1570 [32:54<50:38, 2.78s/it]
31%|███ | 480/1570 [32:56<50:42, 2.79s/it]
{'loss': '-0.004773', 'grad_norm': '1.054', 'learning_rate': '7.876e-05', 'epoch': '1.53'}
+
31%|███ | 480/1570 [32:56<50:42, 2.79s/it]
31%|███ | 481/1570 [32:59<50:47, 2.80s/it]
31%|███ | 482/1570 [33:02<50:36, 2.79s/it]
31%|███ | 483/1570 [33:05<50:38, 2.80s/it]
31%|███ | 484/1570 [33:08<50:34, 2.79s/it]
31%|███ | 485/1570 [33:10<50:30, 2.79s/it]
31%|███ | 486/1570 [33:13<50:36, 2.80s/it]
31%|███ | 487/1570 [33:16<50:32, 2.80s/it]
31%|███ | 488/1570 [33:19<50:33, 2.80s/it]
31%|███ | 489/1570 [33:22<50:25, 2.80s/it]
31%|███ | 490/1570 [33:24<50:30, 2.81s/it]
31%|███▏ | 491/1570 [33:27<50:34, 2.81s/it]
31%|███▏ | 492/1570 [33:30<50:24, 2.81s/it]
31%|███▏ | 493/1570 [33:33<50:20, 2.80s/it]
31%|███▏ | 494/1570 [33:36<49:54, 2.78s/it]
32%|███▏ | 495/1570 [33:38<49:50, 2.78s/it]
32%|███▏ | 496/1570 [33:41<49:44, 2.78s/it]
32%|███▏ | 497/1570 [33:44<49:44, 2.78s/it]
32%|███▏ | 498/1570 [33:47<49:51, 2.79s/it]
32%|███▏ | 499/1570 [33:49<49:45, 2.79s/it]
32%|███▏ | 500/1570 [33:52<49:31, 2.78s/it]
{'loss': '-0.0002439', 'grad_norm': '0.4706', 'learning_rate': '7.71e-05', 'epoch': '1.594'}
+
32%|███▏ | 500/1570 [33:52<49:31, 2.78s/it]
32%|███▏ | 501/1570 [33:55<49:31, 2.78s/it]
32%|███▏ | 502/1570 [33:58<49:39, 2.79s/it]
32%|███▏ | 503/1570 [34:01<49:34, 2.79s/it]
32%|███▏ | 504/1570 [34:03<49:48, 2.80s/it]
32%|███▏ | 505/1570 [34:06<49:47, 2.80s/it]
32%|███▏ | 506/1570 [34:09<49:32, 2.79s/it]
32%|███▏ | 507/1570 [34:12<49:27, 2.79s/it]
32%|███▏ | 508/1570 [34:15<49:29, 2.80s/it]
32%|███▏ | 509/1570 [34:17<49:22, 2.79s/it]
32%|███▏ | 510/1570 [34:20<49:26, 2.80s/it]
33%|███▎ | 511/1570 [34:23<49:17, 2.79s/it]
33%|███▎ | 512/1570 [34:26<49:20, 2.80s/it]
33%|███▎ | 513/1570 [34:29<49:23, 2.80s/it]
33%|███▎ | 514/1570 [34:31<49:22, 2.81s/it]
33%|███▎ | 515/1570 [34:34<49:17, 2.80s/it]
33%|███▎ | 516/1570 [34:37<49:00, 2.79s/it]
33%|███▎ | 517/1570 [34:40<48:49, 2.78s/it]
33%|███▎ | 518/1570 [34:43<48:44, 2.78s/it]
33%|███▎ | 519/1570 [34:45<48:59, 2.80s/it]
33%|███▎ | 520/1570 [34:48<48:55, 2.80s/it]
{'loss': '-0.004392', 'grad_norm': '0.2209', 'learning_rate': '7.54e-05', 'epoch': '1.657'}
+
33%|███▎ | 520/1570 [34:48<48:55, 2.80s/it]
33%|███▎ | 521/1570 [34:51<48:50, 2.79s/it]
33%|███▎ | 522/1570 [34:54<48:48, 2.79s/it]
33%|███▎ | 523/1570 [34:57<48:39, 2.79s/it]
33%|███▎ | 524/1570 [34:59<48:45, 2.80s/it]
33%|███▎ | 525/1570 [35:02<48:46, 2.80s/it]
34%|███▎ | 526/1570 [35:05<48:40, 2.80s/it]
34%|███▎ | 527/1570 [35:08<48:37, 2.80s/it]
34%|███▎ | 528/1570 [35:10<48:20, 2.78s/it]
34%|███▎ | 529/1570 [35:13<48:15, 2.78s/it]
34%|███▍ | 530/1570 [35:16<48:15, 2.78s/it]
34%|███▍ | 531/1570 [35:19<48:29, 2.80s/it]
34%|███▍ | 532/1570 [35:22<48:27, 2.80s/it]
34%|███▍ | 533/1570 [35:25<48:32, 2.81s/it]
34%|███▍ | 534/1570 [35:27<48:34, 2.81s/it]
34%|███▍ | 535/1570 [35:30<48:27, 2.81s/it]
34%|███▍ | 536/1570 [35:33<48:30, 2.82s/it]
34%|███▍ | 537/1570 [35:36<48:10, 2.80s/it]
34%|███▍ | 538/1570 [35:39<48:17, 2.81s/it]
34%|███▍ | 539/1570 [35:41<47:59, 2.79s/it]
34%|███▍ | 540/1570 [35:44<47:57, 2.79s/it]
{'loss': '-0.007542', 'grad_norm': '0.5973', 'learning_rate': '7.366e-05', 'epoch': '1.721'}
+
34%|███▍ | 540/1570 [35:44<47:57, 2.79s/it]
34%|███▍ | 541/1570 [35:47<48:02, 2.80s/it]
35%|███▍ | 542/1570 [35:50<47:57, 2.80s/it]
35%|███▍ | 543/1570 [35:53<47:52, 2.80s/it]
35%|███▍ | 544/1570 [35:55<47:46, 2.79s/it]
35%|███▍ | 545/1570 [35:58<47:47, 2.80s/it]
35%|███▍ | 546/1570 [36:01<47:43, 2.80s/it]
35%|███▍ | 547/1570 [36:04<47:40, 2.80s/it]
35%|███▍ | 548/1570 [36:06<47:31, 2.79s/it]
35%|███▍ | 549/1570 [36:09<47:15, 2.78s/it]
35%|███▌ | 550/1570 [36:12<47:11, 2.78s/it]
35%|███▌ | 551/1570 [36:15<47:36, 2.80s/it]
35%|███▌ | 552/1570 [36:18<47:27, 2.80s/it]
35%|███▌ | 553/1570 [36:20<47:20, 2.79s/it]
35%|███▌ | 554/1570 [36:23<47:25, 2.80s/it]
35%|███▌ | 555/1570 [36:26<47:25, 2.80s/it]
35%|███▌ | 556/1570 [36:29<47:26, 2.81s/it]
35%|███▌ | 557/1570 [36:32<47:28, 2.81s/it]
36%|███▌ | 558/1570 [36:34<47:26, 2.81s/it]
36%|███▌ | 559/1570 [36:37<47:16, 2.81s/it]
36%|███▌ | 560/1570 [36:40<47:11, 2.80s/it]
{'loss': '-0.004147', 'grad_norm': '0.194', 'learning_rate': '7.188e-05', 'epoch': '1.785'}
+
36%|███▌ | 560/1570 [36:40<47:11, 2.80s/it]
36%|███▌ | 561/1570 [36:43<47:10, 2.81s/it]
36%|███▌ | 562/1570 [36:46<47:04, 2.80s/it]
36%|███▌ | 563/1570 [36:48<46:53, 2.79s/it]
36%|███▌ | 564/1570 [36:51<46:52, 2.80s/it]
36%|███▌ | 565/1570 [36:54<47:05, 2.81s/it]
36%|███▌ | 566/1570 [36:57<47:02, 2.81s/it]
36%|███▌ | 567/1570 [37:00<46:48, 2.80s/it]
36%|███▌ | 568/1570 [37:02<46:39, 2.79s/it]
36%|███▌ | 569/1570 [37:05<46:38, 2.80s/it]
36%|███▋ | 570/1570 [37:08<46:30, 2.79s/it]
36%|███▋ | 571/1570 [37:11<46:21, 2.78s/it]
36%|███▋ | 572/1570 [37:14<46:14, 2.78s/it]
36%|███▋ | 573/1570 [37:16<46:26, 2.80s/it]
37%|███▋ | 574/1570 [37:19<46:17, 2.79s/it]
37%|███▋ | 575/1570 [37:22<46:18, 2.79s/it]
37%|███▋ | 576/1570 [37:25<45:58, 2.77s/it]
37%|███▋ | 577/1570 [37:28<46:08, 2.79s/it]
37%|███▋ | 578/1570 [37:30<46:04, 2.79s/it]
37%|███▋ | 579/1570 [37:33<46:05, 2.79s/it]
37%|███▋ | 580/1570 [37:36<46:21, 2.81s/it]
{'loss': '-0.001953', 'grad_norm': '1.247', 'learning_rate': '7.006e-05', 'epoch': '1.849'}
+
37%|███▋ | 580/1570 [37:36<46:21, 2.81s/it]
37%|███▋ | 581/1570 [37:39<46:02, 2.79s/it]
37%|███▋ | 582/1570 [37:42<45:58, 2.79s/it]
37%|███▋ | 583/1570 [37:44<45:59, 2.80s/it]
37%|███▋ | 584/1570 [37:47<46:05, 2.80s/it]
37%|███▋ | 585/1570 [37:50<46:07, 2.81s/it]
37%|███▋ | 586/1570 [37:53<46:03, 2.81s/it]
37%|███▋ | 587/1570 [37:56<45:49, 2.80s/it]
37%|███▋ | 588/1570 [37:58<45:31, 2.78s/it]
38%|███▊ | 589/1570 [38:01<45:44, 2.80s/it]
38%|███▊ | 590/1570 [38:04<45:47, 2.80s/it]
38%|███▊ | 591/1570 [38:07<45:48, 2.81s/it]
38%|███▊ | 592/1570 [38:10<45:49, 2.81s/it]
38%|███▊ | 593/1570 [38:12<45:37, 2.80s/it]
38%|███▊ | 594/1570 [38:15<45:25, 2.79s/it]
38%|███▊ | 595/1570 [38:18<45:20, 2.79s/it]
38%|███▊ | 596/1570 [38:21<45:31, 2.80s/it]
38%|███▊ | 597/1570 [38:24<45:38, 2.81s/it]
38%|███▊ | 598/1570 [38:26<45:35, 2.81s/it]
38%|███▊ | 599/1570 [38:29<45:40, 2.82s/it]
38%|███▊ | 600/1570 [38:32<45:20, 2.80s/it]
{'loss': '-0.008291', 'grad_norm': '0.1432', 'learning_rate': '6.821e-05', 'epoch': '1.913'}
+
38%|███▊ | 600/1570 [38:32<45:20, 2.80s/it]
38%|███▊ | 601/1570 [38:35<45:17, 2.80s/it]
38%|███▊ | 602/1570 [38:38<45:11, 2.80s/it]
38%|███▊ | 603/1570 [38:40<45:09, 2.80s/it]
38%|███▊ | 604/1570 [38:43<45:13, 2.81s/it]
39%|███▊ | 605/1570 [38:46<45:05, 2.80s/it]
39%|███▊ | 606/1570 [38:49<45:08, 2.81s/it]
39%|███▊ | 607/1570 [38:52<45:04, 2.81s/it]
39%|███▊ | 608/1570 [38:54<45:02, 2.81s/it]
39%|███▉ | 609/1570 [38:57<44:42, 2.79s/it]
39%|███▉ | 610/1570 [39:00<44:38, 2.79s/it]
39%|███▉ | 611/1570 [39:03<44:39, 2.79s/it]
39%|███▉ | 612/1570 [39:06<44:40, 2.80s/it]
39%|███▉ | 613/1570 [39:08<44:32, 2.79s/it]
39%|███▉ | 614/1570 [39:11<44:26, 2.79s/it]
39%|███▉ | 615/1570 [39:14<44:30, 2.80s/it]
39%|███▉ | 616/1570 [39:17<44:33, 2.80s/it]
39%|███▉ | 617/1570 [39:20<44:38, 2.81s/it]
39%|███▉ | 618/1570 [39:22<44:32, 2.81s/it]
39%|███▉ | 619/1570 [39:25<44:41, 2.82s/it]
39%|███▉ | 620/1570 [39:28<44:42, 2.82s/it]
{'loss': '-0.006107', 'grad_norm': '0.4658', 'learning_rate': '6.634e-05', 'epoch': '1.976'}
+
39%|███▉ | 620/1570 [39:28<44:42, 2.82s/it]
40%|███▉ | 621/1570 [39:31<44:46, 2.83s/it]
40%|███▉ | 622/1570 [39:34<44:18, 2.80s/it]
40%|███▉ | 623/1570 [39:36<44:06, 2.79s/it]
40%|███▉ | 624/1570 [39:39<44:07, 2.80s/it]
40%|███▉ | 625/1570 [39:42<44:17, 2.81s/it]
40%|███▉ | 626/1570 [39:45<44:04, 2.80s/it]
40%|███▉ | 627/1570 [39:48<44:01, 2.80s/it]
40%|████ | 628/1570 [39:49<36:00, 2.29s/it][INFO|trainer.py:2668] 2026-08-22 20:15:15,262 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 20:15:15,262 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 20:15:15,262 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/csd/checkpoint-628
+[INFO|configuration_utils.py:780] 2026-08-22 20:25:17,431 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 20:25:17,436 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 20:25:17,471 >> chat template saved in results/qwen3/csd/checkpoint-628/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 20:25:17,472 >> tokenizer config file saved in results/qwen3/csd/checkpoint-628/tokenizer_config.json
+
40%|████ | 629/1570 [49:55<47:55:26, 183.34s/it]
40%|████ | 630/1570 [49:57<33:43:50, 129.18s/it]
40%|████ | 631/1570 [50:00<23:48:19, 91.27s/it]
40%|████ | 632/1570 [50:03<16:51:46, 64.72s/it]
40%|████ | 633/1570 [50:06<12:00:41, 46.15s/it]
40%|████ | 634/1570 [50:09<8:37:00, 33.14s/it]
40%|████ | 635/1570 [50:11<6:14:55, 24.06s/it]
41%|████ | 636/1570 [50:14<4:35:09, 17.68s/it]
41%|████ | 637/1570 [50:17<3:25:20, 13.20s/it]
41%|████ | 638/1570 [50:20<2:36:32, 10.08s/it]
41%|████ | 639/1570 [50:23<2:02:20, 7.88s/it]
41%|████ | 640/1570 [50:25<1:38:44, 6.37s/it]
{'loss': '-0.005953', 'grad_norm': '0.2209', 'learning_rate': '6.444e-05', 'epoch': '2.038'}
+
41%|████ | 640/1570 [50:25<1:38:44, 6.37s/it]
41%|████ | 641/1570 [50:28<1:22:00, 5.30s/it]
41%|████ | 642/1570 [50:31<1:10:22, 4.55s/it]
41%|████ | 643/1570 [50:34<1:02:18, 4.03s/it]
41%|████ | 644/1570 [50:37<56:35, 3.67s/it]
41%|████ | 645/1570 [50:40<52:42, 3.42s/it]
41%|████ | 646/1570 [50:42<49:52, 3.24s/it]
41%|████ | 647/1570 [50:45<47:49, 3.11s/it]
41%|████▏ | 648/1570 [50:48<46:16, 3.01s/it]
41%|████▏ | 649/1570 [50:51<45:08, 2.94s/it]
41%|████▏ | 650/1570 [50:53<44:17, 2.89s/it]
41%|████▏ | 651/1570 [50:56<43:58, 2.87s/it]
42%|████▏ | 652/1570 [50:59<43:27, 2.84s/it]
42%|████▏ | 653/1570 [51:02<43:28, 2.84s/it]
42%|████▏ | 654/1570 [51:05<42:56, 2.81s/it]
42%|████▏ | 655/1570 [51:07<42:49, 2.81s/it]
42%|████▏ | 656/1570 [51:10<42:37, 2.80s/it]
42%|████▏ | 657/1570 [51:13<42:27, 2.79s/it]
42%|████▏ | 658/1570 [51:16<42:20, 2.79s/it]
42%|████▏ | 659/1570 [51:19<42:15, 2.78s/it]
42%|████▏ | 660/1570 [51:21<42:17, 2.79s/it]
{'loss': '-0.002542', 'grad_norm': '0.5406', 'learning_rate': '6.251e-05', 'epoch': '2.102'}
+
42%|████▏ | 660/1570 [51:21<42:17, 2.79s/it]
42%|████▏ | 661/1570 [51:24<42:21, 2.80s/it]
42%|████▏ | 662/1570 [51:27<42:21, 2.80s/it]
42%|████▏ | 663/1570 [51:30<42:16, 2.80s/it]
42%|████▏ | 664/1570 [51:33<42:05, 2.79s/it]
42%|████▏ | 665/1570 [51:35<41:56, 2.78s/it]
42%|████▏ | 666/1570 [51:38<41:55, 2.78s/it]
42%|████▏ | 667/1570 [51:41<42:02, 2.79s/it]
43%|████▎ | 668/1570 [51:44<42:03, 2.80s/it]
43%|████▎ | 669/1570 [51:46<41:57, 2.79s/it]
43%|████▎ | 670/1570 [51:49<41:50, 2.79s/it]
43%|████▎ | 671/1570 [51:52<41:54, 2.80s/it]
43%|████▎ | 672/1570 [51:55<41:51, 2.80s/it]
43%|████▎ | 673/1570 [51:58<41:36, 2.78s/it]
43%|████▎ | 674/1570 [52:00<41:52, 2.80s/it]
43%|████▎ | 675/1570 [52:03<41:53, 2.81s/it]
43%|████▎ | 676/1570 [52:06<41:42, 2.80s/it]
43%|████▎ | 677/1570 [52:09<41:37, 2.80s/it]
43%|████▎ | 678/1570 [52:12<41:30, 2.79s/it]
43%|████▎ | 679/1570 [52:14<41:32, 2.80s/it]
43%|████▎ | 680/1570 [52:17<41:35, 2.80s/it]
{'loss': '-0.004494', 'grad_norm': '0.09373', 'learning_rate': '6.057e-05', 'epoch': '2.166'}
+
43%|████▎ | 680/1570 [52:17<41:35, 2.80s/it]
43%|████▎ | 681/1570 [52:20<41:34, 2.81s/it]
43%|████▎ | 682/1570 [52:23<41:26, 2.80s/it]
44%|████▎ | 683/1570 [52:26<41:18, 2.79s/it]
44%|████▎ | 684/1570 [52:28<41:20, 2.80s/it]
44%|████▎ | 685/1570 [52:31<41:22, 2.81s/it]
44%|████▎ | 686/1570 [52:34<41:27, 2.81s/it]
44%|████▍ | 687/1570 [52:37<41:16, 2.81s/it]
44%|████▍ | 688/1570 [52:40<41:21, 2.81s/it]
44%|████▍ | 689/1570 [52:43<41:09, 2.80s/it]
44%|████▍ | 690/1570 [52:45<41:01, 2.80s/it]
44%|████▍ | 691/1570 [52:48<41:00, 2.80s/it]
44%|████▍ | 692/1570 [52:51<41:00, 2.80s/it]
44%|████▍ | 693/1570 [52:54<41:02, 2.81s/it]
44%|████▍ | 694/1570 [52:57<41:02, 2.81s/it]
44%|████▍ | 695/1570 [52:59<40:59, 2.81s/it]
44%|████▍ | 696/1570 [53:02<40:39, 2.79s/it]
44%|████▍ | 697/1570 [53:05<40:24, 2.78s/it]
44%|████▍ | 698/1570 [53:08<40:13, 2.77s/it]
45%|████▍ | 699/1570 [53:10<40:22, 2.78s/it]
45%|████▍ | 700/1570 [53:13<40:17, 2.78s/it]
{'loss': '-0.008207', 'grad_norm': '0.108', 'learning_rate': '5.86e-05', 'epoch': '2.23'}
+
45%|████▍ | 700/1570 [53:13<40:17, 2.78s/it]
45%|████▍ | 701/1570 [53:16<40:26, 2.79s/it]
45%|████▍ | 702/1570 [53:19<40:32, 2.80s/it]
45%|████▍ | 703/1570 [53:22<40:16, 2.79s/it]
45%|████▍ | 704/1570 [53:24<40:19, 2.79s/it]
45%|████▍ | 705/1570 [53:27<40:16, 2.79s/it]
45%|████▍ | 706/1570 [53:30<40:28, 2.81s/it]
45%|████▌ | 707/1570 [53:33<40:20, 2.80s/it]
45%|████▌ | 708/1570 [53:36<40:16, 2.80s/it]
45%|████▌ | 709/1570 [53:38<40:23, 2.82s/it]
45%|████▌ | 710/1570 [53:41<40:09, 2.80s/it]
45%|████▌ | 711/1570 [53:44<40:10, 2.81s/it]
45%|████▌ | 712/1570 [53:47<40:06, 2.80s/it]
45%|████▌ | 713/1570 [53:50<40:06, 2.81s/it]
45%|████▌ | 714/1570 [53:52<40:05, 2.81s/it]
46%|████▌ | 715/1570 [53:55<40:00, 2.81s/it]
46%|████▌ | 716/1570 [53:58<40:02, 2.81s/it]
46%|████▌ | 717/1570 [54:01<39:58, 2.81s/it]
46%|████▌ | 718/1570 [54:04<39:58, 2.81s/it]
46%|████▌ | 719/1570 [54:07<39:58, 2.82s/it]
46%|████▌ | 720/1570 [54:09<39:47, 2.81s/it]
{'loss': '-0.009636', 'grad_norm': '0.05399', 'learning_rate': '5.663e-05', 'epoch': '2.294'}
+
46%|████▌ | 720/1570 [54:09<39:47, 2.81s/it]
46%|████▌ | 721/1570 [54:12<39:29, 2.79s/it]
46%|████▌ | 722/1570 [54:15<39:28, 2.79s/it]
46%|████▌ | 723/1570 [54:18<39:23, 2.79s/it]
46%|████▌ | 724/1570 [54:20<39:24, 2.79s/it]
46%|████▌ | 725/1570 [54:23<39:35, 2.81s/it]
46%|████▌ | 726/1570 [54:26<39:25, 2.80s/it]
46%|████▋ | 727/1570 [54:29<39:20, 2.80s/it]
46%|████▋ | 728/1570 [54:32<39:19, 2.80s/it]
46%|████▋ | 729/1570 [54:34<39:08, 2.79s/it]
46%|████▋ | 730/1570 [54:37<38:51, 2.78s/it]
47%|████▋ | 731/1570 [54:40<38:39, 2.76s/it]
47%|████▋ | 732/1570 [54:43<38:37, 2.77s/it]
47%|████▋ | 733/1570 [54:46<38:36, 2.77s/it]
47%|████▋ | 734/1570 [54:48<38:33, 2.77s/it]
47%|████▋ | 735/1570 [54:51<38:24, 2.76s/it]
47%|████▋ | 736/1570 [54:54<38:25, 2.76s/it]
47%|████▋ | 737/1570 [54:57<38:16, 2.76s/it]
47%|████▋ | 738/1570 [54:59<38:21, 2.77s/it]
47%|████▋ | 739/1570 [55:02<38:11, 2.76s/it]
47%|████▋ | 740/1570 [55:05<38:08, 2.76s/it]
{'loss': '-0.01009', 'grad_norm': '0.2049', 'learning_rate': '5.464e-05', 'epoch': '2.357'}
+
47%|████▋ | 740/1570 [55:05<38:08, 2.76s/it]
47%|████▋ | 741/1570 [55:08<38:16, 2.77s/it]
47%|████▋ | 742/1570 [55:10<38:21, 2.78s/it]
47%|████▋ | 743/1570 [55:13<38:21, 2.78s/it]
47%|████▋ | 744/1570 [55:16<38:24, 2.79s/it]
47%|████▋ | 745/1570 [55:19<38:19, 2.79s/it]
48%|████▊ | 746/1570 [55:22<38:05, 2.77s/it]
48%|████▊ | 747/1570 [55:24<38:05, 2.78s/it]
48%|████▊ | 748/1570 [55:27<38:05, 2.78s/it]
48%|████▊ | 749/1570 [55:30<38:09, 2.79s/it]
48%|████▊ | 750/1570 [55:33<37:51, 2.77s/it]
48%|████▊ | 751/1570 [55:35<37:57, 2.78s/it]
48%|████▊ | 752/1570 [55:38<37:59, 2.79s/it]
48%|████▊ | 753/1570 [55:41<37:56, 2.79s/it]
48%|████▊ | 754/1570 [55:44<37:57, 2.79s/it]
48%|████▊ | 755/1570 [55:47<37:50, 2.79s/it]
48%|████▊ | 756/1570 [55:49<37:42, 2.78s/it]
48%|████▊ | 757/1570 [55:52<37:33, 2.77s/it]
48%|████▊ | 758/1570 [55:55<37:40, 2.78s/it]
48%|████▊ | 759/1570 [55:58<37:43, 2.79s/it]
48%|████▊ | 760/1570 [56:01<37:44, 2.80s/it]
{'loss': '-0.008542', 'grad_norm': '0.08765', 'learning_rate': '5.265e-05', 'epoch': '2.421'}
+
48%|████▊ | 760/1570 [56:01<37:44, 2.80s/it]
48%|████▊ | 761/1570 [56:03<37:45, 2.80s/it]
49%|████▊ | 762/1570 [56:06<37:47, 2.81s/it]
49%|████▊ | 763/1570 [56:09<37:34, 2.79s/it]
49%|████▊ | 764/1570 [56:12<37:33, 2.80s/it]
49%|████▊ | 765/1570 [56:15<37:25, 2.79s/it]
49%|████▉ | 766/1570 [56:17<37:27, 2.80s/it]
49%|████▉ | 767/1570 [56:20<37:26, 2.80s/it]
49%|████▉ | 768/1570 [56:23<37:27, 2.80s/it]
49%|████▉ | 769/1570 [56:26<37:12, 2.79s/it]
49%|████▉ | 770/1570 [56:29<37:19, 2.80s/it]
49%|████▉ | 771/1570 [56:31<37:17, 2.80s/it]
49%|████▉ | 772/1570 [56:34<37:11, 2.80s/it]
49%|████▉ | 773/1570 [56:37<37:05, 2.79s/it]
49%|████▉ | 774/1570 [56:40<36:58, 2.79s/it]
49%|████▉ | 775/1570 [56:42<36:56, 2.79s/it]
49%|████▉ | 776/1570 [56:45<36:53, 2.79s/it]
49%|████▉ | 777/1570 [56:48<36:52, 2.79s/it]
50%|████▉ | 778/1570 [56:51<36:49, 2.79s/it]
50%|████▉ | 779/1570 [56:54<36:43, 2.79s/it]
50%|████▉ | 780/1570 [56:56<36:29, 2.77s/it]
{'loss': '-0.01166', 'grad_norm': '0.2229', 'learning_rate': '5.065e-05', 'epoch': '2.485'}
+
50%|████▉ | 780/1570 [56:56<36:29, 2.77s/it]
50%|████▉ | 781/1570 [56:59<36:30, 2.78s/it]
50%|████▉ | 782/1570 [57:02<36:37, 2.79s/it]
50%|████▉ | 783/1570 [57:05<36:39, 2.79s/it]
50%|████▉ | 784/1570 [57:08<36:38, 2.80s/it]
50%|█████ | 785/1570 [57:10<36:27, 2.79s/it]
50%|█████ | 786/1570 [57:13<36:23, 2.79s/it]
50%|█████ | 787/1570 [57:16<36:22, 2.79s/it]
50%|█████ | 788/1570 [57:19<36:11, 2.78s/it]
50%|█████ | 789/1570 [57:21<36:11, 2.78s/it]
50%|█████ | 790/1570 [57:24<36:16, 2.79s/it]
50%|█████ | 791/1570 [57:27<36:12, 2.79s/it]
50%|█████ | 792/1570 [57:30<36:13, 2.79s/it]
51%|█████ | 793/1570 [57:33<36:08, 2.79s/it]
51%|█████ | 794/1570 [57:35<35:58, 2.78s/it]
51%|█████ | 795/1570 [57:38<35:54, 2.78s/it]
51%|█████ | 796/1570 [57:41<35:57, 2.79s/it]
51%|█████ | 797/1570 [57:44<35:58, 2.79s/it]
51%|█████ | 798/1570 [57:47<35:55, 2.79s/it]
51%|█████ | 799/1570 [57:49<35:52, 2.79s/it]
51%|█████ | 800/1570 [57:52<35:59, 2.80s/it]
{'loss': '-0.009308', 'grad_norm': '0.1851', 'learning_rate': '4.865e-05', 'epoch': '2.549'}
+
51%|█████ | 800/1570 [57:52<35:59, 2.80s/it]
51%|█████ | 801/1570 [57:55<35:59, 2.81s/it]
51%|█████ | 802/1570 [57:58<35:53, 2.80s/it]
51%|█████ | 803/1570 [58:01<35:53, 2.81s/it]
51%|█████ | 804/1570 [58:03<35:46, 2.80s/it]
51%|█████▏ | 805/1570 [58:06<35:35, 2.79s/it]
51%|█████▏ | 806/1570 [58:09<35:32, 2.79s/it]
51%|█████▏ | 807/1570 [58:12<35:21, 2.78s/it]
51%|█████▏ | 808/1570 [58:14<35:14, 2.78s/it]
52%|█████▏ | 809/1570 [58:17<35:20, 2.79s/it]
52%|█████▏ | 810/1570 [58:20<35:18, 2.79s/it]
52%|█████▏ | 811/1570 [58:23<35:16, 2.79s/it]
52%|█████▏ | 812/1570 [58:26<35:21, 2.80s/it]
52%|█████▏ | 813/1570 [58:29<35:22, 2.80s/it]
52%|█████▏ | 814/1570 [58:31<35:19, 2.80s/it]
52%|█████▏ | 815/1570 [58:34<35:12, 2.80s/it]
52%|█████▏ | 816/1570 [58:37<35:25, 2.82s/it]
52%|█████▏ | 817/1570 [58:40<35:13, 2.81s/it]
52%|█████▏ | 818/1570 [58:43<35:04, 2.80s/it]
52%|█████▏ | 819/1570 [58:45<35:06, 2.81s/it]
52%|█████▏ | 820/1570 [58:48<35:05, 2.81s/it]
{'loss': '-0.01487', 'grad_norm': '0.3499', 'learning_rate': '4.665e-05', 'epoch': '2.613'}
+
52%|█████▏ | 820/1570 [58:48<35:05, 2.81s/it]
52%|█████▏ | 821/1570 [58:51<35:03, 2.81s/it]
52%|█████▏ | 822/1570 [58:54<34:48, 2.79s/it]
52%|█████▏ | 823/1570 [58:57<34:41, 2.79s/it]
52%|█████▏ | 824/1570 [58:59<34:39, 2.79s/it]
53%|█████▎ | 825/1570 [59:02<34:28, 2.78s/it]
53%|█████▎ | 826/1570 [59:05<34:33, 2.79s/it]
53%|█████▎ | 827/1570 [59:08<34:37, 2.80s/it]
53%|█████▎ | 828/1570 [59:11<34:48, 2.81s/it]
53%|█████▎ | 829/1570 [59:13<34:39, 2.81s/it]
53%|█████▎ | 830/1570 [59:16<34:40, 2.81s/it]
53%|█████▎ | 831/1570 [59:19<34:37, 2.81s/it]
53%|█████▎ | 832/1570 [59:22<34:24, 2.80s/it]
53%|█████▎ | 833/1570 [59:25<34:32, 2.81s/it]
53%|█████▎ | 834/1570 [59:27<34:25, 2.81s/it]
53%|█████▎ | 835/1570 [59:30<34:26, 2.81s/it]
53%|█████▎ | 836/1570 [59:33<34:18, 2.80s/it]
53%|█████▎ | 837/1570 [59:36<34:08, 2.79s/it]
53%|█████▎ | 838/1570 [59:39<33:57, 2.78s/it]
53%|█████▎ | 839/1570 [59:41<33:50, 2.78s/it]
54%|█████▎ | 840/1570 [59:44<33:36, 2.76s/it]
{'loss': '-0.02025', 'grad_norm': '0.1616', 'learning_rate': '4.466e-05', 'epoch': '2.677'}
+
54%|█████▎ | 840/1570 [59:44<33:36, 2.76s/it]
54%|█████▎ | 841/1570 [59:47<33:29, 2.76s/it]
54%|█████▎ | 842/1570 [59:50<33:27, 2.76s/it]
54%|█████▎ | 843/1570 [59:52<33:23, 2.76s/it]
54%|█████▍ | 844/1570 [59:55<33:26, 2.76s/it]
54%|█████▍ | 845/1570 [59:58<33:27, 2.77s/it]
54%|█████▍ | 846/1570 [1:00:01<33:36, 2.79s/it]
54%|█████▍ | 847/1570 [1:00:03<33:25, 2.77s/it]
54%|█████▍ | 848/1570 [1:00:06<33:23, 2.78s/it]
54%|█████▍ | 849/1570 [1:00:09<33:20, 2.78s/it]
54%|█████▍ | 850/1570 [1:00:12<33:24, 2.78s/it]
54%|█████▍ | 851/1570 [1:00:15<33:20, 2.78s/it]
54%|█████▍ | 852/1570 [1:00:17<33:16, 2.78s/it]
54%|█████▍ | 853/1570 [1:00:20<33:14, 2.78s/it]
54%|█████▍ | 854/1570 [1:00:23<33:10, 2.78s/it]
54%|█████▍ | 855/1570 [1:00:26<33:11, 2.78s/it]
55%|█████▍ | 856/1570 [1:00:28<33:02, 2.78s/it]
55%|█████▍ | 857/1570 [1:00:31<33:04, 2.78s/it]
55%|█████▍ | 858/1570 [1:00:34<33:02, 2.78s/it]
55%|█████▍ | 859/1570 [1:00:37<32:57, 2.78s/it]
55%|█████▍ | 860/1570 [1:00:40<33:08, 2.80s/it]
{'loss': '-0.01862', 'grad_norm': '0.1605', 'learning_rate': '4.268e-05', 'epoch': '2.74'}
+
55%|█████▍ | 860/1570 [1:00:40<33:08, 2.80s/it]
55%|█████▍ | 861/1570 [1:00:42<32:55, 2.79s/it]
55%|█████▍ | 862/1570 [1:00:45<32:54, 2.79s/it]
55%|█████▍ | 863/1570 [1:00:48<32:48, 2.78s/it]
55%|█████▌ | 864/1570 [1:00:51<32:43, 2.78s/it]
55%|█████▌ | 865/1570 [1:00:54<32:44, 2.79s/it]
55%|█████▌ | 866/1570 [1:00:56<32:33, 2.78s/it]
55%|█████▌ | 867/1570 [1:00:59<32:32, 2.78s/it]
55%|█████▌ | 868/1570 [1:01:02<32:23, 2.77s/it]
55%|█████▌ | 869/1570 [1:01:05<32:30, 2.78s/it]
55%|█████▌ | 870/1570 [1:01:07<32:23, 2.78s/it]
55%|█████▌ | 871/1570 [1:01:10<32:26, 2.78s/it]
56%|█████▌ | 872/1570 [1:01:13<32:22, 2.78s/it]
56%|█████▌ | 873/1570 [1:01:16<32:21, 2.78s/it]
56%|█████▌ | 874/1570 [1:01:19<32:18, 2.78s/it]
56%|█████▌ | 875/1570 [1:01:21<32:19, 2.79s/it]
56%|█████▌ | 876/1570 [1:01:24<32:11, 2.78s/it]
56%|█████▌ | 877/1570 [1:01:27<32:09, 2.78s/it]
56%|█████▌ | 878/1570 [1:01:30<32:07, 2.79s/it]
56%|█████▌ | 879/1570 [1:01:33<32:13, 2.80s/it]
56%|█████▌ | 880/1570 [1:01:35<32:10, 2.80s/it]
{'loss': '-0.02096', 'grad_norm': '0.1714', 'learning_rate': '4.071e-05', 'epoch': '2.804'}
+
56%|█████▌ | 880/1570 [1:01:35<32:10, 2.80s/it]
56%|█████▌ | 881/1570 [1:01:38<32:02, 2.79s/it]
56%|█████▌ | 882/1570 [1:01:41<31:56, 2.79s/it]
56%|█████▌ | 883/1570 [1:01:44<31:54, 2.79s/it]
56%|█████▋ | 884/1570 [1:01:46<31:46, 2.78s/it]
56%|█████▋ | 885/1570 [1:01:49<31:29, 2.76s/it]
56%|█████▋ | 886/1570 [1:01:52<31:31, 2.77s/it]
56%|█████▋ | 887/1570 [1:01:55<31:33, 2.77s/it]
57%|█████▋ | 888/1570 [1:01:57<31:40, 2.79s/it]
57%|█████▋ | 889/1570 [1:02:00<31:37, 2.79s/it]
57%|█████▋ | 890/1570 [1:02:03<31:26, 2.77s/it]
57%|█████▋ | 891/1570 [1:02:06<31:25, 2.78s/it]
57%|█████▋ | 892/1570 [1:02:09<31:25, 2.78s/it]
57%|█████▋ | 893/1570 [1:02:11<31:20, 2.78s/it]
57%|█████▋ | 894/1570 [1:02:14<31:28, 2.79s/it]
57%|█████▋ | 895/1570 [1:02:17<31:22, 2.79s/it]
57%|█████▋ | 896/1570 [1:02:20<31:14, 2.78s/it]
57%|█████▋ | 897/1570 [1:02:23<31:19, 2.79s/it]
57%|█████▋ | 898/1570 [1:02:25<31:27, 2.81s/it]
57%|█████▋ | 899/1570 [1:02:28<31:13, 2.79s/it]
57%|█████▋ | 900/1570 [1:02:31<31:12, 2.79s/it]
{'loss': '-0.02042', 'grad_norm': '0.2487', 'learning_rate': '3.875e-05', 'epoch': '2.868'}
+
57%|█████▋ | 900/1570 [1:02:31<31:12, 2.79s/it]
57%|█████▋ | 901/1570 [1:02:34<31:07, 2.79s/it]
57%|█████▋ | 902/1570 [1:02:37<31:08, 2.80s/it]
58%|█████▊ | 903/1570 [1:02:39<31:03, 2.79s/it]
58%|█████▊ | 904/1570 [1:02:42<30:59, 2.79s/it]
58%|█████▊ | 905/1570 [1:02:45<31:02, 2.80s/it]
58%|█████▊ | 906/1570 [1:02:48<30:50, 2.79s/it]
58%|█████▊ | 907/1570 [1:02:51<30:50, 2.79s/it]
58%|█████▊ | 908/1570 [1:02:53<30:47, 2.79s/it]
58%|█████▊ | 909/1570 [1:02:56<30:47, 2.79s/it]
58%|█████▊ | 910/1570 [1:02:59<30:46, 2.80s/it]
58%|█████▊ | 911/1570 [1:03:02<30:41, 2.80s/it]
58%|█████▊ | 912/1570 [1:03:05<30:43, 2.80s/it]
58%|█████▊ | 913/1570 [1:03:07<30:36, 2.79s/it]
58%|█████▊ | 914/1570 [1:03:10<30:34, 2.80s/it]
58%|█████▊ | 915/1570 [1:03:13<30:28, 2.79s/it]
58%|█████▊ | 916/1570 [1:03:16<30:25, 2.79s/it]
58%|█████▊ | 917/1570 [1:03:18<30:12, 2.78s/it]
58%|█████▊ | 918/1570 [1:03:21<30:13, 2.78s/it]
59%|█████▊ | 919/1570 [1:03:24<30:09, 2.78s/it]
59%|█████▊ | 920/1570 [1:03:27<30:15, 2.79s/it]
{'loss': '-0.01855', 'grad_norm': '0.1232', 'learning_rate': '3.682e-05', 'epoch': '2.932'}
+
59%|█████▊ | 920/1570 [1:03:27<30:15, 2.79s/it]
59%|█████▊ | 921/1570 [1:03:30<30:15, 2.80s/it]
59%|█████▊ | 922/1570 [1:03:32<30:16, 2.80s/it]
59%|█████▉ | 923/1570 [1:03:35<30:11, 2.80s/it]
59%|█████▉ | 924/1570 [1:03:38<30:10, 2.80s/it]
59%|█████▉ | 925/1570 [1:03:41<30:08, 2.80s/it]
59%|█████▉ | 926/1570 [1:03:44<29:57, 2.79s/it]
59%|█████▉ | 927/1570 [1:03:46<29:58, 2.80s/it]
59%|█████▉ | 928/1570 [1:03:49<29:51, 2.79s/it]
59%|█████▉ | 929/1570 [1:03:52<29:59, 2.81s/it]
59%|█████▉ | 930/1570 [1:03:55<30:05, 2.82s/it]
59%|█████▉ | 931/1570 [1:03:58<29:56, 2.81s/it]
59%|█████▉ | 932/1570 [1:04:00<29:55, 2.82s/it]
59%|█████▉ | 933/1570 [1:04:03<29:55, 2.82s/it]
59%|█████▉ | 934/1570 [1:04:06<29:55, 2.82s/it]
60%|█████▉ | 935/1570 [1:04:09<29:57, 2.83s/it]
60%|█████▉ | 936/1570 [1:04:12<29:56, 2.83s/it]
60%|█████▉ | 937/1570 [1:04:15<29:49, 2.83s/it]
60%|█████▉ | 938/1570 [1:04:17<29:38, 2.81s/it]
60%|█████▉ | 939/1570 [1:04:20<29:40, 2.82s/it]
60%|█████▉ | 940/1570 [1:04:23<29:38, 2.82s/it]
{'loss': '-0.01882', 'grad_norm': '0.1433', 'learning_rate': '3.49e-05', 'epoch': '2.996'}
+
60%|█████▉ | 940/1570 [1:04:23<29:38, 2.82s/it]
60%|█████▉ | 941/1570 [1:04:26<29:30, 2.81s/it]
60%|██████ | 942/1570 [1:04:27<24:11, 2.31s/it][INFO|trainer.py:2668] 2026-08-22 20:39:53,461 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 20:39:53,461 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 20:39:53,461 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/csd/checkpoint-942
+[INFO|configuration_utils.py:780] 2026-08-22 20:50:02,701 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 20:50:02,706 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 20:50:02,742 >> chat template saved in results/qwen3/csd/checkpoint-942/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 20:50:02,742 >> tokenizer config file saved in results/qwen3/csd/checkpoint-942/tokenizer_config.json
+
60%|██████ | 943/1570 [1:14:40<32:18:08, 185.47s/it]
60%|██████ | 944/1570 [1:14:43<22:43:16, 130.66s/it]
60%|██████ | 945/1570 [1:14:45<16:01:35, 92.31s/it]
60%|██████ | 946/1570 [1:14:48<11:20:41, 65.45s/it]
60%|██████ | 947/1570 [1:14:51<8:04:24, 46.65s/it]
60%|██████ | 948/1570 [1:14:54<5:47:10, 33.49s/it]
60%|██████ | 949/1570 [1:14:57<4:11:19, 24.28s/it]
61%|██████ | 950/1570 [1:14:59<3:04:19, 17.84s/it]
61%|██████ | 951/1570 [1:15:02<2:17:26, 13.32s/it]
61%|██████ | 952/1570 [1:15:05<1:44:44, 10.17s/it]
61%|██████ | 953/1570 [1:15:08<1:21:54, 7.97s/it]
61%|██████ | 954/1570 [1:15:11<1:05:51, 6.42s/it]
61%|██████ | 955/1570 [1:15:13<54:40, 5.33s/it]
61%|██████ | 956/1570 [1:15:16<46:53, 4.58s/it]
61%|██████ | 957/1570 [1:15:19<41:18, 4.04s/it]
61%|██████ | 958/1570 [1:15:22<37:26, 3.67s/it]
61%|██████ | 959/1570 [1:15:25<34:38, 3.40s/it]
61%|██████ | 960/1570 [1:15:27<32:45, 3.22s/it]
{'loss': '-0.02238', 'grad_norm': '0.1492', 'learning_rate': '3.301e-05', 'epoch': '3.057'}
+
61%|██████ | 960/1570 [1:15:27<32:45, 3.22s/it]
61%|██████ | 961/1570 [1:15:30<31:17, 3.08s/it]
61%|██████▏ | 962/1570 [1:15:33<30:18, 2.99s/it]
61%|██████▏ | 963/1570 [1:15:36<29:42, 2.94s/it]
61%|██████▏ | 964/1570 [1:15:39<29:22, 2.91s/it]
61%|██████▏ | 965/1570 [1:15:41<29:02, 2.88s/it]
62%|██████▏ | 966/1570 [1:15:44<28:48, 2.86s/it]
62%|██████▏ | 967/1570 [1:15:47<28:26, 2.83s/it]
62%|██████▏ | 968/1570 [1:15:50<28:24, 2.83s/it]
62%|██████▏ | 969/1570 [1:15:53<28:16, 2.82s/it]
62%|██████▏ | 970/1570 [1:15:55<28:15, 2.83s/it]
62%|██████▏ | 971/1570 [1:15:58<28:04, 2.81s/it]
62%|██████▏ | 972/1570 [1:16:01<27:53, 2.80s/it]
62%|██████▏ | 973/1570 [1:16:04<27:46, 2.79s/it]
62%|██████▏ | 974/1570 [1:16:07<27:46, 2.80s/it]
62%|██████▏ | 975/1570 [1:16:09<27:42, 2.79s/it]
62%|██████▏ | 976/1570 [1:16:12<27:35, 2.79s/it]
62%|██████▏ | 977/1570 [1:16:15<27:38, 2.80s/it]
62%|██████▏ | 978/1570 [1:16:18<27:35, 2.80s/it]
62%|██████▏ | 979/1570 [1:16:21<27:28, 2.79s/it]
62%|██████▏ | 980/1570 [1:16:23<27:32, 2.80s/it]
{'loss': '-0.02528', 'grad_norm': '0.07787', 'learning_rate': '3.114e-05', 'epoch': '3.121'}
+
62%|██████▏ | 980/1570 [1:16:23<27:32, 2.80s/it]
62%|██████▏ | 981/1570 [1:16:26<27:30, 2.80s/it]
63%|██████▎ | 982/1570 [1:16:29<27:29, 2.81s/it]
63%|██████▎ | 983/1570 [1:16:32<27:32, 2.81s/it]
63%|██████▎ | 984/1570 [1:16:35<27:24, 2.81s/it]
63%|██████▎ | 985/1570 [1:16:37<27:19, 2.80s/it]
63%|██████▎ | 986/1570 [1:16:40<27:18, 2.81s/it]
63%|██████▎ | 987/1570 [1:16:43<27:11, 2.80s/it]
63%|██████▎ | 988/1570 [1:16:46<27:15, 2.81s/it]
63%|██████▎ | 989/1570 [1:16:49<27:16, 2.82s/it]
63%|██████▎ | 990/1570 [1:16:51<27:04, 2.80s/it]
63%|██████▎ | 991/1570 [1:16:54<27:07, 2.81s/it]
63%|██████▎ | 992/1570 [1:16:57<27:03, 2.81s/it]
63%|██████▎ | 993/1570 [1:17:00<26:58, 2.80s/it]
63%|██████▎ | 994/1570 [1:17:03<26:52, 2.80s/it]
63%|██████▎ | 995/1570 [1:17:05<26:49, 2.80s/it]
63%|██████▎ | 996/1570 [1:17:08<26:42, 2.79s/it]
64%|██████▎ | 997/1570 [1:17:11<26:36, 2.79s/it]
64%|██████▎ | 998/1570 [1:17:14<26:34, 2.79s/it]
64%|██████▎ | 999/1570 [1:17:17<26:36, 2.80s/it]
64%|██████▎ | 1000/1570 [1:17:19<26:37, 2.80s/it]
{'loss': '-0.02356', 'grad_norm': '0.07071', 'learning_rate': '2.931e-05', 'epoch': '3.185'}
+
64%|██████▎ | 1000/1570 [1:17:19<26:37, 2.80s/it]
64%|██████▍ | 1001/1570 [1:17:22<26:30, 2.79s/it]
64%|██████▍ | 1002/1570 [1:17:25<26:21, 2.78s/it]
64%|██████▍ | 1003/1570 [1:17:28<26:27, 2.80s/it]
64%|██████▍ | 1004/1570 [1:17:31<26:20, 2.79s/it]
64%|██████▍ | 1005/1570 [1:17:33<26:20, 2.80s/it]
64%|██████▍ | 1006/1570 [1:17:36<26:17, 2.80s/it]
64%|██████▍ | 1007/1570 [1:17:39<26:15, 2.80s/it]
64%|██████▍ | 1008/1570 [1:17:42<26:09, 2.79s/it]
64%|██████▍ | 1009/1570 [1:17:45<26:04, 2.79s/it]
64%|██████▍ | 1010/1570 [1:17:47<26:01, 2.79s/it]
64%|██████▍ | 1011/1570 [1:17:50<26:08, 2.81s/it]
64%|██████▍ | 1012/1570 [1:17:53<26:08, 2.81s/it]
65%|██████▍ | 1013/1570 [1:17:56<26:04, 2.81s/it]
65%|██████▍ | 1014/1570 [1:17:59<26:05, 2.82s/it]
65%|██████▍ | 1015/1570 [1:18:01<25:59, 2.81s/it]
65%|██████▍ | 1016/1570 [1:18:04<25:57, 2.81s/it]
65%|██████▍ | 1017/1570 [1:18:07<25:53, 2.81s/it]
65%|██████▍ | 1018/1570 [1:18:10<25:44, 2.80s/it]
65%|██████▍ | 1019/1570 [1:18:13<25:36, 2.79s/it]
65%|██████▍ | 1020/1570 [1:18:15<25:37, 2.80s/it]
{'loss': '-0.01907', 'grad_norm': '0.1004', 'learning_rate': '2.751e-05', 'epoch': '3.249'}
+
65%|██████▍ | 1020/1570 [1:18:15<25:37, 2.80s/it]
65%|██████▌ | 1021/1570 [1:18:18<25:38, 2.80s/it]
65%|██████▌ | 1022/1570 [1:18:21<25:33, 2.80s/it]
65%|██████▌ | 1023/1570 [1:18:24<25:32, 2.80s/it]
65%|██████▌ | 1024/1570 [1:18:27<25:30, 2.80s/it]
65%|██████▌ | 1025/1570 [1:18:29<25:24, 2.80s/it]
65%|██████▌ | 1026/1570 [1:18:32<25:23, 2.80s/it]
65%|██████▌ | 1027/1570 [1:18:35<25:22, 2.80s/it]
65%|██████▌ | 1028/1570 [1:18:38<25:16, 2.80s/it]
66%|██████▌ | 1029/1570 [1:18:41<25:12, 2.80s/it]
66%|██████▌ | 1030/1570 [1:18:43<25:04, 2.79s/it]
66%|██████▌ | 1031/1570 [1:18:46<25:03, 2.79s/it]
66%|██████▌ | 1032/1570 [1:18:49<25:06, 2.80s/it]
66%|██████▌ | 1033/1570 [1:18:52<25:06, 2.80s/it]
66%|██████▌ | 1034/1570 [1:18:55<25:05, 2.81s/it]
66%|██████▌ | 1035/1570 [1:18:57<24:58, 2.80s/it]
66%|██████▌ | 1036/1570 [1:19:00<24:53, 2.80s/it]
66%|██████▌ | 1037/1570 [1:19:03<25:01, 2.82s/it]
66%|██████▌ | 1038/1570 [1:19:06<25:02, 2.82s/it]
66%|██████▌ | 1039/1570 [1:19:09<25:02, 2.83s/it]
66%|██████▌ | 1040/1570 [1:19:12<24:54, 2.82s/it]
{'loss': '-0.02228', 'grad_norm': '0.3305', 'learning_rate': '2.574e-05', 'epoch': '3.313'}
+
66%|██████▌ | 1040/1570 [1:19:12<24:54, 2.82s/it]
66%|██████▋ | 1041/1570 [1:19:14<24:40, 2.80s/it]
66%|██████▋ | 1042/1570 [1:19:17<24:34, 2.79s/it]
66%|██████▋ | 1043/1570 [1:19:20<24:32, 2.79s/it]
66%|██████▋ | 1044/1570 [1:19:23<24:32, 2.80s/it]
67%|██████▋ | 1045/1570 [1:19:25<24:22, 2.79s/it]
67%|██████▋ | 1046/1570 [1:19:28<24:18, 2.78s/it]
67%|██████▋ | 1047/1570 [1:19:31<24:17, 2.79s/it]
67%|██████▋ | 1048/1570 [1:19:34<24:12, 2.78s/it]
67%|██████▋ | 1049/1570 [1:19:37<24:10, 2.78s/it]
67%|██████▋ | 1050/1570 [1:19:39<24:09, 2.79s/it]
67%|██████▋ | 1051/1570 [1:19:42<24:03, 2.78s/it]
67%|██████▋ | 1052/1570 [1:19:45<24:05, 2.79s/it]
67%|██████▋ | 1053/1570 [1:19:48<24:06, 2.80s/it]
67%|██████▋ | 1054/1570 [1:19:51<24:00, 2.79s/it]
67%|██████▋ | 1055/1570 [1:19:53<23:52, 2.78s/it]
67%|██████▋ | 1056/1570 [1:19:56<23:47, 2.78s/it]
67%|██████▋ | 1057/1570 [1:19:59<23:33, 2.75s/it]
67%|██████▋ | 1058/1570 [1:20:02<23:38, 2.77s/it]
67%|██████▋ | 1059/1570 [1:20:04<23:42, 2.78s/it]
68%|██████▊ | 1060/1570 [1:20:07<23:39, 2.78s/it]
{'loss': '-0.02369', 'grad_norm': '0.1215', 'learning_rate': '2.402e-05', 'epoch': '3.377'}
+
68%|██████▊ | 1060/1570 [1:20:07<23:39, 2.78s/it]
68%|██████▊ | 1061/1570 [1:20:10<23:46, 2.80s/it]
68%|██████▊ | 1062/1570 [1:20:13<23:44, 2.80s/it]
68%|██████▊ | 1063/1570 [1:20:16<23:35, 2.79s/it]
68%|██████▊ | 1064/1570 [1:20:18<23:34, 2.80s/it]
68%|██████▊ | 1065/1570 [1:20:21<23:34, 2.80s/it]
68%|██████▊ | 1066/1570 [1:20:24<23:28, 2.80s/it]
68%|██████▊ | 1067/1570 [1:20:27<23:21, 2.79s/it]
68%|██████▊ | 1068/1570 [1:20:30<23:21, 2.79s/it]
68%|██████▊ | 1069/1570 [1:20:32<23:23, 2.80s/it]
68%|██████▊ | 1070/1570 [1:20:35<23:20, 2.80s/it]
68%|██████▊ | 1071/1570 [1:20:38<23:18, 2.80s/it]
68%|██████▊ | 1072/1570 [1:20:41<23:18, 2.81s/it]
68%|██████▊ | 1073/1570 [1:20:44<23:14, 2.81s/it]
68%|██████▊ | 1074/1570 [1:20:46<23:10, 2.80s/it]
68%|██████▊ | 1075/1570 [1:20:49<23:03, 2.80s/it]
69%|██████▊ | 1076/1570 [1:20:52<23:02, 2.80s/it]
69%|██████▊ | 1077/1570 [1:20:55<22:55, 2.79s/it]
69%|██████▊ | 1078/1570 [1:20:58<22:57, 2.80s/it]
69%|██████▊ | 1079/1570 [1:21:00<22:51, 2.79s/it]
69%|██████▉ | 1080/1570 [1:21:03<22:49, 2.80s/it]
{'loss': '-0.02528', 'grad_norm': '0.1637', 'learning_rate': '2.233e-05', 'epoch': '3.44'}
+
69%|██████▉ | 1080/1570 [1:21:03<22:49, 2.80s/it]
69%|██████▉ | 1081/1570 [1:21:06<22:48, 2.80s/it]
69%|██████▉ | 1082/1570 [1:21:09<22:47, 2.80s/it]
69%|██████▉ | 1083/1570 [1:21:12<22:43, 2.80s/it]
69%|██████▉ | 1084/1570 [1:21:14<22:42, 2.80s/it]
69%|██████▉ | 1085/1570 [1:21:17<22:38, 2.80s/it]
69%|██████▉ | 1086/1570 [1:21:20<22:30, 2.79s/it]
69%|██████▉ | 1087/1570 [1:21:23<22:29, 2.79s/it]
69%|██████▉ | 1088/1570 [1:21:26<22:23, 2.79s/it]
69%|██████▉ | 1089/1570 [1:21:28<22:22, 2.79s/it]
69%|██████▉ | 1090/1570 [1:21:31<22:15, 2.78s/it]
69%|██████▉ | 1091/1570 [1:21:35<25:02, 3.14s/it]
70%|██████▉ | 1092/1570 [1:21:39<26:55, 3.38s/it]
70%|██████▉ | 1093/1570 [1:21:43<28:22, 3.57s/it]
70%|██████▉ | 1094/1570 [1:21:47<29:25, 3.71s/it]
70%|██████▉ | 1095/1570 [1:21:51<29:57, 3.79s/it]
70%|██████▉ | 1096/1570 [1:21:55<30:10, 3.82s/it]
70%|██████▉ | 1097/1570 [1:21:59<30:28, 3.87s/it]
70%|██████▉ | 1098/1570 [1:22:03<30:38, 3.90s/it]
70%|███████ | 1099/1570 [1:22:07<30:26, 3.88s/it]
70%|███████ | 1100/1570 [1:22:11<30:37, 3.91s/it]
{'loss': '-0.0267', 'grad_norm': '0.08773', 'learning_rate': '2.069e-05', 'epoch': '3.504'}
+
70%|███████ | 1100/1570 [1:22:11<30:37, 3.91s/it]
70%|███████ | 1101/1570 [1:22:14<30:21, 3.88s/it]
70%|███████ | 1102/1570 [1:22:19<30:39, 3.93s/it]
70%|███████ | 1103/1570 [1:22:22<30:41, 3.94s/it]
70%|███████ | 1104/1570 [1:22:26<30:43, 3.96s/it]
70%|███████ | 1105/1570 [1:22:31<30:52, 3.98s/it]
70%|███████ | 1106/1570 [1:22:34<30:25, 3.93s/it]
71%|███████ | 1107/1570 [1:22:38<30:20, 3.93s/it]
71%|███████ | 1108/1570 [1:22:42<30:08, 3.91s/it]
71%|███████ | 1109/1570 [1:22:46<30:18, 3.94s/it]
71%|███████ | 1110/1570 [1:22:50<30:21, 3.96s/it]
71%|███████ | 1111/1570 [1:22:54<30:30, 3.99s/it]
71%|███████ | 1112/1570 [1:22:58<30:00, 3.93s/it]
71%|███████ | 1113/1570 [1:23:02<30:04, 3.95s/it]
71%|███████ | 1114/1570 [1:23:06<30:13, 3.98s/it]
71%|███████ | 1115/1570 [1:23:10<29:54, 3.94s/it]
71%|███████ | 1116/1570 [1:23:14<30:02, 3.97s/it]
71%|███████ | 1117/1570 [1:23:18<30:04, 3.98s/it]
71%|███████ | 1118/1570 [1:23:22<30:01, 3.98s/it]
71%|███████▏ | 1119/1570 [1:23:26<29:56, 3.98s/it]
71%|███████▏ | 1120/1570 [1:23:30<29:45, 3.97s/it]
{'loss': '-0.02801', 'grad_norm': '0.06677', 'learning_rate': '1.91e-05', 'epoch': '3.568'}
+
71%|███████▏ | 1120/1570 [1:23:30<29:45, 3.97s/it]
71%|███████▏ | 1121/1570 [1:23:34<29:17, 3.91s/it]
71%|███████▏ | 1122/1570 [1:23:38<29:21, 3.93s/it]
72%|███████▏ | 1123/1570 [1:23:42<29:30, 3.96s/it]
72%|███████▏ | 1124/1570 [1:23:46<29:23, 3.95s/it]
72%|███████▏ | 1125/1570 [1:23:50<29:25, 3.97s/it]
72%|███████▏ | 1126/1570 [1:23:54<29:25, 3.98s/it]
72%|███████▏ | 1127/1570 [1:23:57<29:07, 3.94s/it]
72%|███████▏ | 1128/1570 [1:24:01<29:09, 3.96s/it]
72%|███████▏ | 1129/1570 [1:24:05<29:11, 3.97s/it]
72%|███████▏ | 1130/1570 [1:24:09<29:10, 3.98s/it]
72%|███████▏ | 1131/1570 [1:24:13<29:11, 3.99s/it]
72%|███████▏ | 1132/1570 [1:24:17<29:03, 3.98s/it]
72%|███████▏ | 1133/1570 [1:24:21<28:36, 3.93s/it]
72%|███████▏ | 1134/1570 [1:24:25<28:38, 3.94s/it]
72%|███████▏ | 1135/1570 [1:24:29<28:21, 3.91s/it]
72%|███████▏ | 1136/1570 [1:24:33<28:33, 3.95s/it]
72%|███████▏ | 1137/1570 [1:24:37<28:27, 3.94s/it]
72%|███████▏ | 1138/1570 [1:24:41<28:05, 3.90s/it]
73%|███████▎ | 1139/1570 [1:24:45<27:53, 3.88s/it]
73%|███████▎ | 1140/1570 [1:24:48<27:15, 3.80s/it]
{'loss': '-0.03232', 'grad_norm': '0.09503', 'learning_rate': '1.755e-05', 'epoch': '3.632'}
+
73%|███████▎ | 1140/1570 [1:24:48<27:15, 3.80s/it]
73%|███████▎ | 1141/1570 [1:24:52<27:27, 3.84s/it]
73%|███████▎ | 1142/1570 [1:24:56<27:39, 3.88s/it]
73%|███████▎ | 1143/1570 [1:25:00<27:24, 3.85s/it]
73%|███████▎ | 1144/1570 [1:25:04<27:31, 3.88s/it]
73%|███████▎ | 1145/1570 [1:25:08<27:23, 3.87s/it]
73%|███████▎ | 1146/1570 [1:25:12<27:29, 3.89s/it]
73%|███████▎ | 1147/1570 [1:25:16<27:42, 3.93s/it]
73%|███████▎ | 1148/1570 [1:25:20<27:30, 3.91s/it]
73%|███████▎ | 1149/1570 [1:25:24<27:33, 3.93s/it]
73%|███████▎ | 1150/1570 [1:25:27<27:34, 3.94s/it]
73%|███████▎ | 1151/1570 [1:25:31<27:33, 3.95s/it]
73%|███████▎ | 1152/1570 [1:25:35<27:37, 3.96s/it]
73%|███████▎ | 1153/1570 [1:25:39<27:24, 3.94s/it]
74%|███████▎ | 1154/1570 [1:25:43<27:14, 3.93s/it]
74%|███████▎ | 1155/1570 [1:25:47<27:25, 3.97s/it]
74%|███████▎ | 1156/1570 [1:25:51<27:12, 3.94s/it]
74%|███████▎ | 1157/1570 [1:25:55<27:07, 3.94s/it]
74%|███████▍ | 1158/1570 [1:25:59<27:04, 3.94s/it]
74%|███████▍ | 1159/1570 [1:26:03<26:50, 3.92s/it]
74%|███████▍ | 1160/1570 [1:26:07<26:50, 3.93s/it]
{'loss': '-0.03129', 'grad_norm': '0.08011', 'learning_rate': '1.606e-05', 'epoch': '3.696'}
+
74%|███████▍ | 1160/1570 [1:26:07<26:50, 3.93s/it]
74%|███████▍ | 1161/1570 [1:26:11<26:45, 3.93s/it]
74%|███████▍ | 1162/1570 [1:26:15<26:52, 3.95s/it]
74%|███████▍ | 1163/1570 [1:26:19<26:48, 3.95s/it]
74%|███████▍ | 1164/1570 [1:26:23<26:40, 3.94s/it]
74%|███████▍ | 1165/1570 [1:26:27<26:53, 3.98s/it]
74%|███████▍ | 1166/1570 [1:26:31<26:53, 3.99s/it]
74%|███████▍ | 1167/1570 [1:26:35<26:38, 3.97s/it]
74%|███████▍ | 1168/1570 [1:26:39<26:42, 3.99s/it]
74%|███████▍ | 1169/1570 [1:26:43<26:36, 3.98s/it]
75%|███████▍ | 1170/1570 [1:26:47<26:16, 3.94s/it]
75%|███████▍ | 1171/1570 [1:26:51<26:17, 3.95s/it]
75%|███████▍ | 1172/1570 [1:26:54<25:50, 3.89s/it]
75%|███████▍ | 1173/1570 [1:26:58<25:54, 3.92s/it]
75%|███████▍ | 1174/1570 [1:27:02<25:48, 3.91s/it]
75%|███████▍ | 1175/1570 [1:27:06<25:42, 3.90s/it]
75%|███████▍ | 1176/1570 [1:27:10<25:17, 3.85s/it]
75%|███████▍ | 1177/1570 [1:27:14<25:28, 3.89s/it]
75%|███████▌ | 1178/1570 [1:27:18<25:28, 3.90s/it]
75%|███████▌ | 1179/1570 [1:27:22<25:32, 3.92s/it]
75%|███████▌ | 1180/1570 [1:27:26<25:41, 3.95s/it]
{'loss': '-0.03188', 'grad_norm': '0.1588', 'learning_rate': '1.462e-05', 'epoch': '3.759'}
+
75%|███████▌ | 1180/1570 [1:27:26<25:41, 3.95s/it]
75%|███████▌ | 1181/1570 [1:27:30<25:39, 3.96s/it]
75%|███████▌ | 1182/1570 [1:27:34<25:41, 3.97s/it]
75%|███████▌ | 1183/1570 [1:27:38<25:43, 3.99s/it]
75%|███████▌ | 1184/1570 [1:27:42<25:36, 3.98s/it]
75%|███████▌ | 1185/1570 [1:27:45<25:20, 3.95s/it]
76%|███████▌ | 1186/1570 [1:27:50<25:30, 3.99s/it]
76%|███████▌ | 1187/1570 [1:27:54<25:28, 3.99s/it]
76%|███████▌ | 1188/1570 [1:27:57<25:10, 3.96s/it]
76%|███████▌ | 1189/1570 [1:28:01<25:17, 3.98s/it]
76%|███████▌ | 1190/1570 [1:28:06<25:23, 4.01s/it]
76%|███████▌ | 1191/1570 [1:28:09<25:01, 3.96s/it]
76%|███████▌ | 1192/1570 [1:28:13<24:47, 3.94s/it]
76%|███████▌ | 1193/1570 [1:28:17<24:49, 3.95s/it]
76%|███████▌ | 1194/1570 [1:28:21<24:39, 3.93s/it]
76%|███████▌ | 1195/1570 [1:28:25<24:39, 3.95s/it]
76%|███████▌ | 1196/1570 [1:28:29<24:19, 3.90s/it]
76%|███████▌ | 1197/1570 [1:28:33<24:26, 3.93s/it]
76%|███████▋ | 1198/1570 [1:28:37<24:24, 3.94s/it]
76%|███████▋ | 1199/1570 [1:28:41<24:12, 3.91s/it]
76%|███████▋ | 1200/1570 [1:28:45<24:20, 3.95s/it]
{'loss': '-0.0328', 'grad_norm': '0.07065', 'learning_rate': '1.324e-05', 'epoch': '3.823'}
+
76%|███████▋ | 1200/1570 [1:28:45<24:20, 3.95s/it]
76%|███████▋ | 1201/1570 [1:28:49<24:24, 3.97s/it]
77%|███████▋ | 1202/1570 [1:28:53<24:30, 4.00s/it]
77%|███████▋ | 1203/1570 [1:28:57<24:22, 3.98s/it]
77%|███████▋ | 1204/1570 [1:29:01<24:11, 3.97s/it]
77%|███████▋ | 1205/1570 [1:29:05<24:08, 3.97s/it]
77%|███████▋ | 1206/1570 [1:29:09<24:06, 3.97s/it]
77%|███████▋ | 1207/1570 [1:29:13<24:03, 3.98s/it]
77%|███████▋ | 1208/1570 [1:29:17<24:09, 4.00s/it]
77%|███████▋ | 1209/1570 [1:29:21<23:59, 3.99s/it]
77%|███████▋ | 1210/1570 [1:29:25<23:55, 3.99s/it]
77%|███████▋ | 1211/1570 [1:29:29<23:40, 3.96s/it]
77%|███████▋ | 1212/1570 [1:29:33<23:46, 3.98s/it]
77%|███████▋ | 1213/1570 [1:29:36<23:31, 3.95s/it]
77%|███████▋ | 1214/1570 [1:29:40<22:56, 3.87s/it]
77%|███████▋ | 1215/1570 [1:29:44<22:45, 3.85s/it]
77%|███████▋ | 1216/1570 [1:29:48<22:53, 3.88s/it]
78%|███████▊ | 1217/1570 [1:29:52<23:00, 3.91s/it]
78%|███████▊ | 1218/1570 [1:29:56<23:06, 3.94s/it]
78%|███████▊ | 1219/1570 [1:30:00<23:00, 3.93s/it]
78%|███████▊ | 1220/1570 [1:30:04<22:57, 3.94s/it]
{'loss': '-0.03418', 'grad_norm': '0.2448', 'learning_rate': '1.192e-05', 'epoch': '3.887'}
+
78%|███████▊ | 1220/1570 [1:30:04<22:57, 3.94s/it]
78%|███████▊ | 1221/1570 [1:30:08<22:57, 3.95s/it]
78%|███████▊ | 1222/1570 [1:30:12<22:35, 3.90s/it]
78%|███████▊ | 1223/1570 [1:30:16<22:44, 3.93s/it]
78%|███████▊ | 1224/1570 [1:30:20<22:49, 3.96s/it]
78%|███████▊ | 1225/1570 [1:30:23<22:37, 3.93s/it]
78%|███████▊ | 1226/1570 [1:30:27<22:37, 3.95s/it]
78%|███████▊ | 1227/1570 [1:30:31<22:41, 3.97s/it]
78%|███████▊ | 1228/1570 [1:30:35<22:23, 3.93s/it]
78%|███████▊ | 1229/1570 [1:30:39<22:34, 3.97s/it]
78%|███████▊ | 1230/1570 [1:30:43<22:26, 3.96s/it]
78%|███████▊ | 1231/1570 [1:30:47<22:29, 3.98s/it]
78%|███████▊ | 1232/1570 [1:30:51<21:55, 3.89s/it]
79%|███████▊ | 1233/1570 [1:30:55<21:53, 3.90s/it]
79%|███████▊ | 1234/1570 [1:30:59<21:26, 3.83s/it]
79%|███████▊ | 1235/1570 [1:31:02<21:14, 3.80s/it]
79%|███████▊ | 1236/1570 [1:31:06<21:36, 3.88s/it]
79%|███████▉ | 1237/1570 [1:31:10<21:40, 3.90s/it]
79%|███████▉ | 1238/1570 [1:31:14<21:35, 3.90s/it]
79%|███████▉ | 1239/1570 [1:31:18<21:43, 3.94s/it]
79%|███████▉ | 1240/1570 [1:31:22<21:42, 3.95s/it]
{'loss': '-0.03614', 'grad_norm': '0.09534', 'learning_rate': '1.066e-05', 'epoch': '3.951'}
+
79%|███████▉ | 1240/1570 [1:31:22<21:42, 3.95s/it]
79%|███████▉ | 1241/1570 [1:31:26<21:43, 3.96s/it]
79%|███████▉ | 1242/1570 [1:31:30<21:43, 3.97s/it]
79%|███████▉ | 1243/1570 [1:31:34<21:35, 3.96s/it]
79%|███████▉ | 1244/1570 [1:31:38<21:32, 3.96s/it]
79%|███████▉ | 1245/1570 [1:31:42<21:32, 3.98s/it]
79%|███████▉ | 1246/1570 [1:31:46<21:04, 3.90s/it]
79%|███████▉ | 1247/1570 [1:31:50<21:15, 3.95s/it]
79%|███████▉ | 1248/1570 [1:31:54<21:03, 3.92s/it]
80%|███████▉ | 1249/1570 [1:31:58<20:58, 3.92s/it]
80%|███████▉ | 1250/1570 [1:32:01<20:31, 3.85s/it]
80%|███████▉ | 1251/1570 [1:32:05<20:27, 3.85s/it]
80%|███████▉ | 1252/1570 [1:32:09<20:26, 3.86s/it]
80%|███████▉ | 1253/1570 [1:32:13<20:38, 3.91s/it]
80%|███████▉ | 1254/1570 [1:32:17<20:29, 3.89s/it]
80%|███████▉ | 1255/1570 [1:32:21<20:24, 3.89s/it]
80%|████████ | 1256/1570 [1:32:22<16:45, 3.20s/it][INFO|trainer.py:2668] 2026-08-22 21:07:48,886 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 21:07:48,886 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 21:07:48,886 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/csd/checkpoint-1256
+[INFO|configuration_utils.py:780] 2026-08-22 21:17:56,542 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 21:17:56,545 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 21:17:56,580 >> chat template saved in results/qwen3/csd/checkpoint-1256/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 21:17:56,580 >> tokenizer config file saved in results/qwen3/csd/checkpoint-1256/tokenizer_config.json
+
80%|████████ | 1257/1570 [1:42:35<16:10:03, 185.95s/it]
80%|████████ | 1258/1570 [1:42:39<11:23:13, 131.39s/it]
80%|████████ | 1259/1570 [1:42:43<8:02:58, 93.18s/it]
80%|████████ | 1260/1570 [1:42:47<5:43:16, 66.44s/it]
{'loss': '-0.03401', 'grad_norm': '0.5279', 'learning_rate': '9.464e-06', 'epoch': '4.013'}
+
80%|████████ | 1260/1570 [1:42:47<5:43:16, 66.44s/it]
80%|████████ | 1261/1570 [1:42:51<4:05:45, 47.72s/it]
80%|████████ | 1262/1570 [1:42:55<2:57:31, 34.58s/it]
80%|████████ | 1263/1570 [1:42:59<2:10:02, 25.42s/it]
81%|████████ | 1264/1570 [1:43:03<1:36:39, 18.95s/it]
81%|████████ | 1265/1570 [1:43:07<1:13:21, 14.43s/it]
81%|████████ | 1266/1570 [1:43:11<57:16, 11.30s/it]
81%|████████ | 1267/1570 [1:43:14<45:31, 9.01s/it]
81%|████████ | 1268/1570 [1:43:18<37:44, 7.50s/it]
81%|████████ | 1269/1570 [1:43:22<32:16, 6.43s/it]
81%|████████ | 1270/1570 [1:43:26<28:30, 5.70s/it]
81%|████████ | 1271/1570 [1:43:30<25:34, 5.13s/it]
81%|████████ | 1272/1570 [1:43:34<23:30, 4.73s/it]
81%|████████ | 1273/1570 [1:43:38<22:22, 4.52s/it]
81%|████████ | 1274/1570 [1:43:42<21:28, 4.35s/it]
81%|████████ | 1275/1570 [1:43:46<20:49, 4.23s/it]
81%|████████▏ | 1276/1570 [1:43:50<20:09, 4.11s/it]
81%|████████▏ | 1277/1570 [1:43:54<19:56, 4.09s/it]
81%|████████▏ | 1278/1570 [1:43:58<19:40, 4.04s/it]
81%|████████▏ | 1279/1570 [1:44:02<19:25, 4.00s/it]
82%|████████▏ | 1280/1570 [1:44:06<19:24, 4.02s/it]
{'loss': '-0.03716', 'grad_norm': '0.09058', 'learning_rate': '8.332e-06', 'epoch': '4.077'}
+
82%|████████▏ | 1280/1570 [1:44:06<19:24, 4.02s/it]
82%|████████▏ | 1281/1570 [1:44:09<19:07, 3.97s/it]
82%|████████▏ | 1282/1570 [1:44:13<19:00, 3.96s/it]
82%|████████▏ | 1283/1570 [1:44:17<18:44, 3.92s/it]
82%|████████▏ | 1284/1570 [1:44:21<18:38, 3.91s/it]
82%|████████▏ | 1285/1570 [1:44:25<18:39, 3.93s/it]
82%|████████▏ | 1286/1570 [1:44:29<18:40, 3.95s/it]
82%|████████▏ | 1287/1570 [1:44:33<18:44, 3.97s/it]
82%|████████▏ | 1288/1570 [1:44:37<18:29, 3.93s/it]
82%|████████▏ | 1289/1570 [1:44:41<18:28, 3.95s/it]
82%|████████▏ | 1290/1570 [1:44:45<18:19, 3.93s/it]
82%|████████▏ | 1291/1570 [1:44:49<18:18, 3.94s/it]
82%|████████▏ | 1292/1570 [1:44:53<18:05, 3.91s/it]
82%|████████▏ | 1293/1570 [1:44:57<18:04, 3.91s/it]
82%|████████▏ | 1294/1570 [1:45:00<18:02, 3.92s/it]
82%|████████▏ | 1295/1570 [1:45:04<17:52, 3.90s/it]
83%|████████▎ | 1296/1570 [1:45:08<17:47, 3.90s/it]
83%|████████▎ | 1297/1570 [1:45:12<17:47, 3.91s/it]
83%|████████▎ | 1298/1570 [1:45:16<17:36, 3.88s/it]
83%|████████▎ | 1299/1570 [1:45:20<17:46, 3.94s/it]
83%|████████▎ | 1300/1570 [1:45:24<17:41, 3.93s/it]
{'loss': '-0.038', 'grad_norm': '0.05896', 'learning_rate': '7.266e-06', 'epoch': '4.14'}
+
83%|████████▎ | 1300/1570 [1:45:24<17:41, 3.93s/it]
83%|████████▎ | 1301/1570 [1:45:28<17:44, 3.96s/it]
83%|████████▎ | 1302/1570 [1:45:32<17:45, 3.97s/it]
83%|████████▎ | 1303/1570 [1:45:36<17:42, 3.98s/it]
83%|████████▎ | 1304/1570 [1:45:40<17:30, 3.95s/it]
83%|████████▎ | 1305/1570 [1:45:44<17:16, 3.91s/it]
83%|████████▎ | 1306/1570 [1:45:48<17:05, 3.88s/it]
83%|████████▎ | 1307/1570 [1:45:51<17:10, 3.92s/it]
83%|████████▎ | 1308/1570 [1:45:55<17:06, 3.92s/it]
83%|████████▎ | 1309/1570 [1:45:59<17:03, 3.92s/it]
83%|████████▎ | 1310/1570 [1:46:03<16:52, 3.90s/it]
84%|████████▎ | 1311/1570 [1:46:07<16:56, 3.93s/it]
84%|████████▎ | 1312/1570 [1:46:11<16:52, 3.92s/it]
84%|████████▎ | 1313/1570 [1:46:15<16:49, 3.93s/it]
84%|████████▎ | 1314/1570 [1:46:19<16:49, 3.94s/it]
84%|████████▍ | 1315/1570 [1:46:23<16:45, 3.94s/it]
84%|████████▍ | 1316/1570 [1:46:27<16:44, 3.95s/it]
84%|████████▍ | 1317/1570 [1:46:31<16:39, 3.95s/it]
84%|████████▍ | 1318/1570 [1:46:35<16:38, 3.96s/it]
84%|████████▍ | 1319/1570 [1:46:39<16:31, 3.95s/it]
84%|████████▍ | 1320/1570 [1:46:42<16:08, 3.87s/it]
{'loss': '-0.0367', 'grad_norm': '0.08014', 'learning_rate': '6.269e-06', 'epoch': '4.204'}
+
84%|████████▍ | 1320/1570 [1:46:42<16:08, 3.87s/it]
84%|████████▍ | 1321/1570 [1:46:46<16:06, 3.88s/it]
84%|████████▍ | 1322/1570 [1:46:50<16:10, 3.91s/it]
84%|████████▍ | 1323/1570 [1:46:54<16:11, 3.93s/it]
84%|████████▍ | 1324/1570 [1:46:58<15:58, 3.90s/it]
84%|████████▍ | 1325/1570 [1:47:02<16:03, 3.93s/it]
84%|████████▍ | 1326/1570 [1:47:06<15:53, 3.91s/it]
85%|████████▍ | 1327/1570 [1:47:10<15:41, 3.88s/it]
85%|████████▍ | 1328/1570 [1:47:14<15:49, 3.93s/it]
85%|████████▍ | 1329/1570 [1:47:18<15:45, 3.92s/it]
85%|████████▍ | 1330/1570 [1:47:22<15:41, 3.92s/it]
85%|████████▍ | 1331/1570 [1:47:26<15:38, 3.93s/it]
85%|████████▍ | 1332/1570 [1:47:30<15:35, 3.93s/it]
85%|████████▍ | 1333/1570 [1:47:34<15:38, 3.96s/it]
85%|████████▍ | 1334/1570 [1:47:37<15:27, 3.93s/it]
85%|████████▌ | 1335/1570 [1:47:41<15:27, 3.95s/it]
85%|████████▌ | 1336/1570 [1:47:45<15:26, 3.96s/it]
85%|████████▌ | 1337/1570 [1:47:49<15:27, 3.98s/it]
85%|████████▌ | 1338/1570 [1:47:53<15:24, 3.99s/it]
85%|████████▌ | 1339/1570 [1:47:57<15:14, 3.96s/it]
85%|████████▌ | 1340/1570 [1:48:01<15:18, 3.99s/it]
{'loss': '-0.03779', 'grad_norm': '0.08858', 'learning_rate': '5.342e-06', 'epoch': '4.268'}
+
85%|████████▌ | 1340/1570 [1:48:01<15:18, 3.99s/it]
85%|████████▌ | 1341/1570 [1:48:05<15:09, 3.97s/it]
85%|████████▌ | 1342/1570 [1:48:09<15:10, 3.99s/it]
86%|████████▌ | 1343/1570 [1:48:13<15:03, 3.98s/it]
86%|████████▌ | 1344/1570 [1:48:17<14:52, 3.95s/it]
86%|████████▌ | 1345/1570 [1:48:21<14:44, 3.93s/it]
86%|████████▌ | 1346/1570 [1:48:25<14:28, 3.88s/it]
86%|████████▌ | 1347/1570 [1:48:29<14:35, 3.93s/it]
86%|████████▌ | 1348/1570 [1:48:33<14:31, 3.93s/it]
86%|████████▌ | 1349/1570 [1:48:37<14:33, 3.95s/it]
86%|████████▌ | 1350/1570 [1:48:41<14:36, 3.98s/it]
86%|████████▌ | 1351/1570 [1:48:45<14:41, 4.02s/it]
86%|████████▌ | 1352/1570 [1:48:49<14:41, 4.05s/it]
86%|████████▌ | 1353/1570 [1:48:53<14:21, 3.97s/it]
86%|████████▌ | 1354/1570 [1:48:57<14:18, 3.97s/it]
86%|████████▋ | 1355/1570 [1:49:01<14:15, 3.98s/it]
86%|████████▋ | 1356/1570 [1:49:05<14:09, 3.97s/it]
86%|████████▋ | 1357/1570 [1:49:09<14:07, 3.98s/it]
86%|████████▋ | 1358/1570 [1:49:13<14:03, 3.98s/it]
87%|████████▋ | 1359/1570 [1:49:17<13:54, 3.95s/it]
87%|████████▋ | 1360/1570 [1:49:20<13:32, 3.87s/it]
{'loss': '-0.03757', 'grad_norm': '0.07377', 'learning_rate': '4.486e-06', 'epoch': '4.332'}
+
87%|████████▋ | 1360/1570 [1:49:20<13:32, 3.87s/it]
87%|████████▋ | 1361/1570 [1:49:24<13:31, 3.88s/it]
87%|████████▋ | 1362/1570 [1:49:28<13:25, 3.87s/it]
87%|████████▋ | 1363/1570 [1:49:32<13:28, 3.90s/it]
87%|████████▋ | 1364/1570 [1:49:36<13:23, 3.90s/it]
87%|████████▋ | 1365/1570 [1:49:40<13:21, 3.91s/it]
87%|████████▋ | 1366/1570 [1:49:44<13:15, 3.90s/it]
87%|████████▋ | 1367/1570 [1:49:48<13:14, 3.91s/it]
87%|████████▋ | 1368/1570 [1:49:52<13:04, 3.88s/it]
87%|████████▋ | 1369/1570 [1:49:56<13:08, 3.92s/it]
87%|████████▋ | 1370/1570 [1:49:59<12:44, 3.82s/it]
87%|████████▋ | 1371/1570 [1:50:03<12:47, 3.86s/it]
87%|████████▋ | 1372/1570 [1:50:07<12:51, 3.90s/it]
87%|████████▋ | 1373/1570 [1:50:11<12:51, 3.91s/it]
88%|████████▊ | 1374/1570 [1:50:15<12:48, 3.92s/it]
88%|████████▊ | 1375/1570 [1:50:19<12:30, 3.85s/it]
88%|████████▊ | 1376/1570 [1:50:23<12:28, 3.86s/it]
88%|████████▊ | 1377/1570 [1:50:27<12:30, 3.89s/it]
88%|████████▊ | 1378/1570 [1:50:31<12:35, 3.93s/it]
88%|████████▊ | 1379/1570 [1:50:34<12:27, 3.91s/it]
88%|████████▊ | 1380/1570 [1:50:38<12:15, 3.87s/it]
{'loss': '-0.03949', 'grad_norm': '0.08888', 'learning_rate': '3.704e-06', 'epoch': '4.396'}
+
88%|████████▊ | 1380/1570 [1:50:38<12:15, 3.87s/it]
88%|████████▊ | 1381/1570 [1:50:42<12:00, 3.81s/it]
88%|████████▊ | 1382/1570 [1:50:46<11:59, 3.83s/it]
88%|████████▊ | 1383/1570 [1:50:50<12:08, 3.89s/it]
88%|████████▊ | 1384/1570 [1:50:54<12:06, 3.90s/it]
88%|████████▊ | 1385/1570 [1:50:58<12:03, 3.91s/it]
88%|████████▊ | 1386/1570 [1:51:02<12:05, 3.94s/it]
88%|████████▊ | 1387/1570 [1:51:06<11:58, 3.93s/it]
88%|████████▊ | 1388/1570 [1:51:10<12:01, 3.96s/it]
88%|████████▊ | 1389/1570 [1:51:14<12:02, 3.99s/it]
89%|████████▊ | 1390/1570 [1:51:18<12:15, 4.08s/it]
89%|████████▊ | 1391/1570 [1:51:22<11:53, 3.99s/it]
89%|████████▊ | 1392/1570 [1:51:26<11:50, 3.99s/it]
89%|████████▊ | 1393/1570 [1:51:30<11:43, 3.97s/it]
89%|████████▉ | 1394/1570 [1:51:33<11:31, 3.93s/it]
89%|████████▉ | 1395/1570 [1:51:37<11:25, 3.92s/it]
89%|████████▉ | 1396/1570 [1:51:41<11:19, 3.90s/it]
89%|████████▉ | 1397/1570 [1:51:45<11:18, 3.92s/it]
89%|████████▉ | 1398/1570 [1:51:49<11:10, 3.90s/it]
89%|████████▉ | 1399/1570 [1:51:53<10:58, 3.85s/it]
89%|████████▉ | 1400/1570 [1:51:57<11:04, 3.91s/it]
{'loss': '-0.03984', 'grad_norm': '0.1208', 'learning_rate': '2.996e-06', 'epoch': '4.46'}
+
89%|████████▉ | 1400/1570 [1:51:57<11:04, 3.91s/it]
89%|████████▉ | 1401/1570 [1:52:01<10:58, 3.90s/it]
89%|████████▉ | 1402/1570 [1:52:05<10:53, 3.89s/it]
89%|████████▉ | 1403/1570 [1:52:09<10:54, 3.92s/it]
89%|████████▉ | 1404/1570 [1:52:12<10:44, 3.88s/it]
89%|████████▉ | 1405/1570 [1:52:16<10:44, 3.91s/it]
90%|████████▉ | 1406/1570 [1:52:20<10:47, 3.95s/it]
90%|████████▉ | 1407/1570 [1:52:24<10:45, 3.96s/it]
90%|████████▉ | 1408/1570 [1:52:28<10:30, 3.89s/it]
90%|████████▉ | 1409/1570 [1:52:32<10:24, 3.88s/it]
90%|████████▉ | 1410/1570 [1:52:36<10:16, 3.85s/it]
90%|████████▉ | 1411/1570 [1:52:40<10:16, 3.88s/it]
90%|████████▉ | 1412/1570 [1:52:44<10:14, 3.89s/it]
90%|█████████ | 1413/1570 [1:52:48<10:16, 3.92s/it]
90%|█████████ | 1414/1570 [1:52:52<10:17, 3.96s/it]
90%|█████████ | 1415/1570 [1:52:56<10:17, 3.98s/it]
90%|█████████ | 1416/1570 [1:53:00<10:13, 3.98s/it]
90%|█████████ | 1417/1570 [1:53:04<10:09, 3.98s/it]
90%|█████████ | 1418/1570 [1:53:08<10:07, 3.99s/it]
90%|█████████ | 1419/1570 [1:53:12<10:02, 3.99s/it]
90%|█████████ | 1420/1570 [1:53:16<10:00, 4.01s/it]
{'loss': '-0.03817', 'grad_norm': '0.1605', 'learning_rate': '2.363e-06', 'epoch': '4.523'}
+
90%|█████████ | 1420/1570 [1:53:16<10:00, 4.01s/it]
91%|█████████ | 1421/1570 [1:53:20<09:53, 3.99s/it]
91%|█████████ | 1422/1570 [1:53:24<09:52, 4.01s/it]
91%|█████████ | 1423/1570 [1:53:28<09:47, 3.99s/it]
91%|█████████ | 1424/1570 [1:53:31<09:36, 3.95s/it]
91%|█████████ | 1425/1570 [1:53:36<09:38, 3.99s/it]
91%|█████████ | 1426/1570 [1:53:40<09:37, 4.01s/it]
91%|█████████ | 1427/1570 [1:53:43<09:27, 3.97s/it]
91%|█████████ | 1428/1570 [1:53:47<09:12, 3.89s/it]
91%|█████████ | 1429/1570 [1:53:51<09:06, 3.88s/it]
91%|█████████ | 1430/1570 [1:53:55<09:11, 3.94s/it]
91%|█████████ | 1431/1570 [1:53:59<09:11, 3.97s/it]
91%|█████████ | 1432/1570 [1:54:03<09:11, 4.00s/it]
91%|█████████▏| 1433/1570 [1:54:07<09:09, 4.01s/it]
91%|█████████▏| 1434/1570 [1:54:11<09:03, 3.99s/it]
91%|█████████▏| 1435/1570 [1:54:15<09:00, 4.00s/it]
91%|█████████▏| 1436/1570 [1:54:19<08:56, 4.01s/it]
92%|█████████▏| 1437/1570 [1:54:23<08:50, 3.99s/it]
92%|█████████▏| 1438/1570 [1:54:27<08:51, 4.02s/it]
92%|█████████▏| 1439/1570 [1:54:31<08:43, 4.00s/it]
92%|█████████▏| 1440/1570 [1:54:35<08:37, 3.98s/it]
{'loss': '-0.0407', 'grad_norm': '0.06491', 'learning_rate': '1.806e-06', 'epoch': '4.587'}
+
92%|█████████▏| 1440/1570 [1:54:35<08:37, 3.98s/it]
92%|█████████▏| 1441/1570 [1:54:39<08:33, 3.98s/it]
92%|█████████▏| 1442/1570 [1:54:43<08:26, 3.95s/it]
92%|█████████▏| 1443/1570 [1:54:47<08:17, 3.92s/it]
92%|█████████▏| 1444/1570 [1:54:51<08:16, 3.94s/it]
92%|█████████▏| 1445/1570 [1:54:55<08:09, 3.91s/it]
92%|█████████▏| 1446/1570 [1:54:59<08:03, 3.90s/it]
92%|█████████▏| 1447/1570 [1:55:03<08:01, 3.91s/it]
92%|█████████▏| 1448/1570 [1:55:07<07:59, 3.93s/it]
92%|█████████▏| 1449/1570 [1:55:11<07:58, 3.95s/it]
92%|█████████▏| 1450/1570 [1:55:15<07:55, 3.97s/it]
92%|█████████▏| 1451/1570 [1:55:18<07:46, 3.92s/it]
92%|█████████▏| 1452/1570 [1:55:22<07:44, 3.94s/it]
93%|█████████▎| 1453/1570 [1:55:26<07:44, 3.97s/it]
93%|█████████▎| 1454/1570 [1:55:30<07:39, 3.96s/it]
93%|█████████▎| 1455/1570 [1:55:34<07:36, 3.97s/it]
93%|█████████▎| 1456/1570 [1:55:38<07:30, 3.95s/it]
93%|█████████▎| 1457/1570 [1:55:42<07:26, 3.95s/it]
93%|█████████▎| 1458/1570 [1:55:46<07:17, 3.91s/it]
93%|█████████▎| 1459/1570 [1:55:50<07:17, 3.94s/it]
93%|█████████▎| 1460/1570 [1:55:54<07:05, 3.87s/it]
{'loss': '-0.03966', 'grad_norm': '0.106', 'learning_rate': '1.327e-06', 'epoch': '4.651'}
+
93%|█████████▎| 1460/1570 [1:55:54<07:05, 3.87s/it]
93%|█████████▎| 1461/1570 [1:55:57<06:57, 3.83s/it]
93%|█████████▎| 1462/1570 [1:56:01<06:58, 3.88s/it]
93%|█████████▎| 1463/1570 [1:56:05<06:56, 3.90s/it]
93%|█████████▎| 1464/1570 [1:56:09<06:54, 3.91s/it]
93%|█████████▎| 1465/1570 [1:56:13<06:46, 3.87s/it]
93%|█████████▎| 1466/1570 [1:56:17<06:46, 3.91s/it]
93%|█████████▎| 1467/1570 [1:56:21<06:39, 3.88s/it]
94%|█████████▎| 1468/1570 [1:56:25<06:41, 3.94s/it]
94%|█████████▎| 1469/1570 [1:56:29<06:39, 3.95s/it]
94%|█████████▎| 1470/1570 [1:56:33<06:35, 3.96s/it]
94%|█████████▎| 1471/1570 [1:56:37<06:32, 3.96s/it]
94%|█████████▍| 1472/1570 [1:56:41<06:26, 3.94s/it]
94%|█████████▍| 1473/1570 [1:56:45<06:24, 3.96s/it]
94%|█████████▍| 1474/1570 [1:56:49<06:21, 3.98s/it]
94%|█████████▍| 1475/1570 [1:56:53<06:11, 3.91s/it]
94%|█████████▍| 1476/1570 [1:56:57<06:11, 3.96s/it]
94%|█████████▍| 1477/1570 [1:57:01<06:10, 3.98s/it]
94%|█████████▍| 1478/1570 [1:57:05<06:06, 3.98s/it]
94%|█████████▍| 1479/1570 [1:57:09<06:01, 3.97s/it]
94%|█████████▍| 1480/1570 [1:57:13<06:00, 4.00s/it]
{'loss': '-0.04046', 'grad_norm': '0.07826', 'learning_rate': '9.258e-07', 'epoch': '4.715'}
+
94%|█████████▍| 1480/1570 [1:57:13<06:00, 4.00s/it]
94%|█████████▍| 1481/1570 [1:57:17<05:57, 4.01s/it]
94%|█████████▍| 1482/1570 [1:57:21<05:52, 4.00s/it]
94%|█████████▍| 1483/1570 [1:57:24<05:43, 3.95s/it]
95%|█████████▍| 1484/1570 [1:57:28<05:38, 3.93s/it]
95%|█████████▍| 1485/1570 [1:57:32<05:35, 3.95s/it]
95%|█████████▍| 1486/1570 [1:57:36<05:29, 3.92s/it]
95%|█████████▍| 1487/1570 [1:57:40<05:28, 3.95s/it]
95%|█████████▍| 1488/1570 [1:57:44<05:25, 3.96s/it]
95%|█████████▍| 1489/1570 [1:57:48<05:21, 3.97s/it]
95%|█████████▍| 1490/1570 [1:57:52<05:19, 4.00s/it]
95%|█████████▍| 1491/1570 [1:57:56<05:16, 4.01s/it]
95%|█████████▌| 1492/1570 [1:58:00<05:10, 3.98s/it]
95%|█████████▌| 1493/1570 [1:58:04<05:07, 3.99s/it]
95%|█████████▌| 1494/1570 [1:58:08<05:03, 3.99s/it]
95%|█████████▌| 1495/1570 [1:58:12<04:59, 4.00s/it]
95%|█████████▌| 1496/1570 [1:58:16<04:55, 4.00s/it]
95%|█████████▌| 1497/1570 [1:58:20<04:52, 4.01s/it]
95%|█████████▌| 1498/1570 [1:58:24<04:45, 3.97s/it]
95%|█████████▌| 1499/1570 [1:58:28<04:42, 3.98s/it]
96%|█████████▌| 1500/1570 [1:58:32<04:39, 3.99s/it]
{'loss': '-0.04146', 'grad_norm': '0.1433', 'learning_rate': '6.033e-07', 'epoch': '4.779'}
+
96%|█████████▌| 1500/1570 [1:58:32<04:39, 3.99s/it]
96%|█████████▌| 1501/1570 [1:58:36<04:33, 3.96s/it]
96%|█████████▌| 1502/1570 [1:58:40<04:30, 3.98s/it]
96%|█████████▌| 1503/1570 [1:58:44<04:25, 3.96s/it]
96%|█████████▌| 1504/1570 [1:58:48<04:21, 3.96s/it]
96%|█████████▌| 1505/1570 [1:58:52<04:18, 3.97s/it]
96%|█████████▌| 1506/1570 [1:58:56<04:16, 4.00s/it]
96%|█████████▌| 1507/1570 [1:59:00<04:11, 3.99s/it]
96%|█████████▌| 1508/1570 [1:59:04<04:07, 3.99s/it]
96%|█████████▌| 1509/1570 [1:59:08<04:03, 3.99s/it]
96%|█████████▌| 1510/1570 [1:59:12<03:56, 3.95s/it]
96%|█████████▌| 1511/1570 [1:59:16<03:52, 3.95s/it]
96%|█████████▋| 1512/1570 [1:59:20<03:49, 3.96s/it]
96%|█████████▋| 1513/1570 [1:59:24<03:45, 3.95s/it]
96%|█████████▋| 1514/1570 [1:59:28<03:41, 3.96s/it]
96%|█████████▋| 1515/1570 [1:59:32<03:39, 3.98s/it]
97%|█████████▋| 1516/1570 [1:59:36<03:34, 3.97s/it]
97%|█████████▋| 1517/1570 [1:59:39<03:28, 3.92s/it]
97%|█████████▋| 1518/1570 [1:59:43<03:22, 3.89s/it]
97%|█████████▋| 1519/1570 [1:59:47<03:15, 3.84s/it]
97%|█████████▋| 1520/1570 [1:59:51<03:12, 3.85s/it]
{'loss': '-0.03695', 'grad_norm': '0.115', 'learning_rate': '3.599e-07', 'epoch': '4.842'}
+
97%|█████████▋| 1520/1570 [1:59:51<03:12, 3.85s/it]
97%|█████████▋| 1521/1570 [1:59:55<03:10, 3.89s/it]
97%|█████████▋| 1522/1570 [1:59:59<03:06, 3.88s/it]
97%|█████████▋| 1523/1570 [2:00:03<03:03, 3.90s/it]
97%|█████████▋| 1524/1570 [2:00:07<02:59, 3.91s/it]
97%|█████████▋| 1525/1570 [2:00:11<02:57, 3.94s/it]
97%|█████████▋| 1526/1570 [2:00:14<02:49, 3.86s/it]
97%|█████████▋| 1527/1570 [2:00:18<02:44, 3.83s/it]
97%|█████████▋| 1528/1570 [2:00:22<02:39, 3.81s/it]
97%|█████████▋| 1529/1570 [2:00:26<02:38, 3.86s/it]
97%|█████████▋| 1530/1570 [2:00:30<02:35, 3.89s/it]
98%|█████████▊| 1531/1570 [2:00:34<02:33, 3.94s/it]
98%|█████████▊| 1532/1570 [2:00:38<02:30, 3.96s/it]
98%|█████████▊| 1533/1570 [2:00:42<02:24, 3.90s/it]
98%|█████████▊| 1534/1570 [2:00:46<02:21, 3.94s/it]
98%|█████████▊| 1535/1570 [2:00:50<02:18, 3.95s/it]
98%|█████████▊| 1536/1570 [2:00:54<02:15, 4.00s/it]
98%|█████████▊| 1537/1570 [2:00:58<02:10, 3.95s/it]
98%|█████████▊| 1538/1570 [2:01:02<02:07, 3.97s/it]
98%|█████████▊| 1539/1570 [2:01:06<02:03, 3.99s/it]
98%|█████████▊| 1540/1570 [2:01:10<01:59, 3.98s/it]
{'loss': '-0.03869', 'grad_norm': '0.1566', 'learning_rate': '1.961e-07', 'epoch': '4.906'}
+
98%|█████████▊| 1540/1570 [2:01:10<01:59, 3.98s/it]
98%|█████████▊| 1541/1570 [2:01:13<01:53, 3.93s/it]
98%|█████████▊| 1542/1570 [2:01:17<01:49, 3.92s/it]
98%|█████████▊| 1543/1570 [2:01:21<01:46, 3.94s/it]
98%|█████████▊| 1544/1570 [2:01:25<01:42, 3.93s/it]
98%|█████████▊| 1545/1570 [2:01:29<01:38, 3.92s/it]
98%|█████████▊| 1546/1570 [2:01:33<01:33, 3.91s/it]
99%|█████████▊| 1547/1570 [2:01:37<01:28, 3.86s/it]
99%|█████████▊| 1548/1570 [2:01:40<01:24, 3.86s/it]
99%|█████████▊| 1549/1570 [2:01:44<01:18, 3.73s/it]
99%|█████████▊| 1550/1570 [2:01:47<01:08, 3.44s/it]
99%|█████████▉| 1551/1570 [2:01:49<01:01, 3.24s/it]
99%|█████████▉| 1552/1570 [2:01:52<00:55, 3.11s/it]
99%|█████████▉| 1553/1570 [2:01:55<00:51, 3.01s/it]
99%|█████████▉| 1554/1570 [2:01:58<00:47, 2.95s/it]
99%|█████████▉| 1555/1570 [2:02:01<00:43, 2.90s/it]
99%|█████████▉| 1556/1570 [2:02:03<00:40, 2.87s/it]
99%|█████████▉| 1557/1570 [2:02:06<00:37, 2.85s/it]
99%|█████████▉| 1558/1570 [2:02:09<00:33, 2.83s/it]
99%|█████████▉| 1559/1570 [2:02:12<00:30, 2.81s/it]
99%|█████████▉| 1560/1570 [2:02:15<00:28, 2.80s/it]
{'loss': '-0.04158', 'grad_norm': '0.1153', 'learning_rate': '1.121e-07', 'epoch': '4.97'}
+
99%|█████████▉| 1560/1570 [2:02:15<00:28, 2.80s/it]
99%|█████████▉| 1561/1570 [2:02:17<00:25, 2.81s/it]
99%|█████████▉| 1562/1570 [2:02:20<00:22, 2.80s/it]
100%|█████████▉| 1563/1570 [2:02:23<00:19, 2.79s/it]
100%|█████████▉| 1564/1570 [2:02:26<00:16, 2.79s/it]
100%|█████████▉| 1565/1570 [2:02:29<00:13, 2.80s/it]
100%|█████████▉| 1566/1570 [2:02:31<00:11, 2.80s/it]
100%|█████████▉| 1567/1570 [2:02:34<00:08, 2.80s/it]
100%|█████████▉| 1568/1570 [2:02:37<00:05, 2.80s/it]
100%|█████████▉| 1569/1570 [2:02:40<00:02, 2.80s/it]
100%|██████████| 1570/1570 [2:02:41<00:00, 2.29s/it][INFO|trainer.py:2668] 2026-08-22 21:38:07,313 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 21:38:07,314 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 21:38:07,314 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/csd/checkpoint-1570
+[INFO|configuration_utils.py:780] 2026-08-22 21:48:10,873 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 21:48:10,874 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 21:48:10,910 >> chat template saved in results/qwen3/csd/checkpoint-1570/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 21:48:10,910 >> tokenizer config file saved in results/qwen3/csd/checkpoint-1570/tokenizer_config.json
+[INFO|trainer.py:1823] 2026-08-22 21:48:11,551 >>
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
{'train_runtime': '7966', 'train_samples_per_second': '6.294', 'train_steps_per_second': '0.197', 'train_loss': '-0.01306', 'epoch': '5'}
+
100%|██████████| 1570/1570 [2:12:45<00:00, 2.29s/it]
100%|██████████| 1570/1570 [2:12:45<00:00, 5.07s/it]
+[INFO|trainer.py:3815] 2026-08-22 21:48:11,928 >> Saving model checkpoint to results/qwen3/csd
+[INFO|configuration_utils.py:780] 2026-08-22 21:48:12,417 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 21:48:12,418 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 21:48:12,454 >> chat template saved in results/qwen3/csd/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 21:48:12,455 >> tokenizer config file saved in results/qwen3/csd/tokenizer_config.json
+***** train metrics *****
+ epoch = 5.0
+ total_flos = 54682091GF
+ train_loss = -0.0131
+ train_runtime = 2:12:45.63
+ train_samples_per_second = 6.294
+ train_steps_per_second = 0.197
+[INFO|trainer.py:2668] 2026-08-22 21:48:12,583 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 21:48:12,583 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 21:48:12,583 >> Batch size = 16
+[transformers] The following generation flags are not valid and may be ignored: ['temperature', 'top_p']. Set `TRANSFORMERS_VERBOSITY=info` for more details.
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|logging.py:340] 2026-08-22 21:48:12,707 >> The following generation flags are not valid and may be ignored: ['temperature', 'top_p'].
+[INFO|logging.py:355] 2026-08-22 21:48:12,708 >> - `temperature`: `do_sample` is not set to `True`. However, `temperature` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.
+- `top_p`: `do_sample` is not set to `True`. However, `top_p` is set to `0.7` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.
+If you're using a pretrained model, note that some of these attributes may be set through the model's `generation_config.json` file.
+[WARNING|utils.py:1642] 2026-08-22 21:48:12,710 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
0%| | 0/107 [00:00> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
2%|▏ | 2/107 [00:05<04:48, 2.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:48:23,920 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
3%|▎ | 3/107 [00:10<06:41, 3.86s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:48:29,345 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
4%|▎ | 4/107 [00:16<07:45, 4.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:48:34,957 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
5%|▍ | 5/107 [00:22<08:15, 4.86s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:48:40,449 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
6%|▌ | 6/107 [00:26<08:10, 4.86s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:48:45,308 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 7/107 [00:32<08:25, 5.05s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:48:50,769 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 8/107 [00:37<08:32, 5.17s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:48:56,213 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
8%|▊ | 9/107 [00:43<08:53, 5.44s/it][WARNING|utils.py:1642] 2026-08-22 21:49:02,243 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
9%|▉ | 10/107 [00:49<08:54, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:49:07,901 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
10%|█ | 11/107 [00:54<08:22, 5.24s/it][WARNING|utils.py:1642] 2026-08-22 21:49:12,519 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
11%|█ | 12/107 [00:59<08:31, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:49:18,234 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
12%|█▏ | 13/107 [01:05<08:32, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:49:23,847 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
13%|█▎ | 14/107 [01:10<08:25, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:49:29,251 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
14%|█▍ | 15/107 [01:16<08:25, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:49:34,875 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
15%|█▍ | 16/107 [01:22<08:22, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:49:40,453 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
16%|█▌ | 17/107 [01:27<08:25, 5.62s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:49:46,299 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
17%|█▋ | 18/107 [01:33<08:21, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:49:51,984 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
18%|█▊ | 19/107 [01:38<08:09, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:49:57,359 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
19%|█▊ | 20/107 [01:44<08:00, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:50:02,790 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
20%|█▉ | 21/107 [01:49<07:35, 5.29s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:50:07,549 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██ | 22/107 [01:54<07:36, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:50:13,097 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██▏ | 23/107 [02:00<07:32, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:50:18,529 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
22%|██▏ | 24/107 [02:05<07:27, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:50:23,933 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
23%|██▎ | 25/107 [02:11<07:47, 5.70s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:50:30,343 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
24%|██▍ | 26/107 [02:17<07:41, 5.70s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:50:36,059 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
25%|██▌ | 27/107 [02:23<07:32, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:50:41,600 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
26%|██▌ | 28/107 [02:28<07:25, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:50:47,211 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
27%|██▋ | 29/107 [02:34<07:27, 5.74s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:50:53,184 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
28%|██▊ | 30/107 [02:40<07:15, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:50:58,629 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
29%|██▉ | 31/107 [02:44<06:43, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:51:03,145 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
30%|██▉ | 32/107 [02:50<06:39, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:51:08,517 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
31%|███ | 33/107 [02:55<06:39, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:51:14,064 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
32%|███▏ | 34/107 [03:01<06:54, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:51:20,399 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
33%|███▎ | 35/107 [03:07<06:45, 5.63s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:51:25,926 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
34%|███▎ | 36/107 [03:12<06:34, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:51:31,290 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
35%|███▍ | 37/107 [03:17<06:10, 5.30s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:51:35,990 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▌ | 38/107 [03:23<06:14, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:51:41,704 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▋ | 39/107 [03:28<06:00, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:51:46,740 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
37%|███▋ | 40/107 [03:33<05:56, 5.32s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:51:52,106 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
38%|███▊ | 41/107 [03:39<05:56, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:51:57,688 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
39%|███▉ | 42/107 [03:45<06:06, 5.63s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:52:03,865 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
40%|████ | 43/107 [03:51<06:07, 5.74s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:52:09,861 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
41%|████ | 44/107 [03:56<05:55, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:52:15,250 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
42%|████▏ | 45/107 [04:01<05:38, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:52:20,314 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
43%|████▎ | 46/107 [04:07<05:30, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:52:25,640 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
44%|████▍ | 47/107 [04:13<05:33, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:52:31,531 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
45%|████▍ | 48/107 [04:18<05:31, 5.62s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:52:37,288 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
46%|████▌ | 49/107 [04:24<05:22, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:52:42,719 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
47%|████▋ | 50/107 [04:29<05:14, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:52:48,122 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
48%|████▊ | 51/107 [04:35<05:12, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:52:53,832 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
49%|████▊ | 52/107 [04:40<05:01, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:52:59,091 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|████▉ | 53/107 [04:46<05:04, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:53:05,100 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|█████ | 54/107 [04:51<04:52, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:53:10,329 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
51%|█████▏ | 55/107 [04:57<04:49, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:53:16,027 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
52%|█████▏ | 56/107 [05:03<04:47, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:53:21,849 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
53%|█████▎ | 57/107 [05:09<04:44, 5.69s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:53:27,626 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
54%|█████▍ | 58/107 [05:14<04:34, 5.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:53:33,044 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
55%|█████▌ | 59/107 [05:20<04:26, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:53:38,491 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
56%|█████▌ | 60/107 [05:25<04:19, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:53:43,903 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
57%|█████▋ | 61/107 [05:30<04:04, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:53:48,733 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
58%|█████▊ | 62/107 [05:35<03:59, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:53:54,101 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
59%|█████▉ | 63/107 [05:41<03:57, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:53:59,657 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
60%|█████▉ | 64/107 [05:46<03:50, 5.36s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:54:04,926 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
61%|██████ | 65/107 [05:51<03:44, 5.35s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:54:10,246 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
62%|██████▏ | 66/107 [05:57<03:40, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:54:15,684 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
63%|██████▎ | 67/107 [06:02<03:35, 5.40s/it][WARNING|utils.py:1642] 2026-08-22 21:54:21,133 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▎ | 68/107 [06:07<03:24, 5.24s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:54:26,025 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▍ | 69/107 [06:12<03:18, 5.23s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:54:31,232 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
65%|██████▌ | 70/107 [06:18<03:22, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:54:37,302 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
66%|██████▋ | 71/107 [06:24<03:20, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:54:43,056 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
67%|██████▋ | 72/107 [06:29<03:12, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:54:48,357 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
68%|██████▊ | 73/107 [06:35<03:06, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:54:53,885 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
69%|██████▉ | 74/107 [06:40<03:00, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:54:59,291 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
70%|███████ | 75/107 [06:46<02:52, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:55:04,533 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
71%|███████ | 76/107 [06:51<02:47, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:55:09,988 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
72%|███████▏ | 77/107 [06:56<02:42, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:55:15,411 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
73%|███████▎ | 78/107 [07:02<02:37, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:55:20,889 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
74%|███████▍ | 79/107 [07:07<02:32, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:55:26,316 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
75%|███████▍ | 80/107 [07:13<02:29, 5.54s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:55:32,089 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
76%|███████▌ | 81/107 [07:19<02:22, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:55:37,447 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
77%|███████▋ | 82/107 [07:24<02:15, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:55:42,705 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
78%|███████▊ | 83/107 [07:29<02:08, 5.36s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:55:47,947 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▊ | 84/107 [07:34<02:02, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:55:53,144 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▉ | 85/107 [07:39<01:50, 5.01s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:55:57,447 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
80%|████████ | 86/107 [07:44<01:48, 5.17s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:56:03,000 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
81%|████████▏ | 87/107 [07:49<01:39, 4.99s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:56:07,547 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
82%|████████▏ | 88/107 [07:54<01:37, 5.12s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:56:12,970 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
83%|████████▎ | 89/107 [08:00<01:39, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:56:19,370 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
84%|████████▍ | 90/107 [08:06<01:32, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:56:24,733 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
85%|████████▌ | 91/107 [08:11<01:25, 5.34s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:56:29,799 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
86%|████████▌ | 92/107 [08:16<01:18, 5.26s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:56:34,873 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
87%|████████▋ | 93/107 [08:21<01:13, 5.27s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:56:40,147 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
88%|████████▊ | 94/107 [08:27<01:09, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:56:45,617 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
89%|████████▉ | 95/107 [08:32<01:03, 5.32s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:56:50,917 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
90%|████████▉ | 96/107 [08:37<00:58, 5.28s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:56:56,107 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
91%|█████████ | 97/107 [08:43<00:53, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:57:01,554 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
92%|█████████▏| 98/107 [08:48<00:48, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:57:07,173 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 99/107 [08:54<00:43, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:57:12,574 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 100/107 [08:59<00:37, 5.35s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:57:17,771 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
94%|█████████▍| 101/107 [09:05<00:32, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:57:23,471 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
95%|█████████▌| 102/107 [09:10<00:27, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:57:28,965 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
96%|█████████▋| 103/107 [09:15<00:21, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:57:34,182 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
97%|█████████▋| 104/107 [09:21<00:16, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:57:39,643 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
98%|█████████▊| 105/107 [09:27<00:11, 5.52s/it][WARNING|utils.py:1642] 2026-08-22 21:57:45,430 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
99%|█████████▉| 106/107 [09:32<00:05, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 21:57:51,115 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
100%|██████████| 107/107 [09:38<00:00, 5.65s/it]
100%|██████████| 107/107 [09:39<00:00, 5.42s/it]
+***** eval metrics *****
+ epoch = 5.0
+ eval_generator_count = 1712.0
+ eval_generator_exact_match = 96.6121
+ eval_generator_rouge1 = 99.9371
+ eval_generator_rouge2 = 99.8891
+ eval_generator_rougeL = 99.7571
+ eval_loss = 0.0041
+ eval_runtime = 0:09:45.46
+ eval_samples_per_second = 5.831
+ eval_selector_accuracy = 98.5981
+ eval_selector_count = 1712.0
+ eval_steps_per_second = 0.183
+[INFO|modelcard.py:263] 2026-08-22 21:57:58,043 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
diff --git a/qwen3/run_all_logs/20260822_031516/da_kd.log b/qwen3/run_all_logs/20260822_031516/da_kd.log
new file mode 100644
index 0000000000000000000000000000000000000000..a0d1fea5d3e0cd87052abbbbc53338169ba182d4
--- /dev/null
+++ b/qwen3/run_all_logs/20260822_031516/da_kd.log
@@ -0,0 +1,2527 @@
+W0823 02:41:03.898000 1406053 torch/distributed/run.py:803]
+W0823 02:41:03.898000 1406053 torch/distributed/run.py:803] *****************************************
+W0823 02:41:03.898000 1406053 torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+W0823 02:41:03.898000 1406053 torch/distributed/run.py:803] *****************************************
+[INFO|2026-08-23 02:41:11] llamafactory.hparams.parser:651 >> Process rank: 1, world size: 2, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16
+[INFO|2026-08-23 02:41:11] llamafactory.hparams.parser:144 >> Set `ddp_find_unused_parameters` to False in DDP training since LoRA is enabled.
+[INFO|2026-08-23 02:41:11] llamafactory.hparams.parser:651 >> Process rank: 0, world size: 2, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16
+Starting da_kd with LlamaFactory and DeepSpeed=True.
+[INFO|configuration_utils.py:780] 2026-08-23 02:41:13,125 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 02:41:13,129 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-23 02:41:16,253 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 02:41:16,256 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-23 02:41:16,493 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 02:41:16,494 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-23 02:41:17] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_train.jsonl...
+[INFO|2026-08-23 02:41:18] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_eval.jsonl...
+training example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 1947, 19029, 10592, 311, 35592, 304, 279, 17328, 1947, 1707, 429, 9482, 1573, 279, 1042, 220, 16, 21, 15, 15, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 9286, 38050, 756, 414, 330, 13193, 788, 341, 286, 330, 408, 14645, 788, 330, 68689, 756, 286, 330, 2468, 14645, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 37525, 756, 414, 330, 13193, 788, 5613, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 18098, 287, 756, 414, 330, 13193, 788, 341, 286, 330, 11141, 3575, 34043, 788, 330, 16468, 756, 286, 330, 37375, 14645, 788, 330, 68689, 698, 414, 456, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 9286, 38050, 756, 414, 330, 1313, 788, 330, 49087, 2354, 698, 262, 1153, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 37525, 756, 414, 330, 1313, 788, 330, 4648, 37525, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of art movements linked to paintings in the genre art style that ended before the year 1600?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "ArtMovement",
+ "properties": {
+ "end_year": "INTEGER",
+ "start_year": "INTEGER"
+ }
+ },
+ {
+ "label": "Genre",
+ "properties": {}
+ },
+ {
+ "label": "Painting",
+ "properties": {
+ "country_of_origin": "STRING",
+ "creation_year": "INTEGER"
+ }
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "ArtMovement",
+ "type": "associatedWith"
+ },
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "Genre",
+ "type": "hasGenre"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+eval example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 9893, 1483, 304, 2987, 279, 220, 17, 15, 17, 16, 55218, 18144, 73379, 476, 279, 23007, 24847, 7755, 9257, 36331, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 13193, 788, 341, 286, 330, 1028, 788, 330, 7097, 756, 286, 330, 31309, 788, 330, 22852, 756, 286, 330, 4082, 3575, 2259, 26286, 788, 330, 68689, 756, 286, 330, 4082, 3575, 1243, 89827, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 28214, 756, 414, 330, 13193, 788, 5613, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 5657, 788, 330, 28214, 756, 414, 330, 1313, 788, 330, 30210, 1047, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of weapons used in either the 2021 Baghdad suicide bombings or the Botroseya Church bombing?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "TerroristAttack",
+ "properties": {
+ "date": "DATE",
+ "locations": "LIST",
+ "number_of_deaths": "INTEGER",
+ "number_of_injuries": "INTEGER"
+ }
+ },
+ {
+ "label": "Weapon",
+ "properties": {}
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "TerroristAttack",
+ "target": "Weapon",
+ "type": "employs"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+[INFO|configuration_utils.py:780] 2026-08-23 02:41:22,349 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 02:41:22,351 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-23 02:41:22] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training.
+[INFO|modeling_utils.py:769] 2026-08-23 02:41:23,065 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/model.safetensors
+[INFO|modeling_utils.py:839] 2026-08-23 02:41:23,065 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-23 02:41:23,066 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": false
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-23 02:41:23,709 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95
+}
+
+[INFO|2026-08-23 02:41:23] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled.
+[INFO|2026-08-23 02:41:23] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-23 02:41:23] llamafactory.model.adapter:144 >> Upcasting trainable params to float32.
+[INFO|2026-08-23 02:41:23] llamafactory.model.adapter:144 >> Fine-tuning method: LoRA
+[INFO|2026-08-23 02:41:23] llamafactory.model.model_utils.misc:144 >> Found linear modules: up_proj,o_proj,gate_proj,k_proj,down_proj,q_proj,v_proj
+[INFO|2026-08-23 02:41:24] llamafactory.model.loader:144 >> trainable params: 20,185,088 || all params: 616,235,008 || trainable%: 3.2756
+[INFO|configuration_utils.py:780] 2026-08-23 02:41:24,642 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 02:41:24,644 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 02:41:27,427 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-23 02:41:27,671 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 02:41:27,673 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-23 02:41:29,678 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 02:41:29,680 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-23 02:41:29] llamafactory.model.model_utils.kv_cache:144 >> KV cache is enabled for faster generation.
+[INFO|modeling_utils.py:769] 2026-08-23 02:41:29,916 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model.safetensors.index.json
+[INFO|modeling_utils.py:839] 2026-08-23 02:41:29,917 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-23 02:41:29,920 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": true
+}
+
+
Loading weights: 0%| | 0/398 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-23 02:41:31,898 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.7,
+ "top_k": 20,
+ "top_p": 0.8
+}
+
+[INFO|2026-08-23 02:41:31] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-23 02:41:32] llamafactory.model.adapter:144 >> Merged 1 adapter(s).
+[INFO|2026-08-23 02:41:32] llamafactory.model.adapter:144 >> Loaded adapter(s): results/qwen3/teacher_lora
+[INFO|2026-08-23 02:41:32] llamafactory.model.loader:144 >> all params: 4,022,468,096
+[INFO|2026-08-23 02:41:32] llamafactory.train.trainer_utils:144 >> Created reference model from Qwen/Qwen3-4B-Instruct-2507
+
Loading weights: 0%| | 0/398 [00:00> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}.
+Gradient accumulation steps mismatch: GradientAccumulationPlugin has 1, DeepSpeed config has 8. Using DeepSpeed's value.
+[INFO|trainer.py:1469] 2026-08-23 02:41:37,872 >> ***** Running training *****
+[INFO|trainer.py:1470] 2026-08-23 02:41:37,873 >> Num examples = 10,027
+[INFO|trainer.py:1471] 2026-08-23 02:41:37,873 >> Num Epochs = 10
+[INFO|trainer.py:1472] 2026-08-23 02:41:37,873 >> Num update steps per epoch = 314
+[INFO|trainer.py:1473] 2026-08-23 02:41:37,873 >> Instantaneous batch size per device = 2
+[INFO|trainer.py:1476] 2026-08-23 02:41:37,873 >> Total train batch size (w. parallel, distributed & accumulation) = 32
+[INFO|trainer.py:1477] 2026-08-23 02:41:37,873 >> Gradient Accumulation steps = 8
+[INFO|trainer.py:1478] 2026-08-23 02:41:37,873 >> Total optimization steps = 1,728
+[INFO|trainer.py:1479] 2026-08-23 02:41:37,876 >> Number of trainable parameters = 20,185,088
+
0%| | 0/1728 [00:00>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 02:56:40,658 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 02:56:40,658 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/da_kd/checkpoint-314
+[INFO|configuration_utils.py:780] 2026-08-23 03:45:39,136 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 03:45:39,137 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 03:45:39,173 >> chat template saved in results/qwen3/da_kd/checkpoint-314/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 03:45:39,173 >> tokenizer config file saved in results/qwen3/da_kd/checkpoint-314/tokenizer_config.json
+DA-KD epoch=2 ratio=0.9755 active=9782/10027 dds_mean=1043926.7598
+
18%|█▊ | 315/1728 [1:29:28<526:26:09, 1341.24s/it]
18%|█▊ | 316/1728 [1:29:30<368:34:55, 939.73s/it]
18%|█▊ | 317/1728 [1:29:33<258:09:40, 658.67s/it]
18%|█▊ | 318/1728 [1:29:36<180:55:32, 461.94s/it]
18%|█▊ | 319/1728 [1:29:39<126:54:12, 324.24s/it]
19%|█▊ | 320/1728 [1:29:42<89:06:27, 227.83s/it]
{'loss': '0.1408', 'grad_norm': '0.3052', 'learning_rate': '9.191e-06', 'epoch': '1.02'}
+
19%|█▊ | 320/1728 [1:29:42<89:06:27, 227.83s/it]
19%|█▊ | 321/1728 [1:29:45<62:40:06, 160.35s/it]
19%|█▊ | 322/1728 [1:29:48<44:10:40, 113.12s/it]
19%|█▊ | 323/1728 [1:29:51<31:14:42, 80.06s/it]
19%|█▉ | 324/1728 [1:29:54<22:11:19, 56.89s/it]
19%|█▉ | 325/1728 [1:29:57<15:51:59, 40.71s/it]
19%|█▉ | 326/1728 [1:29:59<11:26:07, 29.36s/it]
19%|█▉ | 327/1728 [1:30:02<8:20:02, 21.42s/it]
19%|█▉ | 328/1728 [1:30:05<6:09:58, 15.86s/it]
19%|█▉ | 329/1728 [1:30:08<4:39:10, 11.97s/it]
19%|█▉ | 330/1728 [1:30:11<3:35:46, 9.26s/it]
19%|█▉ | 331/1728 [1:30:14<2:50:45, 7.33s/it]
19%|█▉ | 332/1728 [1:30:17<2:19:41, 6.00s/it]
19%|█▉ | 333/1728 [1:30:20<1:58:03, 5.08s/it]
19%|█▉ | 334/1728 [1:30:23<1:42:19, 4.40s/it]
19%|█▉ | 335/1728 [1:30:25<1:31:45, 3.95s/it]
19%|█▉ | 336/1728 [1:30:28<1:24:25, 3.64s/it]
20%|█▉ | 337/1728 [1:30:31<1:19:06, 3.41s/it]
20%|█▉ | 338/1728 [1:30:34<1:15:17, 3.25s/it]
20%|█▉ | 339/1728 [1:30:37<1:12:59, 3.15s/it]
20%|█▉ | 340/1728 [1:30:40<1:11:16, 3.08s/it]
{'loss': '0.1457', 'grad_norm': '0.5786', 'learning_rate': '9.089e-06', 'epoch': '1.085'}
+
20%|█▉ | 340/1728 [1:30:40<1:11:16, 3.08s/it]
20%|█▉ | 341/1728 [1:30:43<1:09:40, 3.01s/it]
20%|█▉ | 342/1728 [1:30:46<1:08:36, 2.97s/it]
20%|█▉ | 343/1728 [1:30:48<1:07:40, 2.93s/it]
20%|█▉ | 344/1728 [1:30:51<1:07:12, 2.91s/it]
20%|█▉ | 345/1728 [1:30:54<1:06:47, 2.90s/it]
20%|██ | 346/1728 [1:30:57<1:06:30, 2.89s/it]
20%|██ | 347/1728 [1:31:00<1:06:32, 2.89s/it]
20%|██ | 348/1728 [1:31:03<1:06:33, 2.89s/it]
20%|██ | 349/1728 [1:31:06<1:06:10, 2.88s/it]
20%|██ | 350/1728 [1:31:09<1:06:05, 2.88s/it]
20%|██ | 351/1728 [1:31:11<1:05:45, 2.87s/it]
20%|██ | 352/1728 [1:31:14<1:05:53, 2.87s/it]
20%|██ | 353/1728 [1:31:17<1:05:50, 2.87s/it]
20%|██ | 354/1728 [1:31:20<1:05:52, 2.88s/it]
21%|██ | 355/1728 [1:31:23<1:05:46, 2.87s/it]
21%|██ | 356/1728 [1:31:26<1:05:51, 2.88s/it]
21%|██ | 357/1728 [1:31:29<1:05:27, 2.86s/it]
21%|██ | 358/1728 [1:31:32<1:05:19, 2.86s/it]
21%|██ | 359/1728 [1:31:34<1:05:09, 2.86s/it]
21%|██ | 360/1728 [1:31:37<1:05:33, 2.88s/it]
{'loss': '0.1348', 'grad_norm': '0.4215', 'learning_rate': '8.983e-06', 'epoch': '1.15'}
+
21%|██ | 360/1728 [1:31:37<1:05:33, 2.88s/it]
21%|██ | 361/1728 [1:31:40<1:05:45, 2.89s/it]
21%|██ | 362/1728 [1:31:43<1:05:23, 2.87s/it]
21%|██ | 363/1728 [1:31:46<1:05:27, 2.88s/it]
21%|██ | 364/1728 [1:31:49<1:05:25, 2.88s/it]
21%|██ | 365/1728 [1:31:52<1:05:17, 2.87s/it]
21%|██ | 366/1728 [1:31:54<1:04:55, 2.86s/it]
21%|██ | 367/1728 [1:31:57<1:05:10, 2.87s/it]
21%|██▏ | 368/1728 [1:32:00<1:04:44, 2.86s/it]
21%|██▏ | 369/1728 [1:32:03<1:04:43, 2.86s/it]
21%|██▏ | 370/1728 [1:32:06<1:04:39, 2.86s/it]
21%|██▏ | 371/1728 [1:32:09<1:04:48, 2.87s/it]
22%|██▏ | 372/1728 [1:32:12<1:04:56, 2.87s/it]
22%|██▏ | 373/1728 [1:32:15<1:05:07, 2.88s/it]
22%|██▏ | 374/1728 [1:32:17<1:04:58, 2.88s/it]
22%|██▏ | 375/1728 [1:32:20<1:04:59, 2.88s/it]
22%|██▏ | 376/1728 [1:32:23<1:04:57, 2.88s/it]
22%|██▏ | 377/1728 [1:32:26<1:04:59, 2.89s/it]
22%|██▏ | 378/1728 [1:32:29<1:05:08, 2.90s/it]
22%|██▏ | 379/1728 [1:32:32<1:05:09, 2.90s/it]
22%|██▏ | 380/1728 [1:32:35<1:05:16, 2.91s/it]
{'loss': '0.1412', 'grad_norm': '0.0941', 'learning_rate': '8.871e-06', 'epoch': '1.216'}
+
22%|██▏ | 380/1728 [1:32:35<1:05:16, 2.91s/it]
22%|██▏ | 381/1728 [1:32:38<1:05:00, 2.90s/it]
22%|██▏ | 382/1728 [1:32:41<1:05:04, 2.90s/it]
22%|██▏ | 383/1728 [1:32:44<1:04:55, 2.90s/it]
22%|██▏ | 384/1728 [1:32:46<1:04:37, 2.89s/it]
22%|██▏ | 385/1728 [1:32:49<1:05:19, 2.92s/it]
22%|██▏ | 386/1728 [1:32:52<1:05:11, 2.91s/it]
22%|██▏ | 387/1728 [1:32:55<1:04:52, 2.90s/it]
22%|██▏ | 388/1728 [1:32:58<1:06:42, 2.99s/it]
23%|██▎ | 389/1728 [1:33:02<1:10:08, 3.14s/it]
23%|██▎ | 390/1728 [1:33:05<1:11:01, 3.18s/it]
23%|██▎ | 391/1728 [1:33:08<1:10:22, 3.16s/it]
23%|██▎ | 392/1728 [1:33:11<1:08:31, 3.08s/it]
23%|██▎ | 393/1728 [1:33:14<1:07:16, 3.02s/it]
23%|██▎ | 394/1728 [1:33:17<1:06:47, 3.00s/it]
23%|██▎ | 395/1728 [1:33:20<1:06:05, 2.98s/it]
23%|██▎ | 396/1728 [1:33:23<1:06:05, 2.98s/it]
23%|██▎ | 397/1728 [1:33:26<1:05:15, 2.94s/it]
23%|██▎ | 398/1728 [1:33:29<1:05:19, 2.95s/it]
23%|██▎ | 399/1728 [1:33:32<1:04:40, 2.92s/it]
23%|██▎ | 400/1728 [1:33:35<1:05:07, 2.94s/it]
{'loss': '0.1378', 'grad_norm': '0.3203', 'learning_rate': '8.754e-06', 'epoch': '1.281'}
+
23%|██▎ | 400/1728 [1:33:35<1:05:07, 2.94s/it]
23%|██▎ | 401/1728 [1:33:37<1:04:39, 2.92s/it]
23%|██▎ | 402/1728 [1:33:40<1:04:44, 2.93s/it]
23%|██▎ | 403/1728 [1:33:43<1:04:07, 2.90s/it]
23%|██▎ | 404/1728 [1:33:46<1:04:04, 2.90s/it]
23%|██▎ | 405/1728 [1:33:49<1:03:57, 2.90s/it]
23%|██▎ | 406/1728 [1:33:52<1:05:27, 2.97s/it]
24%|██▎ | 407/1728 [1:33:55<1:05:18, 2.97s/it]
24%|██▎ | 408/1728 [1:33:58<1:05:27, 2.98s/it]
24%|██▎ | 409/1728 [1:34:01<1:05:12, 2.97s/it]
24%|██▎ | 410/1728 [1:34:04<1:06:34, 3.03s/it]
24%|██▍ | 411/1728 [1:34:07<1:06:20, 3.02s/it]
24%|██▍ | 412/1728 [1:34:10<1:05:34, 2.99s/it]
24%|██▍ | 413/1728 [1:34:13<1:04:41, 2.95s/it]
24%|██▍ | 414/1728 [1:34:16<1:03:56, 2.92s/it]
24%|██▍ | 415/1728 [1:34:19<1:03:43, 2.91s/it]
24%|██▍ | 416/1728 [1:34:22<1:03:11, 2.89s/it]
24%|██▍ | 417/1728 [1:34:25<1:06:22, 3.04s/it]
24%|██▍ | 418/1728 [1:34:28<1:05:40, 3.01s/it]
24%|██▍ | 419/1728 [1:34:31<1:08:57, 3.16s/it]
24%|██▍ | 420/1728 [1:34:34<1:07:32, 3.10s/it]
{'loss': '0.1561', 'grad_norm': '0.2816', 'learning_rate': '8.632e-06', 'epoch': '1.347'}
+
24%|██▍ | 420/1728 [1:34:34<1:07:32, 3.10s/it]
24%|██▍ | 421/1728 [1:34:37<1:06:23, 3.05s/it]
24%|██▍ | 422/1728 [1:34:40<1:05:13, 3.00s/it]
24%|██▍ | 423/1728 [1:34:43<1:04:10, 2.95s/it]
25%|██▍ | 424/1728 [1:34:46<1:03:52, 2.94s/it]
25%|██▍ | 425/1728 [1:34:49<1:03:25, 2.92s/it]
25%|██▍ | 426/1728 [1:34:52<1:03:11, 2.91s/it]
25%|██▍ | 427/1728 [1:34:55<1:03:36, 2.93s/it]
25%|██▍ | 428/1728 [1:34:58<1:03:03, 2.91s/it]
25%|██▍ | 429/1728 [1:35:00<1:02:50, 2.90s/it]
25%|██▍ | 430/1728 [1:35:03<1:03:09, 2.92s/it]
25%|██▍ | 431/1728 [1:35:07<1:04:29, 2.98s/it]
25%|██▌ | 432/1728 [1:35:10<1:05:38, 3.04s/it]
25%|██▌ | 433/1728 [1:35:13<1:06:45, 3.09s/it]
25%|██▌ | 434/1728 [1:35:16<1:05:38, 3.04s/it]
25%|██▌ | 435/1728 [1:35:19<1:04:27, 2.99s/it]
25%|██▌ | 436/1728 [1:35:22<1:06:32, 3.09s/it]
25%|██▌ | 437/1728 [1:35:25<1:04:57, 3.02s/it]
25%|██▌ | 438/1728 [1:35:28<1:03:48, 2.97s/it]
25%|██▌ | 439/1728 [1:35:31<1:03:04, 2.94s/it]
25%|██▌ | 440/1728 [1:35:33<1:02:19, 2.90s/it]
{'loss': '0.1531', 'grad_norm': '0.1099', 'learning_rate': '8.505e-06', 'epoch': '1.412'}
+
25%|██▌ | 440/1728 [1:35:33<1:02:19, 2.90s/it]
26%|██▌ | 441/1728 [1:35:36<1:01:54, 2.89s/it]
26%|██▌ | 442/1728 [1:35:39<1:02:53, 2.93s/it]
26%|██▌ | 443/1728 [1:35:42<1:02:05, 2.90s/it]
26%|██▌ | 444/1728 [1:35:45<1:04:27, 3.01s/it]
26%|██▌ | 445/1728 [1:35:48<1:03:56, 2.99s/it]
26%|██▌ | 446/1728 [1:35:51<1:03:25, 2.97s/it]
26%|██▌ | 447/1728 [1:35:54<1:02:34, 2.93s/it]
26%|██▌ | 448/1728 [1:35:57<1:02:33, 2.93s/it]
26%|██▌ | 449/1728 [1:36:00<1:02:49, 2.95s/it]
26%|██▌ | 450/1728 [1:36:03<1:02:20, 2.93s/it]
26%|██▌ | 451/1728 [1:36:06<1:02:34, 2.94s/it]
26%|██▌ | 452/1728 [1:36:09<1:01:56, 2.91s/it]
26%|██▌ | 453/1728 [1:36:12<1:01:41, 2.90s/it]
26%|██▋ | 454/1728 [1:36:15<1:04:09, 3.02s/it]
26%|██▋ | 455/1728 [1:36:18<1:03:14, 2.98s/it]
26%|██▋ | 456/1728 [1:36:21<1:02:22, 2.94s/it]
26%|██▋ | 457/1728 [1:36:24<1:02:25, 2.95s/it]
27%|██▋ | 458/1728 [1:36:27<1:02:37, 2.96s/it]
27%|██▋ | 459/1728 [1:36:30<1:03:41, 3.01s/it]
27%|██▋ | 460/1728 [1:36:33<1:02:50, 2.97s/it]
{'loss': '0.1618', 'grad_norm': '0.2695', 'learning_rate': '8.374e-06', 'epoch': '1.478'}
+
27%|██▋ | 460/1728 [1:36:33<1:02:50, 2.97s/it]
27%|██▋ | 461/1728 [1:36:36<1:03:28, 3.01s/it]
27%|██▋ | 462/1728 [1:36:39<1:02:49, 2.98s/it]
27%|██▋ | 463/1728 [1:36:42<1:03:39, 3.02s/it]
27%|██▋ | 464/1728 [1:36:45<1:03:40, 3.02s/it]
27%|██▋ | 465/1728 [1:36:48<1:06:12, 3.14s/it]
27%|██▋ | 466/1728 [1:36:51<1:04:25, 3.06s/it]
27%|██▋ | 467/1728 [1:36:54<1:03:15, 3.01s/it]
27%|██▋ | 468/1728 [1:36:57<1:02:21, 2.97s/it]
27%|██▋ | 469/1728 [1:37:00<1:03:34, 3.03s/it]
27%|██▋ | 470/1728 [1:37:03<1:02:13, 2.97s/it]
27%|██▋ | 471/1728 [1:37:06<1:01:26, 2.93s/it]
27%|██▋ | 472/1728 [1:37:09<1:02:11, 2.97s/it]
27%|██▋ | 473/1728 [1:37:12<1:02:26, 2.99s/it]
27%|██▋ | 474/1728 [1:37:15<1:01:41, 2.95s/it]
27%|██▋ | 475/1728 [1:37:18<1:01:21, 2.94s/it]
28%|██▊ | 476/1728 [1:37:21<1:02:59, 3.02s/it]
28%|██▊ | 477/1728 [1:37:24<1:02:06, 2.98s/it]
28%|██▊ | 478/1728 [1:37:26<1:01:21, 2.95s/it]
28%|██▊ | 479/1728 [1:37:29<1:00:44, 2.92s/it]
28%|██▊ | 480/1728 [1:37:32<1:00:18, 2.90s/it]
{'loss': '0.1348', 'grad_norm': '0.07919', 'learning_rate': '8.239e-06', 'epoch': '1.543'}
+
28%|██▊ | 480/1728 [1:37:32<1:00:18, 2.90s/it]
28%|██▊ | 481/1728 [1:37:35<1:00:28, 2.91s/it]
28%|██▊ | 482/1728 [1:37:38<1:00:16, 2.90s/it]
28%|██▊ | 483/1728 [1:37:41<1:00:40, 2.92s/it]
28%|██▊ | 484/1728 [1:37:44<1:00:31, 2.92s/it]
28%|██▊ | 485/1728 [1:37:47<1:00:48, 2.94s/it]
28%|██▊ | 486/1728 [1:37:50<1:02:17, 3.01s/it]
28%|██▊ | 487/1728 [1:37:53<1:01:57, 3.00s/it]
28%|██▊ | 488/1728 [1:37:56<1:02:02, 3.00s/it]
28%|██▊ | 489/1728 [1:37:59<1:01:41, 2.99s/it]
28%|██▊ | 490/1728 [1:38:02<1:01:39, 2.99s/it]
28%|██▊ | 491/1728 [1:38:05<1:01:05, 2.96s/it]
28%|██▊ | 492/1728 [1:38:08<1:00:50, 2.95s/it]
29%|██▊ | 493/1728 [1:38:11<1:00:23, 2.93s/it]
29%|██▊ | 494/1728 [1:38:14<59:43, 2.90s/it]
29%|██▊ | 495/1728 [1:38:17<1:00:18, 2.94s/it]
29%|██▊ | 496/1728 [1:38:19<59:55, 2.92s/it]
29%|██▉ | 497/1728 [1:38:22<1:00:36, 2.95s/it]
29%|██▉ | 498/1728 [1:38:25<1:00:22, 2.95s/it]
29%|██▉ | 499/1728 [1:38:28<59:49, 2.92s/it]
29%|██▉ | 500/1728 [1:38:31<59:24, 2.90s/it]
{'loss': '0.1504', 'grad_norm': '0.5068', 'learning_rate': '8.099e-06', 'epoch': '1.608'}
+
29%|██▉ | 500/1728 [1:38:31<59:24, 2.90s/it]
29%|██▉ | 501/1728 [1:38:34<59:12, 2.90s/it]
29%|██▉ | 502/1728 [1:38:37<59:09, 2.90s/it]
29%|██▉ | 503/1728 [1:38:40<58:53, 2.88s/it]
29%|██▉ | 504/1728 [1:38:43<58:32, 2.87s/it]
29%|██▉ | 505/1728 [1:38:45<58:11, 2.85s/it]
29%|██▉ | 506/1728 [1:38:48<58:14, 2.86s/it]
29%|██▉ | 507/1728 [1:38:51<58:13, 2.86s/it]
29%|██▉ | 508/1728 [1:38:54<59:10, 2.91s/it]
29%|██▉ | 509/1728 [1:38:57<58:46, 2.89s/it]
30%|██▉ | 510/1728 [1:39:00<59:19, 2.92s/it]
30%|██▉ | 511/1728 [1:39:03<59:03, 2.91s/it]
30%|██▉ | 512/1728 [1:39:06<58:54, 2.91s/it]
30%|██▉ | 513/1728 [1:39:09<1:00:57, 3.01s/it]
30%|██▉ | 514/1728 [1:39:12<1:00:28, 2.99s/it]
30%|██▉ | 515/1728 [1:39:15<59:45, 2.96s/it]
30%|██▉ | 516/1728 [1:39:18<59:21, 2.94s/it]
30%|██▉ | 517/1728 [1:39:21<59:15, 2.94s/it]
30%|██▉ | 518/1728 [1:39:24<59:36, 2.96s/it]
30%|███ | 519/1728 [1:39:27<59:20, 2.95s/it]
30%|███ | 520/1728 [1:39:30<59:34, 2.96s/it]
{'loss': '0.1373', 'grad_norm': '0.2842', 'learning_rate': '7.955e-06', 'epoch': '1.674'}
+
30%|███ | 520/1728 [1:39:30<59:34, 2.96s/it]
30%|███ | 521/1728 [1:39:33<1:00:01, 2.98s/it]
30%|███ | 522/1728 [1:39:35<59:09, 2.94s/it]
30%|███ | 523/1728 [1:39:38<58:39, 2.92s/it]
30%|███ | 524/1728 [1:39:41<58:22, 2.91s/it]
30%|███ | 525/1728 [1:39:44<58:04, 2.90s/it]
30%|███ | 526/1728 [1:39:47<59:24, 2.97s/it]
30%|███ | 527/1728 [1:39:50<1:00:19, 3.01s/it]
31%|███ | 528/1728 [1:39:53<1:00:19, 3.02s/it]
31%|███ | 529/1728 [1:39:56<1:00:25, 3.02s/it]
31%|███ | 530/1728 [1:39:59<59:31, 2.98s/it]
31%|███ | 531/1728 [1:40:02<58:59, 2.96s/it]
31%|███ | 532/1728 [1:40:05<59:31, 2.99s/it]
31%|███ | 533/1728 [1:40:08<58:35, 2.94s/it]
31%|███ | 534/1728 [1:40:11<58:40, 2.95s/it]
31%|███ | 535/1728 [1:40:14<58:07, 2.92s/it]
31%|███ | 536/1728 [1:40:17<57:32, 2.90s/it]
31%|███ | 537/1728 [1:40:20<57:21, 2.89s/it]
31%|███ | 538/1728 [1:40:23<57:17, 2.89s/it]
31%|███ | 539/1728 [1:40:26<58:31, 2.95s/it]
31%|███▏ | 540/1728 [1:40:29<58:35, 2.96s/it]
{'loss': '0.1553', 'grad_norm': '0.1749', 'learning_rate': '7.808e-06', 'epoch': '1.739'}
+
31%|███▏ | 540/1728 [1:40:29<58:35, 2.96s/it]
31%|███▏ | 541/1728 [1:40:32<58:12, 2.94s/it]
31%|███▏ | 542/1728 [1:40:35<59:22, 3.00s/it]
31%|███▏ | 543/1728 [1:40:38<59:03, 2.99s/it]
31%|███▏ | 544/1728 [1:40:41<58:46, 2.98s/it]
32%|███▏ | 545/1728 [1:40:43<58:19, 2.96s/it]
32%|███▏ | 546/1728 [1:40:46<57:49, 2.94s/it]
32%|███▏ | 547/1728 [1:40:49<57:45, 2.93s/it]
32%|███▏ | 548/1728 [1:40:52<57:55, 2.95s/it]
32%|███▏ | 549/1728 [1:40:55<57:45, 2.94s/it]
32%|███▏ | 550/1728 [1:40:58<57:06, 2.91s/it]
32%|███▏ | 551/1728 [1:41:01<57:29, 2.93s/it]
32%|███▏ | 552/1728 [1:41:04<57:02, 2.91s/it]
32%|███▏ | 553/1728 [1:41:07<56:51, 2.90s/it]
32%|███▏ | 554/1728 [1:41:10<57:36, 2.94s/it]
32%|███▏ | 555/1728 [1:41:13<57:05, 2.92s/it]
32%|███▏ | 556/1728 [1:41:16<56:48, 2.91s/it]
32%|███▏ | 557/1728 [1:41:18<56:31, 2.90s/it]
32%|███▏ | 558/1728 [1:41:21<57:20, 2.94s/it]
32%|███▏ | 559/1728 [1:41:24<56:46, 2.91s/it]
32%|███▏ | 560/1728 [1:41:27<56:29, 2.90s/it]
{'loss': '0.1385', 'grad_norm': '0.09138', 'learning_rate': '7.656e-06', 'epoch': '1.805'}
+
32%|███▏ | 560/1728 [1:41:27<56:29, 2.90s/it]
32%|███▏ | 561/1728 [1:41:30<56:11, 2.89s/it]
33%|███▎ | 562/1728 [1:41:33<56:26, 2.90s/it]
33%|███▎ | 563/1728 [1:41:36<56:14, 2.90s/it]
33%|███▎ | 564/1728 [1:41:39<56:10, 2.90s/it]
33%|███▎ | 565/1728 [1:41:42<56:17, 2.90s/it]
33%|███▎ | 566/1728 [1:41:45<56:36, 2.92s/it]
33%|███▎ | 567/1728 [1:41:47<55:55, 2.89s/it]
33%|███▎ | 568/1728 [1:41:50<55:37, 2.88s/it]
33%|███▎ | 569/1728 [1:41:53<55:26, 2.87s/it]
33%|███▎ | 570/1728 [1:41:56<57:34, 2.98s/it]
33%|███▎ | 571/1728 [1:41:59<57:16, 2.97s/it]
33%|███▎ | 572/1728 [1:42:02<56:50, 2.95s/it]
33%|███▎ | 573/1728 [1:42:05<56:11, 2.92s/it]
33%|███▎ | 574/1728 [1:42:08<55:56, 2.91s/it]
33%|███▎ | 575/1728 [1:42:11<55:46, 2.90s/it]
33%|███▎ | 576/1728 [1:42:14<55:30, 2.89s/it]
33%|███▎ | 577/1728 [1:42:17<55:12, 2.88s/it]
33%|███▎ | 578/1728 [1:42:19<55:08, 2.88s/it]
34%|███▎ | 579/1728 [1:42:22<55:27, 2.90s/it]
34%|███▎ | 580/1728 [1:42:25<56:16, 2.94s/it]
{'loss': '0.1535', 'grad_norm': '0.2864', 'learning_rate': '7.502e-06', 'epoch': '1.87'}
+
34%|███▎ | 580/1728 [1:42:25<56:16, 2.94s/it]
34%|███▎ | 581/1728 [1:42:29<57:47, 3.02s/it]
34%|███▎ | 582/1728 [1:42:32<56:59, 2.98s/it]
34%|███▎ | 583/1728 [1:42:34<56:05, 2.94s/it]
34%|███▍ | 584/1728 [1:42:37<55:46, 2.93s/it]
34%|███▍ | 585/1728 [1:42:40<55:34, 2.92s/it]
34%|███▍ | 586/1728 [1:42:43<55:09, 2.90s/it]
34%|███▍ | 587/1728 [1:42:46<55:44, 2.93s/it]
34%|███▍ | 588/1728 [1:42:49<56:31, 2.98s/it]
34%|███▍ | 589/1728 [1:42:52<56:35, 2.98s/it]
34%|███▍ | 590/1728 [1:42:55<56:09, 2.96s/it]
34%|███▍ | 591/1728 [1:42:58<55:31, 2.93s/it]
34%|███▍ | 592/1728 [1:43:01<55:37, 2.94s/it]
34%|███▍ | 593/1728 [1:43:04<55:20, 2.93s/it]
34%|███▍ | 594/1728 [1:43:07<55:57, 2.96s/it]
34%|███▍ | 595/1728 [1:43:10<58:52, 3.12s/it]
34%|███▍ | 596/1728 [1:43:13<57:34, 3.05s/it]
35%|███▍ | 597/1728 [1:43:16<57:16, 3.04s/it]
35%|███▍ | 598/1728 [1:43:19<57:24, 3.05s/it]
35%|███▍ | 599/1728 [1:43:22<56:44, 3.02s/it]
35%|███▍ | 600/1728 [1:43:25<55:34, 2.96s/it]
{'loss': '0.1439', 'grad_norm': '0.7404', 'learning_rate': '7.344e-06', 'epoch': '1.935'}
+
35%|███▍ | 600/1728 [1:43:25<55:34, 2.96s/it]
35%|███▍ | 601/1728 [1:43:28<54:51, 2.92s/it]
35%|███▍ | 602/1728 [1:43:31<54:09, 2.89s/it]
35%|███▍ | 603/1728 [1:43:34<54:03, 2.88s/it]
35%|███▍ | 604/1728 [1:43:37<54:45, 2.92s/it]
35%|███▌ | 605/1728 [1:43:39<54:22, 2.90s/it]
35%|███▌ | 606/1728 [1:43:42<53:55, 2.88s/it]
35%|███▌ | 607/1728 [1:43:45<55:49, 2.99s/it]
35%|███▌ | 608/1728 [1:43:48<55:23, 2.97s/it]
35%|███▌ | 609/1728 [1:43:51<55:02, 2.95s/it]
35%|███▌ | 610/1728 [1:43:54<55:32, 2.98s/it]
35%|███▌ | 611/1728 [1:43:57<54:47, 2.94s/it]
35%|███▌ | 612/1728 [1:44:00<55:11, 2.97s/it]
35%|███▌ | 613/1728 [1:44:03<54:35, 2.94s/it]
36%|███▌ | 614/1728 [1:44:06<54:29, 2.93s/it]
36%|███▌ | 615/1728 [1:44:09<54:13, 2.92s/it]
36%|███▌ | 616/1728 [1:44:12<53:39, 2.90s/it]
36%|███▌ | 617/1728 [1:44:15<53:43, 2.90s/it]
36%|███▌ | 618/1728 [1:44:18<54:07, 2.93s/it]
36%|███▌ | 619/1728 [1:44:21<54:58, 2.97s/it]
36%|███▌ | 620/1728 [1:44:23<50:25, 2.73s/it]
{'loss': '0.1413', 'grad_norm': '0.3328', 'learning_rate': '7.183e-06', 'epoch': '2'}
+
36%|███▌ | 620/1728 [1:44:23<50:25, 2.73s/it][INFO|trainer.py:2668] 2026-08-23 04:26:01,271 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 04:26:01,271 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 04:26:01,271 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/da_kd/checkpoint-620
+[INFO|configuration_utils.py:780] 2026-08-23 05:15:14,207 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 05:15:14,209 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 05:15:14,245 >> chat template saved in results/qwen3/da_kd/checkpoint-620/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 05:15:14,245 >> tokenizer config file saved in results/qwen3/da_kd/checkpoint-620/tokenizer_config.json
+DA-KD epoch=3 ratio=0.9045 active=9070/10027 dds_mean=1045528.4657
+
36%|███▌ | 621/1728 [2:59:04<413:55:42, 1346.11s/it]
36%|███▌ | 622/1728 [2:59:06<289:45:16, 943.14s/it]
36%|███▌ | 623/1728 [2:59:09<202:54:39, 661.07s/it]
36%|███▌ | 624/1728 [2:59:12<142:10:32, 463.62s/it]
36%|███▌ | 625/1728 [2:59:15<99:41:44, 325.39s/it]
36%|███▌ | 626/1728 [2:59:18<69:59:31, 228.65s/it]
36%|███▋ | 627/1728 [2:59:21<49:12:52, 160.92s/it]
36%|███▋ | 628/1728 [2:59:24<34:41:00, 113.51s/it]
36%|███▋ | 629/1728 [2:59:27<24:31:07, 80.32s/it]
36%|███▋ | 630/1728 [2:59:30<17:24:49, 57.09s/it]
37%|███▋ | 631/1728 [2:59:33<12:26:55, 40.85s/it]
37%|███▋ | 632/1728 [2:59:35<8:58:22, 29.47s/it]
37%|███▋ | 633/1728 [2:59:38<6:32:27, 21.50s/it]
37%|███▋ | 634/1728 [2:59:41<4:50:10, 15.91s/it]
37%|███▋ | 635/1728 [2:59:44<3:38:47, 12.01s/it]
37%|███▋ | 636/1728 [2:59:47<2:48:48, 9.28s/it]
37%|███▋ | 637/1728 [2:59:50<2:13:58, 7.37s/it]
37%|███▋ | 638/1728 [2:59:53<1:49:12, 6.01s/it]
37%|███▋ | 639/1728 [2:59:56<1:32:25, 5.09s/it]
37%|███▋ | 640/1728 [2:59:59<1:20:27, 4.44s/it]
{'loss': '0.1387', 'grad_norm': '0.2596', 'learning_rate': '7.019e-06', 'epoch': '2.071'}
+
37%|███▋ | 640/1728 [2:59:59<1:20:27, 4.44s/it]
37%|███▋ | 641/1728 [3:00:02<1:12:41, 4.01s/it]
37%|███▋ | 642/1728 [3:00:05<1:06:33, 3.68s/it]
37%|███▋ | 643/1728 [3:00:07<1:02:01, 3.43s/it]
37%|███▋ | 644/1728 [3:00:10<58:55, 3.26s/it]
37%|███▋ | 645/1728 [3:00:13<57:34, 3.19s/it]
37%|███▋ | 646/1728 [3:00:16<55:44, 3.09s/it]
37%|███▋ | 647/1728 [3:00:19<54:29, 3.02s/it]
38%|███▊ | 648/1728 [3:00:22<55:04, 3.06s/it]
38%|███▊ | 649/1728 [3:00:25<53:50, 2.99s/it]
38%|███▊ | 650/1728 [3:00:28<53:50, 3.00s/it]
38%|███▊ | 651/1728 [3:00:31<53:38, 2.99s/it]
38%|███▊ | 652/1728 [3:00:34<54:19, 3.03s/it]
38%|███▊ | 653/1728 [3:00:37<53:51, 3.01s/it]
38%|███▊ | 654/1728 [3:00:40<54:13, 3.03s/it]
38%|███▊ | 655/1728 [3:00:43<53:24, 2.99s/it]
38%|███▊ | 656/1728 [3:00:46<53:03, 2.97s/it]
38%|███▊ | 657/1728 [3:00:49<54:14, 3.04s/it]
38%|███▊ | 658/1728 [3:00:52<53:22, 2.99s/it]
38%|███▊ | 659/1728 [3:00:55<53:00, 2.98s/it]
38%|███▊ | 660/1728 [3:00:58<52:22, 2.94s/it]
{'loss': '0.1368', 'grad_norm': '0.09935', 'learning_rate': '6.852e-06', 'epoch': '2.141'}
+
38%|███▊ | 660/1728 [3:00:58<52:22, 2.94s/it]
38%|███▊ | 661/1728 [3:01:01<53:03, 2.98s/it]
38%|███▊ | 662/1728 [3:01:04<52:35, 2.96s/it]
38%|███▊ | 663/1728 [3:01:07<53:31, 3.02s/it]
38%|███▊ | 664/1728 [3:01:10<52:48, 2.98s/it]
38%|███▊ | 665/1728 [3:01:13<52:04, 2.94s/it]
39%|███▊ | 666/1728 [3:01:16<52:39, 2.97s/it]
39%|███▊ | 667/1728 [3:01:19<52:05, 2.95s/it]
39%|███▊ | 668/1728 [3:01:22<53:28, 3.03s/it]
39%|███▊ | 669/1728 [3:01:25<53:05, 3.01s/it]
39%|███▉ | 670/1728 [3:01:28<52:38, 2.98s/it]
39%|███▉ | 671/1728 [3:01:31<51:58, 2.95s/it]
39%|███▉ | 672/1728 [3:01:34<52:27, 2.98s/it]
39%|███▉ | 673/1728 [3:01:37<53:42, 3.05s/it]
39%|███▉ | 674/1728 [3:01:40<53:21, 3.04s/it]
39%|███▉ | 675/1728 [3:01:43<53:07, 3.03s/it]
39%|███▉ | 676/1728 [3:01:46<53:01, 3.02s/it]
39%|███▉ | 677/1728 [3:01:49<52:20, 2.99s/it]
39%|███▉ | 678/1728 [3:01:52<51:40, 2.95s/it]
39%|███▉ | 679/1728 [3:01:55<51:02, 2.92s/it]
39%|███▉ | 680/1728 [3:01:57<50:41, 2.90s/it]
{'loss': '0.1348', 'grad_norm': '0.18', 'learning_rate': '6.684e-06', 'epoch': '2.212'}
+
39%|███▉ | 680/1728 [3:01:57<50:41, 2.90s/it]
39%|███▉ | 681/1728 [3:02:00<50:33, 2.90s/it]
39%|███▉ | 682/1728 [3:02:03<50:44, 2.91s/it]
40%|███▉ | 683/1728 [3:02:06<50:31, 2.90s/it]
40%|███▉ | 684/1728 [3:02:09<51:35, 2.96s/it]
40%|███▉ | 685/1728 [3:02:12<51:03, 2.94s/it]
40%|███▉ | 686/1728 [3:02:15<51:52, 2.99s/it]
40%|███▉ | 687/1728 [3:02:18<51:59, 3.00s/it]
40%|███▉ | 688/1728 [3:02:21<53:07, 3.06s/it]
40%|███▉ | 689/1728 [3:02:24<52:46, 3.05s/it]
40%|███▉ | 690/1728 [3:02:27<52:10, 3.02s/it]
40%|███▉ | 691/1728 [3:02:30<51:35, 2.98s/it]
40%|████ | 692/1728 [3:02:33<51:03, 2.96s/it]
40%|████ | 693/1728 [3:02:36<50:54, 2.95s/it]
40%|████ | 694/1728 [3:02:39<50:29, 2.93s/it]
40%|████ | 695/1728 [3:02:42<51:08, 2.97s/it]
40%|████ | 696/1728 [3:02:45<50:52, 2.96s/it]
40%|████ | 697/1728 [3:02:48<50:46, 2.95s/it]
40%|████ | 698/1728 [3:02:51<50:25, 2.94s/it]
40%|████ | 699/1728 [3:02:54<50:10, 2.93s/it]
41%|████ | 700/1728 [3:02:57<49:35, 2.89s/it]
{'loss': '0.1386', 'grad_norm': '0.053', 'learning_rate': '6.513e-06', 'epoch': '2.282'}
+
41%|████ | 700/1728 [3:02:57<49:35, 2.89s/it]
41%|████ | 701/1728 [3:02:59<49:37, 2.90s/it]
41%|████ | 702/1728 [3:03:03<51:12, 2.99s/it]
41%|████ | 703/1728 [3:03:06<51:28, 3.01s/it]
41%|████ | 704/1728 [3:03:09<51:49, 3.04s/it]
41%|████ | 705/1728 [3:03:12<51:05, 3.00s/it]
41%|████ | 706/1728 [3:03:15<51:08, 3.00s/it]
41%|████ | 707/1728 [3:03:18<50:49, 2.99s/it]
41%|████ | 708/1728 [3:03:21<50:17, 2.96s/it]
41%|████ | 709/1728 [3:03:23<49:39, 2.92s/it]
41%|████ | 710/1728 [3:03:27<50:57, 3.00s/it]
41%|████ | 711/1728 [3:03:30<50:37, 2.99s/it]
41%|████ | 712/1728 [3:03:32<50:05, 2.96s/it]
41%|████▏ | 713/1728 [3:03:35<49:30, 2.93s/it]
41%|████▏ | 714/1728 [3:03:38<49:34, 2.93s/it]
41%|████▏ | 715/1728 [3:03:41<50:32, 2.99s/it]
41%|████▏ | 716/1728 [3:03:44<50:46, 3.01s/it]
41%|████▏ | 717/1728 [3:03:47<50:02, 2.97s/it]
42%|████▏ | 718/1728 [3:03:50<50:02, 2.97s/it]
42%|████▏ | 719/1728 [3:03:53<49:23, 2.94s/it]
42%|████▏ | 720/1728 [3:03:56<49:14, 2.93s/it]
{'loss': '0.1328', 'grad_norm': '0.5964', 'learning_rate': '6.34e-06', 'epoch': '2.353'}
+
42%|████▏ | 720/1728 [3:03:56<49:14, 2.93s/it]
42%|████▏ | 721/1728 [3:03:59<50:07, 2.99s/it]
42%|████▏ | 722/1728 [3:04:02<50:03, 2.99s/it]
42%|████▏ | 723/1728 [3:04:06<51:45, 3.09s/it]
42%|████▏ | 724/1728 [3:04:08<50:57, 3.05s/it]
42%|████▏ | 725/1728 [3:04:12<51:14, 3.07s/it]
42%|████▏ | 726/1728 [3:04:14<50:08, 3.00s/it]
42%|████▏ | 727/1728 [3:04:17<49:55, 2.99s/it]
42%|████▏ | 728/1728 [3:04:20<49:35, 2.98s/it]
42%|████▏ | 729/1728 [3:04:23<49:12, 2.96s/it]
42%|████▏ | 730/1728 [3:04:26<48:38, 2.92s/it]
42%|████▏ | 731/1728 [3:04:29<48:46, 2.93s/it]
42%|████▏ | 732/1728 [3:04:32<49:15, 2.97s/it]
42%|████▏ | 733/1728 [3:04:35<49:00, 2.96s/it]
42%|████▏ | 734/1728 [3:04:38<48:56, 2.95s/it]
43%|████▎ | 735/1728 [3:04:41<48:55, 2.96s/it]
43%|████▎ | 736/1728 [3:04:44<48:33, 2.94s/it]
43%|████▎ | 737/1728 [3:04:47<48:38, 2.95s/it]
43%|████▎ | 738/1728 [3:04:50<49:22, 2.99s/it]
43%|████▎ | 739/1728 [3:04:53<49:01, 2.97s/it]
43%|████▎ | 740/1728 [3:04:56<48:39, 2.96s/it]
{'loss': '0.1432', 'grad_norm': '0.06561', 'learning_rate': '6.165e-06', 'epoch': '2.423'}
+
43%|████▎ | 740/1728 [3:04:56<48:39, 2.96s/it]
43%|████▎ | 741/1728 [3:04:59<48:25, 2.94s/it]
43%|████▎ | 742/1728 [3:05:02<48:03, 2.92s/it]
43%|████▎ | 743/1728 [3:05:04<47:54, 2.92s/it]
43%|████▎ | 744/1728 [3:05:07<47:51, 2.92s/it]
43%|████▎ | 745/1728 [3:05:10<47:18, 2.89s/it]
43%|████▎ | 746/1728 [3:05:13<47:43, 2.92s/it]
43%|████▎ | 747/1728 [3:05:16<47:57, 2.93s/it]
43%|████▎ | 748/1728 [3:05:19<50:05, 3.07s/it]
43%|████▎ | 749/1728 [3:05:23<50:00, 3.06s/it]
43%|████▎ | 750/1728 [3:05:26<49:52, 3.06s/it]
43%|████▎ | 751/1728 [3:05:28<48:51, 3.00s/it]
44%|████▎ | 752/1728 [3:05:31<48:31, 2.98s/it]
44%|████▎ | 753/1728 [3:05:35<50:38, 3.12s/it]
44%|████▎ | 754/1728 [3:05:38<49:39, 3.06s/it]
44%|████▎ | 755/1728 [3:05:41<48:35, 3.00s/it]
44%|████▍ | 756/1728 [3:05:44<48:38, 3.00s/it]
44%|████▍ | 757/1728 [3:05:47<48:28, 3.00s/it]
44%|████▍ | 758/1728 [3:05:50<48:34, 3.00s/it]
44%|████▍ | 759/1728 [3:05:53<48:01, 2.97s/it]
44%|████▍ | 760/1728 [3:05:55<47:27, 2.94s/it]
{'loss': '0.1456', 'grad_norm': '0.03184', 'learning_rate': '5.989e-06', 'epoch': '2.494'}
+
44%|████▍ | 760/1728 [3:05:55<47:27, 2.94s/it]
44%|████▍ | 761/1728 [3:05:58<48:01, 2.98s/it]
44%|████▍ | 762/1728 [3:06:02<49:05, 3.05s/it]
44%|████▍ | 763/1728 [3:06:05<48:14, 3.00s/it]
44%|████▍ | 764/1728 [3:06:07<47:35, 2.96s/it]
44%|████▍ | 765/1728 [3:06:10<47:17, 2.95s/it]
44%|████▍ | 766/1728 [3:06:13<47:53, 2.99s/it]
44%|████▍ | 767/1728 [3:06:17<48:44, 3.04s/it]
44%|████▍ | 768/1728 [3:06:19<47:44, 2.98s/it]
45%|████▍ | 769/1728 [3:06:23<48:02, 3.01s/it]
45%|████▍ | 770/1728 [3:06:26<50:54, 3.19s/it]
45%|████▍ | 771/1728 [3:06:29<49:30, 3.10s/it]
45%|████▍ | 772/1728 [3:06:32<48:35, 3.05s/it]
45%|████▍ | 773/1728 [3:06:35<49:37, 3.12s/it]
45%|████▍ | 774/1728 [3:06:38<49:07, 3.09s/it]
45%|████▍ | 775/1728 [3:06:41<48:17, 3.04s/it]
45%|████▍ | 776/1728 [3:06:44<47:43, 3.01s/it]
45%|████▍ | 777/1728 [3:06:47<46:58, 2.96s/it]
45%|████▌ | 778/1728 [3:06:50<48:24, 3.06s/it]
45%|████▌ | 779/1728 [3:06:53<47:30, 3.00s/it]
45%|████▌ | 780/1728 [3:06:56<47:07, 2.98s/it]
{'loss': '0.1385', 'grad_norm': '0.03236', 'learning_rate': '5.812e-06', 'epoch': '2.564'}
+
45%|████▌ | 780/1728 [3:06:56<47:07, 2.98s/it]
45%|████▌ | 781/1728 [3:06:59<46:47, 2.96s/it]
45%|████▌ | 782/1728 [3:07:02<46:27, 2.95s/it]
45%|████▌ | 783/1728 [3:07:05<46:07, 2.93s/it]
45%|████▌ | 784/1728 [3:07:08<46:31, 2.96s/it]
45%|████▌ | 785/1728 [3:07:11<46:16, 2.94s/it]
45%|████▌ | 786/1728 [3:07:14<46:37, 2.97s/it]
46%|████▌ | 787/1728 [3:07:17<46:53, 2.99s/it]
46%|████▌ | 788/1728 [3:07:20<46:39, 2.98s/it]
46%|████▌ | 789/1728 [3:07:23<46:00, 2.94s/it]
46%|████▌ | 790/1728 [3:07:26<46:56, 3.00s/it]
46%|████▌ | 791/1728 [3:07:29<47:27, 3.04s/it]
46%|████▌ | 792/1728 [3:07:32<47:11, 3.03s/it]
46%|████▌ | 793/1728 [3:07:35<47:15, 3.03s/it]
46%|████▌ | 794/1728 [3:07:38<47:25, 3.05s/it]
46%|████▌ | 795/1728 [3:07:41<46:53, 3.02s/it]
46%|████▌ | 796/1728 [3:07:44<46:13, 2.98s/it]
46%|████▌ | 797/1728 [3:07:47<45:59, 2.96s/it]
46%|████▌ | 798/1728 [3:07:50<48:06, 3.10s/it]
46%|████▌ | 799/1728 [3:07:53<48:18, 3.12s/it]
46%|████▋ | 800/1728 [3:07:56<47:24, 3.06s/it]
{'loss': '0.1314', 'grad_norm': '0.2541', 'learning_rate': '5.634e-06', 'epoch': '2.635'}
+
46%|████▋ | 800/1728 [3:07:56<47:24, 3.06s/it]
46%|████▋ | 801/1728 [3:07:59<46:29, 3.01s/it]
46%|████▋ | 802/1728 [3:08:02<47:08, 3.05s/it]
46%|████▋ | 803/1728 [3:08:05<47:19, 3.07s/it]
47%|████▋ | 804/1728 [3:08:08<46:32, 3.02s/it]
47%|████▋ | 805/1728 [3:08:11<45:43, 2.97s/it]
47%|████▋ | 806/1728 [3:08:14<45:35, 2.97s/it]
47%|████▋ | 807/1728 [3:08:17<47:18, 3.08s/it]
47%|████▋ | 808/1728 [3:08:21<48:30, 3.16s/it]
47%|████▋ | 809/1728 [3:08:24<48:08, 3.14s/it]
47%|████▋ | 810/1728 [3:08:27<47:21, 3.10s/it]
47%|████▋ | 811/1728 [3:08:30<46:28, 3.04s/it]
47%|████▋ | 812/1728 [3:08:33<45:40, 2.99s/it]
47%|████▋ | 813/1728 [3:08:36<46:09, 3.03s/it]
47%|████▋ | 814/1728 [3:08:39<46:27, 3.05s/it]
47%|████▋ | 815/1728 [3:08:42<45:44, 3.01s/it]
47%|████▋ | 816/1728 [3:08:45<45:20, 2.98s/it]
47%|████▋ | 817/1728 [3:08:48<45:18, 2.98s/it]
47%|████▋ | 818/1728 [3:08:51<44:38, 2.94s/it]
47%|████▋ | 819/1728 [3:08:53<44:24, 2.93s/it]
47%|████▋ | 820/1728 [3:08:56<43:59, 2.91s/it]
{'loss': '0.1449', 'grad_norm': '0.02482', 'learning_rate': '5.455e-06', 'epoch': '2.705'}
+
47%|████▋ | 820/1728 [3:08:56<43:59, 2.91s/it]
48%|████▊ | 821/1728 [3:08:59<43:49, 2.90s/it]
48%|████▊ | 822/1728 [3:09:02<45:21, 3.00s/it]
48%|████▊ | 823/1728 [3:09:05<44:45, 2.97s/it]
48%|████▊ | 824/1728 [3:09:09<45:34, 3.02s/it]
48%|████▊ | 825/1728 [3:09:12<46:52, 3.12s/it]
48%|████▊ | 826/1728 [3:09:15<45:57, 3.06s/it]
48%|████▊ | 827/1728 [3:09:18<45:17, 3.02s/it]
48%|████▊ | 828/1728 [3:09:21<45:08, 3.01s/it]
48%|████▊ | 829/1728 [3:09:24<44:38, 2.98s/it]
48%|████▊ | 830/1728 [3:09:26<43:55, 2.93s/it]
48%|████▊ | 831/1728 [3:09:30<45:08, 3.02s/it]
48%|████▊ | 832/1728 [3:09:33<44:32, 2.98s/it]
48%|████▊ | 833/1728 [3:09:36<46:32, 3.12s/it]
48%|████▊ | 834/1728 [3:09:39<45:25, 3.05s/it]
48%|████▊ | 835/1728 [3:09:42<44:40, 3.00s/it]
48%|████▊ | 836/1728 [3:09:45<45:23, 3.05s/it]
48%|████▊ | 837/1728 [3:09:48<45:14, 3.05s/it]
48%|████▊ | 838/1728 [3:09:51<44:57, 3.03s/it]
49%|████▊ | 839/1728 [3:09:54<44:56, 3.03s/it]
49%|████▊ | 840/1728 [3:09:57<44:11, 2.99s/it]
{'loss': '0.1485', 'grad_norm': '0.09887', 'learning_rate': '5.275e-06', 'epoch': '2.776'}
+
49%|████▊ | 840/1728 [3:09:57<44:11, 2.99s/it]
49%|████▊ | 841/1728 [3:10:00<44:19, 3.00s/it]
49%|████▊ | 842/1728 [3:10:03<44:10, 2.99s/it]
49%|████▉ | 843/1728 [3:10:06<43:41, 2.96s/it]
49%|████▉ | 844/1728 [3:10:09<43:16, 2.94s/it]
49%|████▉ | 845/1728 [3:10:12<43:24, 2.95s/it]
49%|████▉ | 846/1728 [3:10:14<42:57, 2.92s/it]
49%|████▉ | 847/1728 [3:10:17<43:16, 2.95s/it]
49%|████▉ | 848/1728 [3:10:21<44:58, 3.07s/it]
49%|████▉ | 849/1728 [3:10:24<44:38, 3.05s/it]
49%|████▉ | 850/1728 [3:10:27<44:39, 3.05s/it]
49%|████▉ | 851/1728 [3:10:30<44:19, 3.03s/it]
49%|████▉ | 852/1728 [3:10:33<45:03, 3.09s/it]
49%|████▉ | 853/1728 [3:10:36<43:52, 3.01s/it]
49%|████▉ | 854/1728 [3:10:39<46:07, 3.17s/it]
49%|████▉ | 855/1728 [3:10:42<44:46, 3.08s/it]
50%|████▉ | 856/1728 [3:10:45<43:58, 3.03s/it]
50%|████▉ | 857/1728 [3:10:48<43:36, 3.00s/it]
50%|████▉ | 858/1728 [3:10:51<43:07, 2.97s/it]
50%|████▉ | 859/1728 [3:10:54<42:40, 2.95s/it]
50%|████▉ | 860/1728 [3:10:57<42:30, 2.94s/it]
{'loss': '0.1438', 'grad_norm': '0.737', 'learning_rate': '5.095e-06', 'epoch': '2.847'}
+
50%|████▉ | 860/1728 [3:10:57<42:30, 2.94s/it]
50%|████▉ | 861/1728 [3:11:00<43:04, 2.98s/it]
50%|████▉ | 862/1728 [3:11:03<43:49, 3.04s/it]
50%|████▉ | 863/1728 [3:11:06<44:27, 3.08s/it]
50%|█████ | 864/1728 [3:11:09<43:42, 3.04s/it]
50%|█████ | 865/1728 [3:11:13<44:41, 3.11s/it]
50%|█████ | 866/1728 [3:11:16<44:18, 3.08s/it]
50%|█████ | 867/1728 [3:11:18<43:34, 3.04s/it]
50%|█████ | 868/1728 [3:11:21<42:57, 3.00s/it]
50%|█████ | 869/1728 [3:11:24<43:00, 3.00s/it]
50%|█████ | 870/1728 [3:11:27<42:31, 2.97s/it]
50%|█████ | 871/1728 [3:11:30<42:17, 2.96s/it]
50%|█████ | 872/1728 [3:11:33<41:51, 2.93s/it]
51%|█████ | 873/1728 [3:11:36<41:30, 2.91s/it]
51%|█████ | 874/1728 [3:11:39<41:24, 2.91s/it]
51%|█████ | 875/1728 [3:11:42<41:18, 2.91s/it]
51%|█████ | 876/1728 [3:11:45<41:00, 2.89s/it]
51%|█████ | 877/1728 [3:11:47<40:55, 2.89s/it]
51%|█████ | 878/1728 [3:11:50<40:52, 2.89s/it]
51%|█████ | 879/1728 [3:11:53<40:28, 2.86s/it]
51%|█████ | 880/1728 [3:11:56<40:41, 2.88s/it]
{'loss': '0.1391', 'grad_norm': '0.06732', 'learning_rate': '4.915e-06', 'epoch': '2.917'}
+
51%|█████ | 880/1728 [3:11:56<40:41, 2.88s/it]
51%|█████ | 881/1728 [3:11:59<40:47, 2.89s/it]
51%|█████ | 882/1728 [3:12:02<40:49, 2.90s/it]
51%|█████ | 883/1728 [3:12:05<40:58, 2.91s/it]
51%|█████ | 884/1728 [3:12:08<40:47, 2.90s/it]
51%|█████ | 885/1728 [3:12:11<40:45, 2.90s/it]
51%|█████▏ | 886/1728 [3:12:14<40:42, 2.90s/it]
51%|█████▏ | 887/1728 [3:12:16<40:31, 2.89s/it]
51%|█████▏ | 888/1728 [3:12:19<40:13, 2.87s/it]
51%|█████▏ | 889/1728 [3:12:22<39:57, 2.86s/it]
52%|█████▏ | 890/1728 [3:12:25<40:16, 2.88s/it]
52%|█████▏ | 891/1728 [3:12:28<40:14, 2.88s/it]
52%|█████▏ | 892/1728 [3:12:31<40:18, 2.89s/it]
52%|█████▏ | 893/1728 [3:12:34<40:11, 2.89s/it]
52%|█████▏ | 894/1728 [3:12:37<40:12, 2.89s/it]
52%|█████▏ | 895/1728 [3:12:39<40:03, 2.89s/it]
52%|█████▏ | 896/1728 [3:12:42<40:08, 2.90s/it]
52%|█████▏ | 897/1728 [3:12:45<40:12, 2.90s/it]
52%|█████▏ | 898/1728 [3:12:48<39:58, 2.89s/it]
52%|█████▏ | 899/1728 [3:12:51<39:54, 2.89s/it]
52%|█████▏ | 900/1728 [3:12:54<39:56, 2.89s/it]
{'loss': '0.1466', 'grad_norm': '0.1113', 'learning_rate': '4.735e-06', 'epoch': '2.988'}
+
52%|█████▏ | 900/1728 [3:12:54<39:56, 2.89s/it]
52%|█████▏ | 901/1728 [3:12:57<39:43, 2.88s/it]
52%|█████▏ | 902/1728 [3:13:00<39:35, 2.88s/it]
52%|█████▏ | 903/1728 [3:13:03<39:38, 2.88s/it]
52%|█████▏ | 904/1728 [3:13:04<33:58, 2.47s/it][INFO|trainer.py:2668] 2026-08-23 05:54:42,470 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 05:54:42,471 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 05:54:42,471 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/da_kd/checkpoint-904
+[INFO|configuration_utils.py:780] 2026-08-23 06:42:35,244 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 06:42:35,247 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 06:42:35,282 >> chat template saved in results/qwen3/da_kd/checkpoint-904/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 06:42:35,282 >> tokenizer config file saved in results/qwen3/da_kd/checkpoint-904/tokenizer_config.json
+DA-KD epoch=4 ratio=0.7939 active=7960/10027 dds_mean=1042129.4140
+
52%|█████▏ | 905/1728 [4:26:26<302:16:33, 1322.23s/it]
52%|█████▏ | 906/1728 [4:26:29<211:31:54, 926.42s/it]
52%|█████▏ | 907/1728 [4:26:31<148:05:18, 649.35s/it]
53%|█████▎ | 908/1728 [4:26:34<103:43:48, 455.40s/it]
53%|█████▎ | 909/1728 [4:26:37<72:43:13, 319.65s/it]
53%|█████▎ | 910/1728 [4:26:40<51:02:15, 224.62s/it]
53%|█████▎ | 911/1728 [4:26:43<35:52:39, 158.09s/it]
53%|█████▎ | 912/1728 [4:26:46<25:16:45, 111.53s/it]
53%|█████▎ | 913/1728 [4:26:49<17:51:51, 78.91s/it]
53%|█████▎ | 914/1728 [4:26:52<12:41:08, 56.10s/it]
53%|█████▎ | 915/1728 [4:26:54<9:03:51, 40.14s/it]
53%|█████▎ | 916/1728 [4:26:57<6:31:56, 28.96s/it]
53%|█████▎ | 917/1728 [4:27:00<4:45:31, 21.12s/it]
53%|█████▎ | 918/1728 [4:27:03<3:31:02, 15.63s/it]
53%|█████▎ | 919/1728 [4:27:06<2:39:06, 11.80s/it]
53%|█████▎ | 920/1728 [4:27:09<2:02:49, 9.12s/it]
{'loss': '0.1698', 'grad_norm': '0.01115', 'learning_rate': '4.556e-06', 'epoch': '3.064'}
+
53%|█████▎ | 920/1728 [4:27:09<2:02:49, 9.12s/it]
53%|█████▎ | 921/1728 [4:27:12<1:37:31, 7.25s/it]
53%|█████▎ | 922/1728 [4:27:14<1:19:49, 5.94s/it]
53%|█████▎ | 923/1728 [4:27:17<1:07:34, 5.04s/it]
53%|█████▎ | 924/1728 [4:27:20<58:40, 4.38s/it]
54%|█████▎ | 925/1728 [4:27:23<52:45, 3.94s/it]
54%|█████▎ | 926/1728 [4:27:26<48:20, 3.62s/it]
54%|█████▎ | 927/1728 [4:27:29<45:07, 3.38s/it]
54%|█████▎ | 928/1728 [4:27:32<43:01, 3.23s/it]
54%|█████▍ | 929/1728 [4:27:35<41:30, 3.12s/it]
54%|█████▍ | 930/1728 [4:27:37<40:33, 3.05s/it]
54%|█████▍ | 931/1728 [4:27:40<39:43, 2.99s/it]
54%|█████▍ | 932/1728 [4:27:43<39:08, 2.95s/it]
54%|█████▍ | 933/1728 [4:27:46<38:43, 2.92s/it]
54%|█████▍ | 934/1728 [4:27:49<38:32, 2.91s/it]
54%|█████▍ | 935/1728 [4:27:52<38:24, 2.91s/it]
54%|█████▍ | 936/1728 [4:27:55<38:22, 2.91s/it]
54%|█████▍ | 937/1728 [4:27:58<38:22, 2.91s/it]
54%|█████▍ | 938/1728 [4:28:00<38:05, 2.89s/it]
54%|█████▍ | 939/1728 [4:28:03<37:58, 2.89s/it]
54%|█████▍ | 940/1728 [4:28:06<37:44, 2.87s/it]
{'loss': '0.134', 'grad_norm': '0.1791', 'learning_rate': '4.377e-06', 'epoch': '3.145'}
+
54%|█████▍ | 940/1728 [4:28:06<37:44, 2.87s/it]
54%|█████▍ | 941/1728 [4:28:09<37:39, 2.87s/it]
55%|█████▍ | 942/1728 [4:28:12<37:35, 2.87s/it]
55%|█████▍ | 943/1728 [4:28:15<37:31, 2.87s/it]
55%|█████▍ | 944/1728 [4:28:18<37:18, 2.85s/it]
55%|█████▍ | 945/1728 [4:28:20<37:14, 2.85s/it]
55%|█████▍ | 946/1728 [4:28:23<37:03, 2.84s/it]
55%|█████▍ | 947/1728 [4:28:26<37:12, 2.86s/it]
55%|█████▍ | 948/1728 [4:28:29<37:11, 2.86s/it]
55%|█████▍ | 949/1728 [4:28:32<37:07, 2.86s/it]
55%|█████▍ | 950/1728 [4:28:35<37:01, 2.86s/it]
55%|█████▌ | 951/1728 [4:28:38<37:21, 2.88s/it]
55%|█████▌ | 952/1728 [4:28:41<37:22, 2.89s/it]
55%|█████▌ | 953/1728 [4:28:43<37:22, 2.89s/it]
55%|█████▌ | 954/1728 [4:28:46<37:17, 2.89s/it]
55%|█████▌ | 955/1728 [4:28:49<37:16, 2.89s/it]
55%|█████▌ | 956/1728 [4:28:52<37:08, 2.89s/it]
55%|█████▌ | 957/1728 [4:28:55<36:43, 2.86s/it]
55%|█████▌ | 958/1728 [4:28:58<36:54, 2.88s/it]
55%|█████▌ | 959/1728 [4:29:01<36:45, 2.87s/it]
56%|█████▌ | 960/1728 [4:29:04<36:40, 2.86s/it]
{'loss': '0.1646', 'grad_norm': '0.01232', 'learning_rate': '4.199e-06', 'epoch': '3.225'}
+
56%|█████▌ | 960/1728 [4:29:04<36:40, 2.86s/it]
56%|█████▌ | 961/1728 [4:29:06<36:40, 2.87s/it]
56%|█████▌ | 962/1728 [4:29:09<36:33, 2.86s/it]
56%|█████▌ | 963/1728 [4:29:12<36:19, 2.85s/it]
56%|█████▌ | 964/1728 [4:29:15<36:25, 2.86s/it]
56%|█████▌ | 965/1728 [4:29:18<36:20, 2.86s/it]
56%|█████▌ | 966/1728 [4:29:21<36:38, 2.89s/it]
56%|█████▌ | 967/1728 [4:29:24<36:40, 2.89s/it]
56%|█████▌ | 968/1728 [4:29:27<36:37, 2.89s/it]
56%|█████▌ | 969/1728 [4:29:29<36:25, 2.88s/it]
56%|█████▌ | 970/1728 [4:29:32<36:23, 2.88s/it]
56%|█████▌ | 971/1728 [4:29:35<36:17, 2.88s/it]
56%|█████▋ | 972/1728 [4:29:38<36:10, 2.87s/it]
56%|█████▋ | 973/1728 [4:29:41<36:02, 2.86s/it]
56%|█████▋ | 974/1728 [4:29:44<36:09, 2.88s/it]
56%|█████▋ | 975/1728 [4:29:47<36:21, 2.90s/it]
56%|█████▋ | 976/1728 [4:29:50<36:14, 2.89s/it]
57%|█████▋ | 977/1728 [4:29:52<36:05, 2.88s/it]
57%|█████▋ | 978/1728 [4:29:55<36:03, 2.88s/it]
57%|█████▋ | 979/1728 [4:29:58<35:50, 2.87s/it]
57%|█████▋ | 980/1728 [4:30:01<36:00, 2.89s/it]
{'loss': '0.1543', 'grad_norm': '0.0355', 'learning_rate': '4.023e-06', 'epoch': '3.306'}
+
57%|█████▋ | 980/1728 [4:30:01<36:00, 2.89s/it]
57%|█████▋ | 981/1728 [4:30:04<35:59, 2.89s/it]
57%|█████▋ | 982/1728 [4:30:07<35:52, 2.89s/it]
57%|█████▋ | 983/1728 [4:30:10<35:41, 2.87s/it]
57%|█████▋ | 984/1728 [4:30:13<35:43, 2.88s/it]
57%|█████▋ | 985/1728 [4:30:15<35:28, 2.86s/it]
57%|█████▋ | 986/1728 [4:30:18<35:23, 2.86s/it]
57%|█████▋ | 987/1728 [4:30:21<35:16, 2.86s/it]
57%|█████▋ | 988/1728 [4:30:24<35:21, 2.87s/it]
57%|█████▋ | 989/1728 [4:30:27<35:17, 2.87s/it]
57%|█████▋ | 990/1728 [4:30:30<35:16, 2.87s/it]
57%|█████▋ | 991/1728 [4:30:33<35:16, 2.87s/it]
57%|█████▋ | 992/1728 [4:30:36<35:02, 2.86s/it]
57%|█████▋ | 993/1728 [4:30:38<34:59, 2.86s/it]
58%|█████▊ | 994/1728 [4:30:41<35:00, 2.86s/it]
58%|█████▊ | 995/1728 [4:30:44<35:00, 2.87s/it]
58%|█████▊ | 996/1728 [4:30:47<34:59, 2.87s/it]
58%|█████▊ | 997/1728 [4:30:50<34:40, 2.85s/it]
58%|█████▊ | 998/1728 [4:30:53<34:45, 2.86s/it]
58%|█████▊ | 999/1728 [4:30:56<34:46, 2.86s/it]
58%|█████▊ | 1000/1728 [4:30:58<34:46, 2.87s/it]
{'loss': '0.1552', 'grad_norm': '0.009848', 'learning_rate': '3.847e-06', 'epoch': '3.386'}
+
58%|█████▊ | 1000/1728 [4:30:58<34:46, 2.87s/it]
58%|█████▊ | 1001/1728 [4:31:01<34:48, 2.87s/it]
58%|█████▊ | 1002/1728 [4:31:04<34:48, 2.88s/it]
58%|█████▊ | 1003/1728 [4:31:07<34:31, 2.86s/it]
58%|█████▊ | 1004/1728 [4:31:10<34:22, 2.85s/it]
58%|█████▊ | 1005/1728 [4:31:13<34:26, 2.86s/it]
58%|█████▊ | 1006/1728 [4:31:16<34:20, 2.85s/it]
58%|█████▊ | 1007/1728 [4:31:18<34:20, 2.86s/it]
58%|█████▊ | 1008/1728 [4:31:21<34:22, 2.86s/it]
58%|█████▊ | 1009/1728 [4:31:24<34:14, 2.86s/it]
58%|█████▊ | 1010/1728 [4:31:27<34:16, 2.86s/it]
59%|█████▊ | 1011/1728 [4:31:30<34:22, 2.88s/it]
59%|█████▊ | 1012/1728 [4:31:33<34:12, 2.87s/it]
59%|█████▊ | 1013/1728 [4:31:36<34:12, 2.87s/it]
59%|█████▊ | 1014/1728 [4:31:39<34:10, 2.87s/it]
59%|█████▊ | 1015/1728 [4:31:41<34:09, 2.87s/it]
59%|█████▉ | 1016/1728 [4:31:44<34:10, 2.88s/it]
59%|█████▉ | 1017/1728 [4:31:47<34:09, 2.88s/it]
59%|█████▉ | 1018/1728 [4:31:50<34:07, 2.88s/it]
59%|█████▉ | 1019/1728 [4:31:53<34:00, 2.88s/it]
59%|█████▉ | 1020/1728 [4:31:56<33:52, 2.87s/it]
{'loss': '0.1526', 'grad_norm': '0.197', 'learning_rate': '3.673e-06', 'epoch': '3.466'}
+
59%|█████▉ | 1020/1728 [4:31:56<33:52, 2.87s/it]
59%|█████▉ | 1021/1728 [4:31:59<33:52, 2.87s/it]
59%|█████▉ | 1022/1728 [4:32:02<33:47, 2.87s/it]
59%|█████▉ | 1023/1728 [4:32:04<33:45, 2.87s/it]
59%|█████▉ | 1024/1728 [4:32:07<33:39, 2.87s/it]
59%|█████▉ | 1025/1728 [4:32:10<33:33, 2.86s/it]
59%|█████▉ | 1026/1728 [4:32:13<33:33, 2.87s/it]
59%|█████▉ | 1027/1728 [4:32:16<33:36, 2.88s/it]
59%|█████▉ | 1028/1728 [4:32:19<33:39, 2.89s/it]
60%|█████▉ | 1029/1728 [4:32:22<33:39, 2.89s/it]
60%|█████▉ | 1030/1728 [4:32:25<33:30, 2.88s/it]
60%|█████▉ | 1031/1728 [4:32:27<33:24, 2.88s/it]
60%|█████▉ | 1032/1728 [4:32:30<33:07, 2.86s/it]
60%|█████▉ | 1033/1728 [4:32:33<33:16, 2.87s/it]
60%|█████▉ | 1034/1728 [4:32:36<33:10, 2.87s/it]
60%|█████▉ | 1035/1728 [4:32:39<32:54, 2.85s/it]
60%|█████▉ | 1036/1728 [4:32:42<32:48, 2.85s/it]
60%|██████ | 1037/1728 [4:32:45<32:50, 2.85s/it]
60%|██████ | 1038/1728 [4:32:47<32:49, 2.85s/it]
60%|██████ | 1039/1728 [4:32:50<32:55, 2.87s/it]
60%|██████ | 1040/1728 [4:32:53<32:43, 2.85s/it]
{'loss': '0.1523', 'grad_norm': '0.008719', 'learning_rate': '3.502e-06', 'epoch': '3.547'}
+
60%|██████ | 1040/1728 [4:32:53<32:43, 2.85s/it]
60%|██████ | 1041/1728 [4:32:56<32:50, 2.87s/it]
60%|██████ | 1042/1728 [4:32:59<32:43, 2.86s/it]
60%|██████ | 1043/1728 [4:33:02<32:49, 2.87s/it]
60%|██████ | 1044/1728 [4:33:05<32:38, 2.86s/it]
60%|██████ | 1045/1728 [4:33:08<32:48, 2.88s/it]
61%|██████ | 1046/1728 [4:33:10<32:40, 2.87s/it]
61%|██████ | 1047/1728 [4:33:13<32:37, 2.87s/it]
61%|██████ | 1048/1728 [4:33:16<32:30, 2.87s/it]
61%|██████ | 1049/1728 [4:33:19<32:32, 2.88s/it]
61%|██████ | 1050/1728 [4:33:22<32:26, 2.87s/it]
61%|██████ | 1051/1728 [4:33:25<32:24, 2.87s/it]
61%|██████ | 1052/1728 [4:33:28<32:23, 2.87s/it]
61%|██████ | 1053/1728 [4:33:30<32:21, 2.88s/it]
61%|██████ | 1054/1728 [4:33:33<32:08, 2.86s/it]
61%|██████ | 1055/1728 [4:33:36<32:01, 2.85s/it]
61%|██████ | 1056/1728 [4:33:39<32:00, 2.86s/it]
61%|██████ | 1057/1728 [4:33:42<32:05, 2.87s/it]
61%|██████ | 1058/1728 [4:33:45<31:47, 2.85s/it]
61%|██████▏ | 1059/1728 [4:33:48<31:51, 2.86s/it]
61%|██████▏ | 1060/1728 [4:33:50<31:49, 2.86s/it]
{'loss': '0.153', 'grad_norm': '0.02149', 'learning_rate': '3.332e-06', 'epoch': '3.627'}
+
61%|██████▏ | 1060/1728 [4:33:50<31:49, 2.86s/it]
61%|██████▏ | 1061/1728 [4:33:53<31:50, 2.86s/it]
61%|██████▏ | 1062/1728 [4:33:56<31:48, 2.87s/it]
62%|██████▏ | 1063/1728 [4:33:59<31:40, 2.86s/it]
62%|██████▏ | 1064/1728 [4:34:02<31:41, 2.86s/it]
62%|██████▏ | 1065/1728 [4:34:05<31:41, 2.87s/it]
62%|██████▏ | 1066/1728 [4:34:08<31:37, 2.87s/it]
62%|██████▏ | 1067/1728 [4:34:11<31:31, 2.86s/it]
62%|██████▏ | 1068/1728 [4:34:13<31:35, 2.87s/it]
62%|██████▏ | 1069/1728 [4:34:16<31:26, 2.86s/it]
62%|██████▏ | 1070/1728 [4:34:19<31:23, 2.86s/it]
62%|██████▏ | 1071/1728 [4:34:22<31:24, 2.87s/it]
62%|██████▏ | 1072/1728 [4:34:25<31:22, 2.87s/it]
62%|██████▏ | 1073/1728 [4:34:28<31:17, 2.87s/it]
62%|██████▏ | 1074/1728 [4:34:31<31:08, 2.86s/it]
62%|██████▏ | 1075/1728 [4:34:33<31:06, 2.86s/it]
62%|██████▏ | 1076/1728 [4:34:36<31:10, 2.87s/it]
62%|██████▏ | 1077/1728 [4:34:39<31:04, 2.86s/it]
62%|██████▏ | 1078/1728 [4:34:42<31:00, 2.86s/it]
62%|██████▏ | 1079/1728 [4:34:45<30:55, 2.86s/it]
62%|██████▎ | 1080/1728 [4:34:48<31:03, 2.88s/it]
{'loss': '0.1539', 'grad_norm': '0.179', 'learning_rate': '3.164e-06', 'epoch': '3.708'}
+
62%|██████▎ | 1080/1728 [4:34:48<31:03, 2.88s/it]
63%|██████▎ | 1081/1728 [4:34:51<30:57, 2.87s/it]
63%|██████▎ | 1082/1728 [4:34:54<30:53, 2.87s/it]
63%|██████▎ | 1083/1728 [4:34:56<30:55, 2.88s/it]
63%|██████▎ | 1084/1728 [4:34:59<30:54, 2.88s/it]
63%|██████▎ | 1085/1728 [4:35:02<30:57, 2.89s/it]
63%|██████▎ | 1086/1728 [4:35:05<30:55, 2.89s/it]
63%|██████▎ | 1087/1728 [4:35:08<30:35, 2.86s/it]
63%|██████▎ | 1088/1728 [4:35:11<30:33, 2.86s/it]
63%|██████▎ | 1089/1728 [4:35:14<30:15, 2.84s/it]
63%|██████▎ | 1090/1728 [4:35:16<30:12, 2.84s/it]
63%|██████▎ | 1091/1728 [4:35:19<30:09, 2.84s/it]
63%|██████▎ | 1092/1728 [4:35:22<30:01, 2.83s/it]
63%|██████▎ | 1093/1728 [4:35:25<29:54, 2.83s/it]
63%|██████▎ | 1094/1728 [4:35:28<29:50, 2.82s/it]
63%|██████▎ | 1095/1728 [4:35:31<29:48, 2.83s/it]
63%|██████▎ | 1096/1728 [4:35:33<29:44, 2.82s/it]
63%|██████▎ | 1097/1728 [4:35:36<29:43, 2.83s/it]
64%|██████▎ | 1098/1728 [4:35:39<29:39, 2.82s/it]
64%|██████▎ | 1099/1728 [4:35:42<29:40, 2.83s/it]
64%|██████▎ | 1100/1728 [4:35:45<29:35, 2.83s/it]
{'loss': '0.1518', 'grad_norm': '0.02855', 'learning_rate': '2.999e-06', 'epoch': '3.788'}
+
64%|██████▎ | 1100/1728 [4:35:45<29:35, 2.83s/it]
64%|██████▎ | 1101/1728 [4:35:48<29:48, 2.85s/it]
64%|██████▍ | 1102/1728 [4:35:50<29:45, 2.85s/it]
64%|██████▍ | 1103/1728 [4:35:53<29:35, 2.84s/it]
64%|██████▍ | 1104/1728 [4:35:56<29:28, 2.83s/it]
64%|██████▍ | 1105/1728 [4:35:59<29:32, 2.84s/it]
64%|██████▍ | 1106/1728 [4:36:02<29:17, 2.83s/it]
64%|██████▍ | 1107/1728 [4:36:05<29:21, 2.84s/it]
64%|██████▍ | 1108/1728 [4:36:07<29:23, 2.84s/it]
64%|██████▍ | 1109/1728 [4:36:10<29:22, 2.85s/it]
64%|██████▍ | 1110/1728 [4:36:13<29:12, 2.84s/it]
64%|██████▍ | 1111/1728 [4:36:16<29:07, 2.83s/it]
64%|██████▍ | 1112/1728 [4:36:19<29:03, 2.83s/it]
64%|██████▍ | 1113/1728 [4:36:22<29:17, 2.86s/it]
64%|██████▍ | 1114/1728 [4:36:24<29:09, 2.85s/it]
65%|██████▍ | 1115/1728 [4:36:27<29:03, 2.84s/it]
65%|██████▍ | 1116/1728 [4:36:30<28:55, 2.84s/it]
65%|██████▍ | 1117/1728 [4:36:33<28:47, 2.83s/it]
65%|██████▍ | 1118/1728 [4:36:36<28:42, 2.82s/it]
65%|██████▍ | 1119/1728 [4:36:39<28:41, 2.83s/it]
65%|██████▍ | 1120/1728 [4:36:41<28:36, 2.82s/it]
{'loss': '0.1629', 'grad_norm': '0.0591', 'learning_rate': '2.836e-06', 'epoch': '3.868'}
+
65%|██████▍ | 1120/1728 [4:36:41<28:36, 2.82s/it]
65%|██████▍ | 1121/1728 [4:36:44<28:39, 2.83s/it]
65%|██████▍ | 1122/1728 [4:36:47<28:21, 2.81s/it]
65%|██████▍ | 1123/1728 [4:36:50<28:18, 2.81s/it]
65%|██████▌ | 1124/1728 [4:36:53<28:23, 2.82s/it]
65%|██████▌ | 1125/1728 [4:36:55<28:13, 2.81s/it]
65%|██████▌ | 1126/1728 [4:36:58<28:16, 2.82s/it]
65%|██████▌ | 1127/1728 [4:37:01<28:26, 2.84s/it]
65%|██████▌ | 1128/1728 [4:37:04<28:30, 2.85s/it]
65%|██████▌ | 1129/1728 [4:37:07<28:26, 2.85s/it]
65%|██████▌ | 1130/1728 [4:37:10<28:21, 2.85s/it]
65%|██████▌ | 1131/1728 [4:37:13<28:31, 2.87s/it]
66%|██████▌ | 1132/1728 [4:37:15<28:24, 2.86s/it]
66%|██████▌ | 1133/1728 [4:37:18<28:23, 2.86s/it]
66%|██████▌ | 1134/1728 [4:37:21<28:23, 2.87s/it]
66%|██████▌ | 1135/1728 [4:37:24<28:08, 2.85s/it]
66%|██████▌ | 1136/1728 [4:37:27<28:14, 2.86s/it]
66%|██████▌ | 1137/1728 [4:37:30<28:23, 2.88s/it]
66%|██████▌ | 1138/1728 [4:37:33<28:18, 2.88s/it]
66%|██████▌ | 1139/1728 [4:37:36<28:23, 2.89s/it]
66%|██████▌ | 1140/1728 [4:37:39<28:17, 2.89s/it]
{'loss': '0.146', 'grad_norm': '0.05181', 'learning_rate': '2.677e-06', 'epoch': '3.949'}
+
66%|██████▌ | 1140/1728 [4:37:39<28:17, 2.89s/it]
66%|██████▌ | 1141/1728 [4:37:41<28:10, 2.88s/it]
66%|██████▌ | 1142/1728 [4:37:44<28:00, 2.87s/it]
66%|██████▌ | 1143/1728 [4:37:47<27:53, 2.86s/it]
66%|██████▌ | 1144/1728 [4:37:50<27:52, 2.86s/it]
66%|██████▋ | 1145/1728 [4:37:53<27:51, 2.87s/it]
66%|██████▋ | 1146/1728 [4:37:56<27:36, 2.85s/it]
66%|██████▋ | 1147/1728 [4:37:58<27:27, 2.83s/it]
66%|██████▋ | 1148/1728 [4:38:01<27:26, 2.84s/it]
66%|██████▋ | 1149/1728 [4:38:04<27:22, 2.84s/it]
67%|██████▋ | 1150/1728 [4:38:07<27:25, 2.85s/it]
67%|██████▋ | 1151/1728 [4:38:10<27:26, 2.85s/it]
67%|██████▋ | 1152/1728 [4:38:13<27:33, 2.87s/it]
67%|██████▋ | 1153/1728 [4:38:15<25:38, 2.68s/it][INFO|trainer.py:2668] 2026-08-23 07:19:53,372 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 07:19:53,373 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 07:19:53,373 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/da_kd/checkpoint-1153
+[INFO|configuration_utils.py:780] 2026-08-23 08:07:37,246 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 08:07:37,248 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 08:07:37,282 >> chat template saved in results/qwen3/da_kd/checkpoint-1153/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 08:07:37,282 >> tokenizer config file saved in results/qwen3/da_kd/checkpoint-1153/tokenizer_config.json
+DA-KD epoch=5 ratio=0.6545 active=6564/10027 dds_mean=1042645.1163
+
67%|██████▋ | 1154/1728 [5:51:25<210:17:02, 1318.85s/it]
67%|██████▋ | 1155/1728 [5:51:28<147:04:44, 924.06s/it]
67%|██████▋ | 1156/1728 [5:51:31<102:54:48, 647.71s/it]
67%|██████▋ | 1157/1728 [5:51:33<72:02:50, 454.24s/it]
67%|██████▋ | 1158/1728 [5:51:36<50:28:51, 318.83s/it]
67%|██████▋ | 1159/1728 [5:51:39<35:24:27, 224.02s/it]
67%|██████▋ | 1160/1728 [5:51:42<24:52:39, 157.68s/it]
{'loss': '0.1772', 'grad_norm': '0.003416', 'learning_rate': '2.521e-06', 'epoch': '4.034'}
+
67%|██████▋ | 1160/1728 [5:51:42<24:52:39, 157.68s/it]
67%|██████▋ | 1161/1728 [5:51:45<17:31:09, 111.23s/it]
67%|██████▋ | 1162/1728 [5:51:48<12:22:45, 78.74s/it]
67%|██████▋ | 1163/1728 [5:51:51<8:47:05, 55.97s/it]
67%|██████▋ | 1164/1728 [5:51:54<6:16:27, 40.05s/it]
67%|██████▋ | 1165/1728 [5:51:56<4:30:59, 28.88s/it]
67%|██████▋ | 1166/1728 [5:51:59<3:17:28, 21.08s/it]
68%|██████▊ | 1167/1728 [5:52:02<2:25:52, 15.60s/it]
68%|██████▊ | 1168/1728 [5:52:05<1:49:54, 11.78s/it]
68%|██████▊ | 1169/1728 [5:52:08<1:24:54, 9.11s/it]
68%|██████▊ | 1170/1728 [5:52:11<1:07:21, 7.24s/it]
68%|██████▊ | 1171/1728 [5:52:14<54:58, 5.92s/it]
68%|██████▊ | 1172/1728 [5:52:16<46:21, 5.00s/it]
68%|██████▊ | 1173/1728 [5:52:19<40:17, 4.36s/it]
68%|██████▊ | 1174/1728 [5:52:22<36:14, 3.92s/it]
68%|██████▊ | 1175/1728 [5:52:25<33:06, 3.59s/it]
68%|██████▊ | 1176/1728 [5:52:28<31:04, 3.38s/it]
68%|██████▊ | 1177/1728 [5:52:31<29:29, 3.21s/it]
68%|██████▊ | 1178/1728 [5:52:34<28:28, 3.11s/it]
68%|██████▊ | 1179/1728 [5:52:36<27:50, 3.04s/it]
68%|██████▊ | 1180/1728 [5:52:39<27:20, 2.99s/it]
{'loss': '0.1715', 'grad_norm': '0.05329', 'learning_rate': '2.368e-06', 'epoch': '4.132'}
+
68%|██████▊ | 1180/1728 [5:52:39<27:20, 2.99s/it]
68%|██████▊ | 1181/1728 [5:52:42<26:45, 2.94s/it]
68%|██████▊ | 1182/1728 [5:52:45<26:30, 2.91s/it]
68%|██████▊ | 1183/1728 [5:52:48<26:11, 2.88s/it]
69%|██████▊ | 1184/1728 [5:52:51<26:11, 2.89s/it]
69%|██████▊ | 1185/1728 [5:52:54<26:07, 2.89s/it]
69%|██████▊ | 1186/1728 [5:52:56<26:01, 2.88s/it]
69%|██████▊ | 1187/1728 [5:52:59<25:55, 2.88s/it]
69%|██████▉ | 1188/1728 [5:53:02<25:54, 2.88s/it]
69%|██████▉ | 1189/1728 [5:53:05<25:51, 2.88s/it]
69%|██████▉ | 1190/1728 [5:53:08<25:51, 2.88s/it]
69%|██████▉ | 1191/1728 [5:53:11<25:45, 2.88s/it]
69%|██████▉ | 1192/1728 [5:53:14<25:44, 2.88s/it]
69%|██████▉ | 1193/1728 [5:53:17<25:39, 2.88s/it]
69%|██████▉ | 1194/1728 [5:53:19<25:37, 2.88s/it]
69%|██████▉ | 1195/1728 [5:53:22<25:24, 2.86s/it]
69%|██████▉ | 1196/1728 [5:53:25<25:23, 2.86s/it]
69%|██████▉ | 1197/1728 [5:53:29<26:44, 3.02s/it]
69%|██████▉ | 1198/1728 [5:53:32<27:54, 3.16s/it]
69%|██████▉ | 1199/1728 [5:53:36<28:48, 3.27s/it]
69%|██████▉ | 1200/1728 [5:53:39<29:25, 3.34s/it]
{'loss': '0.1666', 'grad_norm': '0.004535', 'learning_rate': '2.218e-06', 'epoch': '4.229'}
+
69%|██████▉ | 1200/1728 [5:53:39<29:25, 3.34s/it]
70%|██████▉ | 1201/1728 [5:53:43<29:57, 3.41s/it]
70%|██████▉ | 1202/1728 [5:53:46<30:15, 3.45s/it]
70%|██████▉ | 1203/1728 [5:53:49<29:40, 3.39s/it]
70%|██████▉ | 1204/1728 [5:53:53<29:51, 3.42s/it]
70%|██████▉ | 1205/1728 [5:53:57<30:12, 3.46s/it]
70%|██████▉ | 1206/1728 [5:54:00<30:29, 3.50s/it]
70%|██████▉ | 1207/1728 [5:54:04<30:33, 3.52s/it]
70%|██████▉ | 1208/1728 [5:54:07<30:40, 3.54s/it]
70%|██████▉ | 1209/1728 [5:54:11<30:35, 3.54s/it]
70%|███████ | 1210/1728 [5:54:14<30:33, 3.54s/it]
70%|███████ | 1211/1728 [5:54:18<30:08, 3.50s/it]
70%|███████ | 1212/1728 [5:54:21<29:55, 3.48s/it]
70%|███████ | 1213/1728 [5:54:24<29:14, 3.41s/it]
70%|███████ | 1214/1728 [5:54:28<28:55, 3.38s/it]
70%|███████ | 1215/1728 [5:54:31<29:17, 3.43s/it]
70%|███████ | 1216/1728 [5:54:35<28:53, 3.39s/it]
70%|███████ | 1217/1728 [5:54:38<29:09, 3.42s/it]
70%|███████ | 1218/1728 [5:54:41<28:40, 3.37s/it]
71%|███████ | 1219/1728 [5:54:45<28:35, 3.37s/it]
71%|███████ | 1220/1728 [5:54:48<28:54, 3.41s/it]
{'loss': '0.1989', 'grad_norm': '0.03272', 'learning_rate': '2.072e-06', 'epoch': '4.327'}
+
71%|███████ | 1220/1728 [5:54:48<28:54, 3.41s/it]
71%|███████ | 1221/1728 [5:54:52<28:41, 3.40s/it]
71%|███████ | 1222/1728 [5:54:55<28:38, 3.40s/it]
71%|███████ | 1223/1728 [5:54:58<28:59, 3.45s/it]
71%|███████ | 1224/1728 [5:55:02<29:11, 3.47s/it]
71%|███████ | 1225/1728 [5:55:06<29:28, 3.52s/it]
71%|███████ | 1226/1728 [5:55:09<29:04, 3.47s/it]
71%|███████ | 1227/1728 [5:55:12<28:43, 3.44s/it]
71%|███████ | 1228/1728 [5:55:16<28:32, 3.43s/it]
71%|███████ | 1229/1728 [5:55:19<28:30, 3.43s/it]
71%|███████ | 1230/1728 [5:55:23<28:23, 3.42s/it]
71%|███████ | 1231/1728 [5:55:26<28:39, 3.46s/it]
71%|███████▏ | 1232/1728 [5:55:30<28:41, 3.47s/it]
71%|███████▏ | 1233/1728 [5:55:33<28:33, 3.46s/it]
71%|███████▏ | 1234/1728 [5:55:37<28:54, 3.51s/it]
71%|███████▏ | 1235/1728 [5:55:40<28:43, 3.50s/it]
72%|███████▏ | 1236/1728 [5:55:44<28:46, 3.51s/it]
72%|███████▏ | 1237/1728 [5:55:47<28:14, 3.45s/it]
72%|███████▏ | 1238/1728 [5:55:50<27:47, 3.40s/it]
72%|███████▏ | 1239/1728 [5:55:54<27:50, 3.42s/it]
72%|███████▏ | 1240/1728 [5:55:57<27:33, 3.39s/it]
{'loss': '0.1835', 'grad_norm': '0.03355', 'learning_rate': '1.931e-06', 'epoch': '4.424'}
+
72%|███████▏ | 1240/1728 [5:55:57<27:33, 3.39s/it]
72%|███████▏ | 1241/1728 [5:56:01<27:48, 3.43s/it]
72%|███████▏ | 1242/1728 [5:56:04<28:01, 3.46s/it]
72%|███████▏ | 1243/1728 [5:56:08<28:08, 3.48s/it]
72%|███████▏ | 1244/1728 [5:56:11<28:16, 3.51s/it]
72%|███████▏ | 1245/1728 [5:56:15<27:55, 3.47s/it]
72%|███████▏ | 1246/1728 [5:56:18<28:08, 3.50s/it]
72%|███████▏ | 1247/1728 [5:56:22<28:06, 3.51s/it]
72%|███████▏ | 1248/1728 [5:56:25<28:08, 3.52s/it]
72%|███████▏ | 1249/1728 [5:56:29<27:57, 3.50s/it]
72%|███████▏ | 1250/1728 [5:56:32<28:09, 3.53s/it]
72%|███████▏ | 1251/1728 [5:56:36<27:43, 3.49s/it]
72%|███████▏ | 1252/1728 [5:56:39<27:53, 3.52s/it]
73%|███████▎ | 1253/1728 [5:56:43<27:28, 3.47s/it]
73%|███████▎ | 1254/1728 [5:56:46<26:18, 3.33s/it]
73%|███████▎ | 1255/1728 [5:56:49<26:31, 3.36s/it]
73%|███████▎ | 1256/1728 [5:56:52<26:24, 3.36s/it]
73%|███████▎ | 1257/1728 [5:56:56<26:35, 3.39s/it]
73%|███████▎ | 1258/1728 [5:57:00<27:05, 3.46s/it]
73%|███████▎ | 1259/1728 [5:57:03<27:04, 3.46s/it]
73%|███████▎ | 1260/1728 [5:57:07<27:06, 3.47s/it]
{'loss': '0.1809', 'grad_norm': '0.04594', 'learning_rate': '1.793e-06', 'epoch': '4.522'}
+
73%|███████▎ | 1260/1728 [5:57:07<27:06, 3.47s/it]
73%|███████▎ | 1261/1728 [5:57:10<27:23, 3.52s/it]
73%|███████▎ | 1262/1728 [5:57:14<27:17, 3.51s/it]
73%|███████▎ | 1263/1728 [5:57:17<27:20, 3.53s/it]
73%|███████▎ | 1264/1728 [5:57:21<26:59, 3.49s/it]
73%|███████▎ | 1265/1728 [5:57:24<27:01, 3.50s/it]
73%|███████▎ | 1266/1728 [5:57:28<26:42, 3.47s/it]
73%|███████▎ | 1267/1728 [5:57:31<26:50, 3.49s/it]
73%|███████▎ | 1268/1728 [5:57:35<26:39, 3.48s/it]
73%|███████▎ | 1269/1728 [5:57:38<26:25, 3.45s/it]
73%|███████▎ | 1270/1728 [5:57:41<26:04, 3.42s/it]
74%|███████▎ | 1271/1728 [5:57:45<26:19, 3.46s/it]
74%|███████▎ | 1272/1728 [5:57:48<26:02, 3.43s/it]
74%|███████▎ | 1273/1728 [5:57:52<26:02, 3.43s/it]
74%|███████▎ | 1274/1728 [5:57:55<25:54, 3.42s/it]
74%|███████▍ | 1275/1728 [5:57:58<25:47, 3.42s/it]
74%|███████▍ | 1276/1728 [5:58:02<25:59, 3.45s/it]
74%|███████▍ | 1277/1728 [5:58:05<25:52, 3.44s/it]
74%|███████▍ | 1278/1728 [5:58:09<25:54, 3.45s/it]
74%|███████▍ | 1279/1728 [5:58:12<25:31, 3.41s/it]
74%|███████▍ | 1280/1728 [5:58:16<25:44, 3.45s/it]
{'loss': '0.1857', 'grad_norm': '0.1048', 'learning_rate': '1.66e-06', 'epoch': '4.619'}
+
74%|███████▍ | 1280/1728 [5:58:16<25:44, 3.45s/it]
74%|███████▍ | 1281/1728 [5:58:19<25:38, 3.44s/it]
74%|███████▍ | 1282/1728 [5:58:22<24:15, 3.26s/it]
74%|███████▍ | 1283/1728 [5:58:25<24:31, 3.31s/it]
74%|███████▍ | 1284/1728 [5:58:29<24:54, 3.37s/it]
74%|███████▍ | 1285/1728 [5:58:32<25:21, 3.44s/it]
74%|███████▍ | 1286/1728 [5:58:36<25:14, 3.43s/it]
74%|███████▍ | 1287/1728 [5:58:39<25:27, 3.46s/it]
75%|███████▍ | 1288/1728 [5:58:43<25:09, 3.43s/it]
75%|███████▍ | 1289/1728 [5:58:46<24:25, 3.34s/it]
75%|███████▍ | 1290/1728 [5:58:49<24:46, 3.39s/it]
75%|███████▍ | 1291/1728 [5:58:53<25:06, 3.45s/it]
75%|███████▍ | 1292/1728 [5:58:56<24:54, 3.43s/it]
75%|███████▍ | 1293/1728 [5:59:00<25:03, 3.46s/it]
75%|███████▍ | 1294/1728 [5:59:03<25:02, 3.46s/it]
75%|███████▍ | 1295/1728 [5:59:07<25:00, 3.46s/it]
75%|███████▌ | 1296/1728 [5:59:10<25:05, 3.48s/it]
75%|███████▌ | 1297/1728 [5:59:14<24:57, 3.47s/it]
75%|███████▌ | 1298/1728 [5:59:17<24:37, 3.44s/it]
75%|███████▌ | 1299/1728 [5:59:21<24:25, 3.42s/it]
75%|███████▌ | 1300/1728 [5:59:24<24:09, 3.39s/it]
{'loss': '0.178', 'grad_norm': '0.05551', 'learning_rate': '1.531e-06', 'epoch': '4.717'}
+
75%|███████▌ | 1300/1728 [5:59:24<24:09, 3.39s/it]
75%|███████▌ | 1301/1728 [5:59:27<24:33, 3.45s/it]
75%|███████▌ | 1302/1728 [5:59:31<24:44, 3.49s/it]
75%|███████▌ | 1303/1728 [5:59:34<24:32, 3.46s/it]
75%|███████▌ | 1304/1728 [5:59:38<24:16, 3.44s/it]
76%|███████▌ | 1305/1728 [5:59:41<24:35, 3.49s/it]
76%|███████▌ | 1306/1728 [5:59:45<24:20, 3.46s/it]
76%|███████▌ | 1307/1728 [5:59:48<24:06, 3.44s/it]
76%|███████▌ | 1308/1728 [5:59:52<24:20, 3.48s/it]
76%|███████▌ | 1309/1728 [5:59:55<24:31, 3.51s/it]
76%|███████▌ | 1310/1728 [5:59:59<24:25, 3.51s/it]
76%|███████▌ | 1311/1728 [6:00:02<24:15, 3.49s/it]
76%|███████▌ | 1312/1728 [6:00:06<24:19, 3.51s/it]
76%|███████▌ | 1313/1728 [6:00:09<24:23, 3.53s/it]
76%|███████▌ | 1314/1728 [6:00:13<24:17, 3.52s/it]
76%|███████▌ | 1315/1728 [6:00:16<24:08, 3.51s/it]
76%|███████▌ | 1316/1728 [6:00:20<24:20, 3.55s/it]
76%|███████▌ | 1317/1728 [6:00:24<24:25, 3.57s/it]
76%|███████▋ | 1318/1728 [6:00:27<24:23, 3.57s/it]
76%|███████▋ | 1319/1728 [6:00:31<23:50, 3.50s/it]
76%|███████▋ | 1320/1728 [6:00:34<23:33, 3.46s/it]
{'loss': '0.2', 'grad_norm': '0.007899', 'learning_rate': '1.407e-06', 'epoch': '4.814'}
+
76%|███████▋ | 1320/1728 [6:00:34<23:33, 3.46s/it]
76%|███████▋ | 1321/1728 [6:00:38<23:52, 3.52s/it]
77%|███████▋ | 1322/1728 [6:00:41<23:28, 3.47s/it]
77%|███████▋ | 1323/1728 [6:00:44<23:22, 3.46s/it]
77%|███████▋ | 1324/1728 [6:00:48<23:26, 3.48s/it]
77%|███████▋ | 1325/1728 [6:00:52<23:36, 3.52s/it]
77%|███████▋ | 1326/1728 [6:00:55<23:45, 3.55s/it]
77%|███████▋ | 1327/1728 [6:00:58<23:19, 3.49s/it]
77%|███████▋ | 1328/1728 [6:01:02<23:22, 3.51s/it]
77%|███████▋ | 1329/1728 [6:01:05<23:11, 3.49s/it]
77%|███████▋ | 1330/1728 [6:01:09<23:23, 3.53s/it]
77%|███████▋ | 1331/1728 [6:01:13<23:11, 3.51s/it]
77%|███████▋ | 1332/1728 [6:01:16<23:15, 3.52s/it]
77%|███████▋ | 1333/1728 [6:01:19<22:37, 3.44s/it]
77%|███████▋ | 1334/1728 [6:01:23<22:33, 3.44s/it]
77%|███████▋ | 1335/1728 [6:01:26<22:33, 3.45s/it]
77%|███████▋ | 1336/1728 [6:01:30<22:43, 3.48s/it]
77%|███████▋ | 1337/1728 [6:01:33<22:41, 3.48s/it]
77%|███████▋ | 1338/1728 [6:01:37<22:39, 3.48s/it]
77%|███████▋ | 1339/1728 [6:01:40<22:48, 3.52s/it]
78%|███████▊ | 1340/1728 [6:01:44<22:34, 3.49s/it]
{'loss': '0.1797', 'grad_norm': '0.09339', 'learning_rate': '1.287e-06', 'epoch': '4.912'}
+
78%|███████▊ | 1340/1728 [6:01:44<22:34, 3.49s/it]
78%|███████▊ | 1341/1728 [6:01:47<22:28, 3.48s/it]
78%|███████▊ | 1342/1728 [6:01:51<22:20, 3.47s/it]
78%|███████▊ | 1343/1728 [6:01:54<22:17, 3.47s/it]
78%|███████▊ | 1344/1728 [6:01:58<22:03, 3.45s/it]
78%|███████▊ | 1345/1728 [6:02:01<22:13, 3.48s/it]
78%|███████▊ | 1346/1728 [6:02:05<22:19, 3.51s/it]
78%|███████▊ | 1347/1728 [6:02:08<21:55, 3.45s/it]
78%|███████▊ | 1348/1728 [6:02:11<21:41, 3.42s/it]
78%|███████▊ | 1349/1728 [6:02:15<21:46, 3.45s/it]
78%|███████▊ | 1350/1728 [6:02:18<21:55, 3.48s/it]
78%|███████▊ | 1351/1728 [6:02:22<21:38, 3.45s/it]
78%|███████▊ | 1352/1728 [6:02:25<21:38, 3.45s/it]
78%|███████▊ | 1353/1728 [6:02:29<21:15, 3.40s/it]
78%|███████▊ | 1354/1728 [6:02:32<21:36, 3.47s/it]
78%|███████▊ | 1355/1728 [6:02:36<21:42, 3.49s/it]
78%|███████▊ | 1356/1728 [6:02:39<21:35, 3.48s/it]
79%|███████▊ | 1357/1728 [6:02:43<21:19, 3.45s/it]
79%|███████▊ | 1358/1728 [6:02:46<21:15, 3.45s/it]
79%|███████▊ | 1359/1728 [6:02:47<15:55, 2.59s/it][INFO|trainer.py:2668] 2026-08-23 08:44:25,006 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 08:44:25,006 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 08:44:25,006 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/da_kd/checkpoint-1359
+[INFO|configuration_utils.py:780] 2026-08-23 09:33:31,181 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 09:33:31,182 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 09:33:31,218 >> chat template saved in results/qwen3/da_kd/checkpoint-1359/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 09:33:31,218 >> tokenizer config file saved in results/qwen3/da_kd/checkpoint-1359/tokenizer_config.json
+DA-KD epoch=6 ratio=0.5000 active=5014/10027 dds_mean=1042010.7783
+
79%|███████▊ | 1360/1728 [7:17:19<137:19:37, 1343.42s/it]
{'loss': '0.1971', 'grad_norm': '0.006825', 'learning_rate': '1.173e-06', 'epoch': '5.006'}
+
79%|███████▊ | 1360/1728 [7:17:19<137:19:37, 1343.42s/it]
79%|███████▉ | 1361/1728 [7:17:21<95:57:17, 941.25s/it]
79%|███████▉ | 1362/1728 [7:17:24<67:04:11, 659.70s/it]
79%|███████▉ | 1363/1728 [7:17:27<46:54:24, 462.64s/it]
79%|███████▉ | 1364/1728 [7:17:30<32:49:44, 324.68s/it]
79%|███████▉ | 1365/1728 [7:17:33<23:00:02, 228.11s/it]
79%|███████▉ | 1366/1728 [7:17:35<16:08:25, 160.51s/it]
79%|███████▉ | 1367/1728 [7:17:38<11:21:12, 113.22s/it]
79%|███████▉ | 1368/1728 [7:17:41<8:00:36, 80.10s/it]
79%|███████▉ | 1369/1728 [7:17:44<5:40:32, 56.92s/it]
79%|███████▉ | 1370/1728 [7:17:47<4:02:43, 40.68s/it]
79%|███████▉ | 1371/1728 [7:17:50<2:54:27, 29.32s/it]
79%|███████▉ | 1372/1728 [7:17:52<2:06:46, 21.37s/it]
79%|███████▉ | 1373/1728 [7:17:55<1:33:23, 15.78s/it]
80%|███████▉ | 1374/1728 [7:17:58<1:10:07, 11.89s/it]
80%|███████▉ | 1375/1728 [7:18:01<53:54, 9.16s/it]
80%|███████▉ | 1376/1728 [7:18:03<42:32, 7.25s/it]
80%|███████▉ | 1377/1728 [7:18:06<34:31, 5.90s/it]
80%|███████▉ | 1378/1728 [7:18:09<29:02, 4.98s/it]
80%|███████▉ | 1379/1728 [7:18:12<25:13, 4.34s/it]
80%|███████▉ | 1380/1728 [7:18:15<22:30, 3.88s/it]
{'loss': '0.2513', 'grad_norm': '0.0514', 'learning_rate': '1.063e-06', 'epoch': '5.134'}
+
80%|███████▉ | 1380/1728 [7:18:15<22:30, 3.88s/it]
80%|███████▉ | 1381/1728 [7:18:18<20:39, 3.57s/it]
80%|███████▉ | 1382/1728 [7:18:20<19:20, 3.35s/it]
80%|████████ | 1383/1728 [7:18:23<18:27, 3.21s/it]
80%|████████ | 1384/1728 [7:18:26<17:42, 3.09s/it]
80%|████████ | 1385/1728 [7:18:29<17:10, 3.00s/it]
80%|████████ | 1386/1728 [7:18:32<16:40, 2.93s/it]
80%|████████ | 1387/1728 [7:18:34<16:24, 2.89s/it]
80%|████████ | 1388/1728 [7:18:37<16:19, 2.88s/it]
80%|████████ | 1389/1728 [7:18:40<16:09, 2.86s/it]
80%|████████ | 1390/1728 [7:18:43<16:42, 2.97s/it]
80%|████████ | 1391/1728 [7:18:46<16:23, 2.92s/it]
81%|████████ | 1392/1728 [7:18:49<16:04, 2.87s/it]
81%|████████ | 1393/1728 [7:18:52<15:54, 2.85s/it]
81%|████████ | 1394/1728 [7:18:54<15:42, 2.82s/it]
81%|████████ | 1395/1728 [7:18:57<15:41, 2.83s/it]
81%|████████ | 1396/1728 [7:19:00<15:39, 2.83s/it]
81%|████████ | 1397/1728 [7:19:03<15:34, 2.82s/it]
81%|████████ | 1398/1728 [7:19:06<15:34, 2.83s/it]
81%|████████ | 1399/1728 [7:19:09<15:38, 2.85s/it]
81%|████████ | 1400/1728 [7:19:12<15:34, 2.85s/it]
{'loss': '0.2237', 'grad_norm': '0.0127', 'learning_rate': '9.594e-07', 'epoch': '5.262'}
+
81%|████████ | 1400/1728 [7:19:12<15:34, 2.85s/it]
81%|████████ | 1401/1728 [7:19:14<15:35, 2.86s/it]
81%|████████ | 1402/1728 [7:19:17<15:45, 2.90s/it]
81%|████████ | 1403/1728 [7:19:20<15:38, 2.89s/it]
81%|████████▏ | 1404/1728 [7:19:23<15:39, 2.90s/it]
81%|████████▏ | 1405/1728 [7:19:26<15:37, 2.90s/it]
81%|████████▏ | 1406/1728 [7:19:29<15:27, 2.88s/it]
81%|████████▏ | 1407/1728 [7:19:32<15:23, 2.88s/it]
81%|████████▏ | 1408/1728 [7:19:35<15:16, 2.87s/it]
82%|████████▏ | 1409/1728 [7:19:37<15:08, 2.85s/it]
82%|████████▏ | 1410/1728 [7:19:40<15:01, 2.83s/it]
82%|████████▏ | 1411/1728 [7:19:43<14:58, 2.83s/it]
82%|████████▏ | 1412/1728 [7:19:46<14:59, 2.85s/it]
82%|████████▏ | 1413/1728 [7:19:49<14:55, 2.84s/it]
82%|████████▏ | 1414/1728 [7:19:52<15:02, 2.87s/it]
82%|████████▏ | 1415/1728 [7:19:55<15:03, 2.89s/it]
82%|████████▏ | 1416/1728 [7:19:58<14:59, 2.88s/it]
82%|████████▏ | 1417/1728 [7:20:00<14:55, 2.88s/it]
82%|████████▏ | 1418/1728 [7:20:03<14:48, 2.87s/it]
82%|████████▏ | 1419/1728 [7:20:06<14:53, 2.89s/it]
82%|████████▏ | 1420/1728 [7:20:09<14:50, 2.89s/it]
{'loss': '0.2287', 'grad_norm': '0.02553', 'learning_rate': '8.608e-07', 'epoch': '5.389'}
+
82%|████████▏ | 1420/1728 [7:20:09<14:50, 2.89s/it]
82%|████████▏ | 1421/1728 [7:20:12<14:42, 2.87s/it]
82%|████████▏ | 1422/1728 [7:20:15<14:36, 2.86s/it]
82%|████████▏ | 1423/1728 [7:20:18<14:31, 2.86s/it]
82%|████████▏ | 1424/1728 [7:20:20<14:28, 2.86s/it]
82%|████████▏ | 1425/1728 [7:20:23<14:32, 2.88s/it]
83%|████████▎ | 1426/1728 [7:20:26<14:31, 2.88s/it]
83%|████████▎ | 1427/1728 [7:20:29<14:24, 2.87s/it]
83%|████████▎ | 1428/1728 [7:20:32<14:53, 2.98s/it]
83%|████████▎ | 1429/1728 [7:20:35<14:38, 2.94s/it]
83%|████████▎ | 1430/1728 [7:20:38<14:46, 2.98s/it]
83%|████████▎ | 1431/1728 [7:20:41<14:34, 2.94s/it]
83%|████████▎ | 1432/1728 [7:20:44<14:26, 2.93s/it]
83%|████████▎ | 1433/1728 [7:20:47<14:10, 2.88s/it]
83%|████████▎ | 1434/1728 [7:20:50<14:09, 2.89s/it]
83%|████████▎ | 1435/1728 [7:20:53<14:00, 2.87s/it]
83%|████████▎ | 1436/1728 [7:20:56<14:08, 2.91s/it]
83%|████████▎ | 1437/1728 [7:20:58<13:59, 2.88s/it]
83%|████████▎ | 1438/1728 [7:21:01<14:04, 2.91s/it]
83%|████████▎ | 1439/1728 [7:21:04<13:54, 2.89s/it]
83%|████████▎ | 1440/1728 [7:21:07<14:06, 2.94s/it]
{'loss': '0.2352', 'grad_norm': '0.001655', 'learning_rate': '7.677e-07', 'epoch': '5.517'}
+
83%|████████▎ | 1440/1728 [7:21:07<14:06, 2.94s/it]
83%|████████▎ | 1441/1728 [7:21:10<14:01, 2.93s/it]
83%|████████▎ | 1442/1728 [7:21:13<13:46, 2.89s/it]
84%|████████▎ | 1443/1728 [7:21:16<13:37, 2.87s/it]
84%|████████▎ | 1444/1728 [7:21:19<13:34, 2.87s/it]
84%|████████▎ | 1445/1728 [7:21:22<13:45, 2.92s/it]
84%|████████▎ | 1446/1728 [7:21:25<13:51, 2.95s/it]
84%|████████▎ | 1447/1728 [7:21:28<14:04, 3.01s/it]
84%|████████▍ | 1448/1728 [7:21:31<13:55, 2.99s/it]
84%|████████▍ | 1449/1728 [7:21:34<13:44, 2.96s/it]
84%|████████▍ | 1450/1728 [7:21:37<13:37, 2.94s/it]
84%|████████▍ | 1451/1728 [7:21:40<13:40, 2.96s/it]
84%|████████▍ | 1452/1728 [7:21:43<13:47, 3.00s/it]
84%|████████▍ | 1453/1728 [7:21:45<13:32, 2.95s/it]
84%|████████▍ | 1454/1728 [7:21:48<13:20, 2.92s/it]
84%|████████▍ | 1455/1728 [7:21:51<13:12, 2.90s/it]
84%|████████▍ | 1456/1728 [7:21:54<13:13, 2.92s/it]
84%|████████▍ | 1457/1728 [7:21:57<13:11, 2.92s/it]
84%|████████▍ | 1458/1728 [7:22:00<13:00, 2.89s/it]
84%|████████▍ | 1459/1728 [7:22:03<12:56, 2.89s/it]
84%|████████▍ | 1460/1728 [7:22:06<12:51, 2.88s/it]
{'loss': '0.2269', 'grad_norm': '0.004166', 'learning_rate': '6.803e-07', 'epoch': '5.644'}
+
84%|████████▍ | 1460/1728 [7:22:06<12:51, 2.88s/it]
85%|████████▍ | 1461/1728 [7:22:08<12:44, 2.86s/it]
85%|████████▍ | 1462/1728 [7:22:11<12:37, 2.85s/it]
85%|████████▍ | 1463/1728 [7:22:14<12:39, 2.87s/it]
85%|████████▍ | 1464/1728 [7:22:17<12:40, 2.88s/it]
85%|████████▍ | 1465/1728 [7:22:20<12:59, 2.96s/it]
85%|████████▍ | 1466/1728 [7:22:24<13:24, 3.07s/it]
85%|████████▍ | 1467/1728 [7:22:27<13:12, 3.04s/it]
85%|████████▍ | 1468/1728 [7:22:30<13:08, 3.03s/it]
85%|████████▌ | 1469/1728 [7:22:32<12:53, 2.99s/it]
85%|████████▌ | 1470/1728 [7:22:35<12:42, 2.95s/it]
85%|████████▌ | 1471/1728 [7:22:38<12:43, 2.97s/it]
85%|████████▌ | 1472/1728 [7:22:41<12:29, 2.93s/it]
85%|████████▌ | 1473/1728 [7:22:44<12:21, 2.91s/it]
85%|████████▌ | 1474/1728 [7:22:47<12:13, 2.89s/it]
85%|████████▌ | 1475/1728 [7:22:50<12:11, 2.89s/it]
85%|████████▌ | 1476/1728 [7:22:53<12:15, 2.92s/it]
85%|████████▌ | 1477/1728 [7:22:56<12:37, 3.02s/it]
86%|████████▌ | 1478/1728 [7:22:59<12:41, 3.05s/it]
86%|████████▌ | 1479/1728 [7:23:02<12:38, 3.04s/it]
86%|████████▌ | 1480/1728 [7:23:05<12:24, 3.00s/it]
{'loss': '0.2226', 'grad_norm': '0.004447', 'learning_rate': '5.986e-07', 'epoch': '5.772'}
+
86%|████████▌ | 1480/1728 [7:23:05<12:24, 3.00s/it]
86%|████████▌ | 1481/1728 [7:23:08<12:09, 2.95s/it]
86%|████████▌ | 1482/1728 [7:23:11<11:56, 2.91s/it]
86%|████████▌ | 1483/1728 [7:23:14<12:07, 2.97s/it]
86%|████████▌ | 1484/1728 [7:23:17<11:56, 2.93s/it]
86%|████████▌ | 1485/1728 [7:23:19<11:46, 2.91s/it]
86%|████████▌ | 1486/1728 [7:23:22<11:45, 2.91s/it]
86%|████████▌ | 1487/1728 [7:23:25<11:38, 2.90s/it]
86%|████████▌ | 1488/1728 [7:23:28<11:34, 2.89s/it]
86%|████████▌ | 1489/1728 [7:23:31<11:29, 2.88s/it]
86%|████████▌ | 1490/1728 [7:23:34<11:19, 2.85s/it]
86%|████████▋ | 1491/1728 [7:23:37<11:12, 2.84s/it]
86%|████████▋ | 1492/1728 [7:23:39<11:09, 2.84s/it]
86%|████████▋ | 1493/1728 [7:23:42<11:07, 2.84s/it]
86%|████████▋ | 1494/1728 [7:23:45<11:00, 2.82s/it]
87%|████████▋ | 1495/1728 [7:23:48<11:05, 2.86s/it]
87%|████████▋ | 1496/1728 [7:23:51<11:04, 2.87s/it]
87%|████████▋ | 1497/1728 [7:23:54<11:01, 2.86s/it]
87%|████████▋ | 1498/1728 [7:23:57<11:05, 2.89s/it]
87%|████████▋ | 1499/1728 [7:24:00<10:58, 2.87s/it]
87%|████████▋ | 1500/1728 [7:24:02<10:55, 2.87s/it]
{'loss': '0.2292', 'grad_norm': '0.008641', 'learning_rate': '5.228e-07', 'epoch': '5.9'}
+
87%|████████▋ | 1500/1728 [7:24:02<10:55, 2.87s/it]
87%|████████▋ | 1501/1728 [7:24:05<10:47, 2.85s/it]
87%|████████▋ | 1502/1728 [7:24:08<10:43, 2.85s/it]
87%|████████▋ | 1503/1728 [7:24:11<10:40, 2.85s/it]
87%|████████▋ | 1504/1728 [7:24:14<10:46, 2.89s/it]
87%|████████▋ | 1505/1728 [7:24:17<10:40, 2.87s/it]
87%|████████▋ | 1506/1728 [7:24:20<10:38, 2.88s/it]
87%|████████▋ | 1507/1728 [7:24:22<10:32, 2.86s/it]
87%|████████▋ | 1508/1728 [7:24:25<10:24, 2.84s/it]
87%|████████▋ | 1509/1728 [7:24:28<10:19, 2.83s/it]
87%|████████▋ | 1510/1728 [7:24:31<10:28, 2.88s/it]
87%|████████▋ | 1511/1728 [7:24:34<10:22, 2.87s/it]
88%|████████▊ | 1512/1728 [7:24:37<10:16, 2.86s/it]
88%|████████▊ | 1513/1728 [7:24:40<10:17, 2.87s/it]
88%|████████▊ | 1514/1728 [7:24:43<10:45, 3.02s/it]
88%|████████▊ | 1515/1728 [7:24:46<10:36, 2.99s/it]
88%|████████▊ | 1516/1728 [7:24:48<09:46, 2.76s/it][INFO|trainer.py:2668] 2026-08-23 10:06:26,504 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 10:06:26,504 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 10:06:26,504 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/da_kd/checkpoint-1516
+[INFO|configuration_utils.py:780] 2026-08-23 10:54:09,803 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 10:54:09,804 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 10:54:09,840 >> chat template saved in results/qwen3/da_kd/checkpoint-1516/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 10:54:09,840 >> tokenizer config file saved in results/qwen3/da_kd/checkpoint-1516/tokenizer_config.json
+DA-KD epoch=7 ratio=0.3455 active=3464/10027 dds_mean=1041704.3107
+
88%|████████▊ | 1517/1728 [8:37:59<77:19:11, 1319.20s/it]
88%|████████▊ | 1518/1728 [8:38:02<53:54:55, 924.26s/it]
88%|████████▊ | 1519/1728 [8:38:05<37:36:34, 647.82s/it]
88%|████████▊ | 1520/1728 [8:38:07<26:14:52, 454.29s/it]
{'loss': '0.248', 'grad_norm': '0.01355', 'learning_rate': '4.531e-07', 'epoch': '6.037'}
+
88%|████████▊ | 1520/1728 [8:38:07<26:14:52, 454.29s/it]
88%|████████▊ | 1521/1728 [8:38:10<18:20:00, 318.85s/it]
88%|████████▊ | 1522/1728 [8:38:13<12:49:07, 224.02s/it]
88%|████████▊ | 1523/1728 [8:38:16<8:58:41, 157.66s/it]
88%|████████▊ | 1524/1728 [8:38:19<6:18:10, 111.23s/it]
88%|████████▊ | 1525/1728 [8:38:21<4:26:18, 78.71s/it]
88%|████████▊ | 1526/1728 [8:38:24<3:08:20, 55.94s/it]
88%|████████▊ | 1527/1728 [8:38:27<2:13:55, 39.98s/it]
88%|████████▊ | 1528/1728 [8:38:30<1:36:01, 28.81s/it]
88%|████████▊ | 1529/1728 [8:38:32<1:09:36, 20.99s/it]
89%|████████▊ | 1530/1728 [8:38:35<51:25, 15.58s/it]
89%|████████▊ | 1531/1728 [8:38:38<38:48, 11.82s/it]
89%|████████▊ | 1532/1728 [8:38:41<29:44, 9.11s/it]
89%|████████▊ | 1533/1728 [8:38:44<23:44, 7.30s/it]
89%|████████▉ | 1534/1728 [8:38:47<19:12, 5.94s/it]
89%|████████▉ | 1535/1728 [8:38:50<16:04, 5.00s/it]
89%|████████▉ | 1536/1728 [8:38:53<13:53, 4.34s/it]
89%|████████▉ | 1537/1728 [8:38:55<12:17, 3.86s/it]
89%|████████▉ | 1538/1728 [8:38:59<11:39, 3.68s/it]
89%|████████▉ | 1539/1728 [8:39:02<11:00, 3.50s/it]
89%|████████▉ | 1540/1728 [8:39:05<10:18, 3.29s/it]
{'loss': '0.3149', 'grad_norm': '0.004221', 'learning_rate': '3.894e-07', 'epoch': '6.222'}
+
89%|████████▉ | 1540/1728 [8:39:05<10:18, 3.29s/it]
89%|████████▉ | 1541/1728 [8:39:07<09:40, 3.10s/it]
89%|████████▉ | 1542/1728 [8:39:10<09:25, 3.04s/it]
89%|████████▉ | 1543/1728 [8:39:13<09:05, 2.95s/it]
89%|████████▉ | 1544/1728 [8:39:16<08:51, 2.89s/it]
89%|████████▉ | 1545/1728 [8:39:18<08:42, 2.86s/it]
89%|████████▉ | 1546/1728 [8:39:21<08:34, 2.83s/it]
90%|████████▉ | 1547/1728 [8:39:24<08:27, 2.80s/it]
90%|████████▉ | 1548/1728 [8:39:27<08:24, 2.80s/it]
90%|████████▉ | 1549/1728 [8:39:29<08:16, 2.78s/it]
90%|████████▉ | 1550/1728 [8:39:33<08:39, 2.92s/it]
90%|████████▉ | 1551/1728 [8:39:35<08:33, 2.90s/it]
90%|████████▉ | 1552/1728 [8:39:38<08:25, 2.87s/it]
90%|████████▉ | 1553/1728 [8:39:41<08:19, 2.85s/it]
90%|████████▉ | 1554/1728 [8:39:44<08:10, 2.82s/it]
90%|████████▉ | 1555/1728 [8:39:47<08:03, 2.80s/it]
90%|█████████ | 1556/1728 [8:39:49<08:04, 2.82s/it]
90%|█████████ | 1557/1728 [8:39:52<07:56, 2.79s/it]
90%|█████████ | 1558/1728 [8:39:55<07:53, 2.79s/it]
90%|█████████ | 1559/1728 [8:39:58<07:52, 2.79s/it]
90%|█████████ | 1560/1728 [8:40:01<07:50, 2.80s/it]
{'loss': '0.358', 'grad_norm': '0.02224', 'learning_rate': '3.318e-07', 'epoch': '6.406'}
+
90%|█████████ | 1560/1728 [8:40:01<07:50, 2.80s/it]
90%|█████████ | 1561/1728 [8:40:03<07:44, 2.78s/it]
90%|█████████ | 1562/1728 [8:40:06<07:39, 2.77s/it]
90%|█████████ | 1563/1728 [8:40:09<07:35, 2.76s/it]
91%|█████████ | 1564/1728 [8:40:12<07:31, 2.75s/it]
91%|█████████ | 1565/1728 [8:40:14<07:31, 2.77s/it]
91%|█████████ | 1566/1728 [8:40:17<07:27, 2.76s/it]
91%|█████████ | 1567/1728 [8:40:20<07:27, 2.78s/it]
91%|█████████ | 1568/1728 [8:40:23<07:35, 2.85s/it]
91%|█████████ | 1569/1728 [8:40:26<07:28, 2.82s/it]
91%|█████████ | 1570/1728 [8:40:28<07:20, 2.79s/it]
91%|█████████ | 1571/1728 [8:40:31<07:15, 2.77s/it]
91%|█████████ | 1572/1728 [8:40:34<07:10, 2.76s/it]
91%|█████████ | 1573/1728 [8:40:37<07:24, 2.87s/it]
91%|█████████ | 1574/1728 [8:40:40<07:16, 2.83s/it]
91%|█████████ | 1575/1728 [8:40:42<07:07, 2.79s/it]
91%|█████████ | 1576/1728 [8:40:45<07:01, 2.77s/it]
91%|█████████▏| 1577/1728 [8:40:48<07:20, 2.92s/it]
91%|█████████▏| 1578/1728 [8:40:51<07:09, 2.86s/it]
91%|█████████▏| 1579/1728 [8:40:54<07:05, 2.86s/it]
91%|█████████▏| 1580/1728 [8:40:57<07:03, 2.86s/it]
{'loss': '0.3606', 'grad_norm': '0.006539', 'learning_rate': '2.805e-07', 'epoch': '6.591'}
+
91%|█████████▏| 1580/1728 [8:40:57<07:03, 2.86s/it]
91%|█████████▏| 1581/1728 [8:41:00<06:56, 2.83s/it]
92%|█████████▏| 1582/1728 [8:41:02<06:51, 2.82s/it]
92%|█████████▏| 1583/1728 [8:41:05<06:51, 2.84s/it]
92%|█████████▏| 1584/1728 [8:41:08<06:45, 2.82s/it]
92%|█████████▏| 1585/1728 [8:41:11<06:40, 2.80s/it]
92%|█████████▏| 1586/1728 [8:41:14<06:37, 2.80s/it]
92%|█████████▏| 1587/1728 [8:41:17<06:44, 2.87s/it]
92%|█████████▏| 1588/1728 [8:41:19<06:35, 2.82s/it]
92%|█████████▏| 1589/1728 [8:41:22<06:29, 2.80s/it]
92%|█████████▏| 1590/1728 [8:41:25<06:25, 2.79s/it]
92%|█████████▏| 1591/1728 [8:41:28<06:24, 2.80s/it]
92%|█████████▏| 1592/1728 [8:41:31<06:21, 2.81s/it]
92%|█████████▏| 1593/1728 [8:41:33<06:21, 2.82s/it]
92%|█████████▏| 1594/1728 [8:41:36<06:22, 2.85s/it]
92%|█████████▏| 1595/1728 [8:41:39<06:24, 2.89s/it]
92%|█████████▏| 1596/1728 [8:41:42<06:15, 2.85s/it]
92%|█████████▏| 1597/1728 [8:41:45<06:10, 2.83s/it]
92%|█████████▏| 1598/1728 [8:41:48<06:08, 2.83s/it]
93%|█████████▎| 1599/1728 [8:41:50<06:05, 2.83s/it]
93%|█████████▎| 1600/1728 [8:41:53<05:58, 2.80s/it]
{'loss': '0.326', 'grad_norm': '0.006554', 'learning_rate': '2.355e-07', 'epoch': '6.776'}
+
93%|█████████▎| 1600/1728 [8:41:53<05:58, 2.80s/it]
93%|█████████▎| 1601/1728 [8:41:56<05:56, 2.81s/it]
93%|█████████▎| 1602/1728 [8:41:59<05:52, 2.80s/it]
93%|█████████▎| 1603/1728 [8:42:02<05:47, 2.78s/it]
93%|█████████▎| 1604/1728 [8:42:04<05:44, 2.78s/it]
93%|█████████▎| 1605/1728 [8:42:07<05:40, 2.77s/it]
93%|█████████▎| 1606/1728 [8:42:10<05:56, 2.92s/it]
93%|█████████▎| 1607/1728 [8:42:13<05:50, 2.89s/it]
93%|█████████▎| 1608/1728 [8:42:16<05:40, 2.84s/it]
93%|█████████▎| 1609/1728 [8:42:19<05:35, 2.82s/it]
93%|█████████▎| 1610/1728 [8:42:21<05:30, 2.80s/it]
93%|█████████▎| 1611/1728 [8:42:24<05:28, 2.81s/it]
93%|█████████▎| 1612/1728 [8:42:27<05:29, 2.84s/it]
93%|█████████▎| 1613/1728 [8:42:30<05:22, 2.80s/it]
93%|█████████▎| 1614/1728 [8:42:33<05:29, 2.89s/it]
93%|█████████▎| 1615/1728 [8:42:36<05:28, 2.90s/it]
94%|█████████▎| 1616/1728 [8:42:39<05:23, 2.89s/it]
94%|█████████▎| 1617/1728 [8:42:42<05:17, 2.86s/it]
94%|█████████▎| 1618/1728 [8:42:44<05:14, 2.86s/it]
94%|█████████▎| 1619/1728 [8:42:47<05:07, 2.82s/it]
94%|█████████▍| 1620/1728 [8:42:50<05:06, 2.84s/it]
{'loss': '0.3416', 'grad_norm': '0.004699', 'learning_rate': '1.969e-07', 'epoch': '6.961'}
+
94%|█████████▍| 1620/1728 [8:42:50<05:06, 2.84s/it]
94%|█████████▍| 1621/1728 [8:42:53<05:00, 2.81s/it]
94%|█████████▍| 1622/1728 [8:42:56<04:56, 2.80s/it]
94%|█████████▍| 1623/1728 [8:42:58<04:55, 2.81s/it]
94%|█████████▍| 1624/1728 [8:43:01<04:56, 2.85s/it]
94%|█████████▍| 1625/1728 [8:43:02<03:50, 2.24s/it][INFO|trainer.py:2668] 2026-08-23 11:24:40,534 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 11:24:40,534 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 11:24:40,534 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/da_kd/checkpoint-1625
+[INFO|configuration_utils.py:780] 2026-08-23 12:12:48,456 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 12:12:48,457 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 12:12:48,493 >> chat template saved in results/qwen3/da_kd/checkpoint-1625/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 12:12:48,493 >> tokenizer config file saved in results/qwen3/da_kd/checkpoint-1625/tokenizer_config.json
+DA-KD epoch=8 ratio=0.2061 active=2068/10027 dds_mean=1041867.0569
+
94%|█████████▍| 1626/1728 [9:56:38<37:34:46, 1326.34s/it]
94%|█████████▍| 1627/1728 [9:56:41<26:04:18, 929.30s/it]
94%|█████████▍| 1628/1728 [9:56:44<18:05:30, 651.31s/it]
94%|█████████▍| 1629/1728 [9:56:46<12:33:37, 456.74s/it]
94%|█████████▍| 1630/1728 [9:56:49<8:43:43, 320.64s/it]
94%|█████████▍| 1631/1728 [9:56:52<6:04:10, 225.26s/it]
94%|█████████▍| 1632/1728 [9:56:55<4:13:45, 158.60s/it]
95%|█████████▍| 1633/1728 [9:56:58<2:57:15, 111.95s/it]
95%|█████████▍| 1634/1728 [9:57:01<2:04:02, 79.18s/it]
95%|█████████▍| 1635/1728 [9:57:04<1:27:10, 56.24s/it]
95%|█████████▍| 1636/1728 [9:57:06<1:01:38, 40.21s/it]
95%|█████████▍| 1637/1728 [9:57:09<43:54, 28.95s/it]
95%|█████████▍| 1638/1728 [9:57:12<31:38, 21.09s/it]
95%|█████████▍| 1639/1728 [9:57:15<23:08, 15.60s/it]
95%|█████████▍| 1640/1728 [9:57:17<17:12, 11.74s/it]
{'loss': '0.4238', 'grad_norm': '0.001789', 'learning_rate': '1.647e-07', 'epoch': '7.232'}
+
95%|█████████▍| 1640/1728 [9:57:17<17:12, 11.74s/it]
95%|█████████▍| 1641/1728 [9:57:20<13:06, 9.04s/it]
95%|█████████▌| 1642/1728 [9:57:23<10:17, 7.18s/it]
95%|█████████▌| 1643/1728 [9:57:26<08:16, 5.84s/it]
95%|█████████▌| 1644/1728 [9:57:29<06:53, 4.92s/it]
95%|█████████▌| 1645/1728 [9:57:31<05:53, 4.25s/it]
95%|█████████▌| 1646/1728 [9:57:34<05:10, 3.78s/it]
95%|█████████▌| 1647/1728 [9:57:37<04:40, 3.46s/it]
95%|█████████▌| 1648/1728 [9:57:39<04:18, 3.24s/it]
95%|█████████▌| 1649/1728 [9:57:42<04:03, 3.08s/it]
95%|█████████▌| 1650/1728 [9:57:45<03:51, 2.97s/it]
96%|█████████▌| 1651/1728 [9:57:47<03:42, 2.89s/it]
96%|█████████▌| 1652/1728 [9:57:50<03:35, 2.83s/it]
96%|█████████▌| 1653/1728 [9:57:53<03:29, 2.80s/it]
96%|█████████▌| 1654/1728 [9:57:56<03:24, 2.76s/it]
96%|█████████▌| 1655/1728 [9:57:58<03:20, 2.75s/it]
96%|█████████▌| 1656/1728 [9:58:01<03:17, 2.74s/it]
96%|█████████▌| 1657/1728 [9:58:04<03:15, 2.75s/it]
96%|█████████▌| 1658/1728 [9:58:06<03:11, 2.74s/it]
96%|█████████▌| 1659/1728 [9:58:09<03:07, 2.72s/it]
96%|█████████▌| 1660/1728 [9:58:12<03:04, 2.71s/it]
{'loss': '0.4718', 'grad_norm': '0.002567', 'learning_rate': '1.389e-07', 'epoch': '7.542'}
+
96%|█████████▌| 1660/1728 [9:58:12<03:04, 2.71s/it]
96%|█████████▌| 1661/1728 [9:58:15<03:01, 2.70s/it]
96%|█████████▌| 1662/1728 [9:58:17<02:59, 2.72s/it]
96%|█████████▌| 1663/1728 [9:58:20<02:57, 2.73s/it]
96%|█████████▋| 1664/1728 [9:58:23<02:53, 2.71s/it]
96%|█████████▋| 1665/1728 [9:58:25<02:50, 2.71s/it]
96%|█████████▋| 1666/1728 [9:58:28<02:47, 2.71s/it]
96%|█████████▋| 1667/1728 [9:58:31<02:44, 2.70s/it]
97%|█████████▋| 1668/1728 [9:58:34<02:41, 2.70s/it]
97%|█████████▋| 1669/1728 [9:58:36<02:38, 2.68s/it]
97%|█████████▋| 1670/1728 [9:58:39<02:35, 2.69s/it]
97%|█████████▋| 1671/1728 [9:58:42<02:33, 2.69s/it]
97%|█████████▋| 1672/1728 [9:58:44<02:30, 2.68s/it]
97%|█████████▋| 1673/1728 [9:58:47<02:28, 2.70s/it]
97%|█████████▋| 1674/1728 [9:58:50<02:25, 2.69s/it]
97%|█████████▋| 1675/1728 [9:58:52<02:22, 2.68s/it]
97%|█████████▋| 1676/1728 [9:58:55<02:19, 2.68s/it]
97%|█████████▋| 1677/1728 [9:58:58<02:16, 2.67s/it]
97%|█████████▋| 1678/1728 [9:59:00<02:13, 2.67s/it]
97%|█████████▋| 1679/1728 [9:59:03<02:10, 2.66s/it]
97%|█████████▋| 1680/1728 [9:59:06<02:08, 2.67s/it]
{'loss': '0.4646', 'grad_norm': '0.001038', 'learning_rate': '1.196e-07', 'epoch': '7.851'}
+
97%|█████████▋| 1680/1728 [9:59:06<02:08, 2.67s/it]
97%|█████████▋| 1681/1728 [9:59:08<02:06, 2.68s/it]
97%|█████████▋| 1682/1728 [9:59:11<02:03, 2.69s/it]
97%|█████████▋| 1683/1728 [9:59:14<02:01, 2.70s/it]
97%|█████████▋| 1684/1728 [9:59:16<01:58, 2.70s/it]
98%|█████████▊| 1685/1728 [9:59:19<01:55, 2.69s/it]
98%|█████████▊| 1686/1728 [9:59:22<01:53, 2.71s/it]
98%|█████████▊| 1687/1728 [9:59:25<01:51, 2.72s/it]
98%|█████████▊| 1688/1728 [9:59:27<01:48, 2.72s/it]
98%|█████████▊| 1689/1728 [9:59:30<01:46, 2.72s/it]
98%|█████████▊| 1690/1728 [9:59:32<01:33, 2.46s/it][INFO|trainer.py:2668] 2026-08-23 12:41:10,298 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 12:41:10,298 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 12:41:10,298 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/da_kd/checkpoint-1690
+[INFO|configuration_utils.py:780] 2026-08-23 13:29:03,965 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 13:29:03,968 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 13:29:04,003 >> chat template saved in results/qwen3/da_kd/checkpoint-1690/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 13:29:04,003 >> tokenizer config file saved in results/qwen3/da_kd/checkpoint-1690/tokenizer_config.json
+DA-KD epoch=9 ratio=0.0955 active=958/10027 dds_mean=1042068.3715
+
98%|█████████▊| 1691/1728 [11:12:53<13:35:12, 1321.95s/it]
98%|█████████▊| 1692/1728 [11:12:55<9:15:43, 926.20s/it]
98%|█████████▊| 1693/1728 [11:12:58<6:18:40, 649.16s/it]
98%|█████████▊| 1694/1728 [11:13:01<4:17:57, 455.22s/it]
98%|█████████▊| 1695/1728 [11:13:04<2:55:41, 319.45s/it]
98%|█████████▊| 1696/1728 [11:13:06<1:59:42, 224.44s/it]
98%|█████████▊| 1697/1728 [11:13:09<1:21:35, 157.91s/it]
98%|█████████▊| 1698/1728 [11:13:12<55:40, 111.36s/it]
98%|█████████▊| 1699/1728 [11:13:14<38:03, 78.76s/it]
98%|█████████▊| 1700/1728 [11:13:17<26:06, 55.94s/it]
{'loss': '0.4759', 'grad_norm': '0.0009913', 'learning_rate': '1.069e-07', 'epoch': '8.333'}
+
98%|█████████▊| 1700/1728 [11:13:17<26:06, 55.94s/it]
98%|█████████▊| 1701/1728 [11:13:20<17:59, 39.97s/it]
98%|█████████▊| 1702/1728 [11:13:22<12:28, 28.78s/it]
99%|█████████▊| 1703/1728 [11:13:25<08:43, 20.93s/it]
99%|█████████▊| 1704/1728 [11:13:28<06:10, 15.45s/it]
99%|█████████▊| 1705/1728 [11:13:30<04:26, 11.61s/it]
99%|█████████▊| 1706/1728 [11:13:33<03:16, 8.93s/it]
99%|█████████▉| 1707/1728 [11:13:36<02:28, 7.05s/it]
99%|█████████▉| 1708/1728 [11:13:38<01:54, 5.74s/it]
99%|█████████▉| 1709/1728 [11:13:41<01:31, 4.82s/it]
99%|█████████▉| 1710/1728 [11:13:44<01:15, 4.17s/it]
99%|█████████▉| 1711/1728 [11:13:46<01:03, 3.72s/it]
99%|█████████▉| 1712/1728 [11:13:49<00:54, 3.41s/it]
99%|█████████▉| 1713/1728 [11:13:52<00:47, 3.18s/it]
99%|█████████▉| 1714/1728 [11:13:54<00:42, 3.05s/it]
99%|█████████▉| 1715/1728 [11:13:57<00:38, 2.94s/it]
99%|█████████▉| 1716/1728 [11:14:00<00:34, 2.87s/it]
99%|█████████▉| 1717/1728 [11:14:03<00:30, 2.81s/it]
99%|█████████▉| 1718/1728 [11:14:05<00:27, 2.78s/it]
99%|█████████▉| 1719/1728 [11:14:08<00:24, 2.75s/it]
100%|█████████▉| 1720/1728 [11:14:11<00:21, 2.74s/it]
{'loss': '0.5103', 'grad_norm': '0.00103', 'learning_rate': '1.007e-07', 'epoch': '9'}
+
100%|█████████▉| 1720/1728 [11:14:11<00:21, 2.74s/it][INFO|trainer.py:2668] 2026-08-23 13:55:49,006 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 13:55:49,006 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 13:55:49,006 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/da_kd/checkpoint-1720
+[INFO|configuration_utils.py:780] 2026-08-23 14:44:18,631 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 14:44:18,634 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 14:44:18,670 >> chat template saved in results/qwen3/da_kd/checkpoint-1720/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 14:44:18,670 >> tokenizer config file saved in results/qwen3/da_kd/checkpoint-1720/tokenizer_config.json
diff --git a/qwen3/run_all_logs/20260822_031516/fkl.log b/qwen3/run_all_logs/20260822_031516/fkl.log
new file mode 100644
index 0000000000000000000000000000000000000000..7679b9106a1f20cf65f23574d503530d3b3669e4
--- /dev/null
+++ b/qwen3/run_all_logs/20260822_031516/fkl.log
@@ -0,0 +1,2100 @@
+W0822 10:10:21.489000 1363392 torch/distributed/run.py:803]
+W0822 10:10:21.489000 1363392 torch/distributed/run.py:803] *****************************************
+W0822 10:10:21.489000 1363392 torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+W0822 10:10:21.489000 1363392 torch/distributed/run.py:803] *****************************************
+[INFO|2026-08-22 10:10:28] llamafactory.hparams.parser:651 >> Process rank: 1, world size: 2, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16
+[INFO|2026-08-22 10:10:28] llamafactory.hparams.parser:144 >> Set `ddp_find_unused_parameters` to False in DDP training since LoRA is enabled.
+[INFO|2026-08-22 10:10:28] llamafactory.hparams.parser:651 >> Process rank: 0, world size: 2, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16
+Starting fkl with LlamaFactory and DeepSpeed=True.
+[INFO|configuration_utils.py:780] 2026-08-22 10:10:29,715 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 10:10:29,723 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 10:10:32,969 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 10:10:32,972 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 10:10:33,210 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 10:10:33,213 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 10:10:34] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_train.jsonl...
+[INFO|2026-08-22 10:10:35] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_eval.jsonl...
+training example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 1947, 19029, 10592, 311, 35592, 304, 279, 17328, 1947, 1707, 429, 9482, 1573, 279, 1042, 220, 16, 21, 15, 15, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 9286, 38050, 756, 414, 330, 13193, 788, 341, 286, 330, 408, 14645, 788, 330, 68689, 756, 286, 330, 2468, 14645, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 37525, 756, 414, 330, 13193, 788, 5613, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 18098, 287, 756, 414, 330, 13193, 788, 341, 286, 330, 11141, 3575, 34043, 788, 330, 16468, 756, 286, 330, 37375, 14645, 788, 330, 68689, 698, 414, 456, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 9286, 38050, 756, 414, 330, 1313, 788, 330, 49087, 2354, 698, 262, 1153, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 37525, 756, 414, 330, 1313, 788, 330, 4648, 37525, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of art movements linked to paintings in the genre art style that ended before the year 1600?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "ArtMovement",
+ "properties": {
+ "end_year": "INTEGER",
+ "start_year": "INTEGER"
+ }
+ },
+ {
+ "label": "Genre",
+ "properties": {}
+ },
+ {
+ "label": "Painting",
+ "properties": {
+ "country_of_origin": "STRING",
+ "creation_year": "INTEGER"
+ }
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "ArtMovement",
+ "type": "associatedWith"
+ },
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "Genre",
+ "type": "hasGenre"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+eval example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 9893, 1483, 304, 2987, 279, 220, 17, 15, 17, 16, 55218, 18144, 73379, 476, 279, 23007, 24847, 7755, 9257, 36331, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 13193, 788, 341, 286, 330, 1028, 788, 330, 7097, 756, 286, 330, 31309, 788, 330, 22852, 756, 286, 330, 4082, 3575, 2259, 26286, 788, 330, 68689, 756, 286, 330, 4082, 3575, 1243, 89827, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 28214, 756, 414, 330, 13193, 788, 5613, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 5657, 788, 330, 28214, 756, 414, 330, 1313, 788, 330, 30210, 1047, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of weapons used in either the 2021 Baghdad suicide bombings or the Botroseya Church bombing?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "TerroristAttack",
+ "properties": {
+ "date": "DATE",
+ "locations": "LIST",
+ "number_of_deaths": "INTEGER",
+ "number_of_injuries": "INTEGER"
+ }
+ },
+ {
+ "label": "Weapon",
+ "properties": {}
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "TerroristAttack",
+ "target": "Weapon",
+ "type": "employs"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+[INFO|configuration_utils.py:780] 2026-08-22 10:10:39,217 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 10:10:39,218 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 10:10:39] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training.
+[INFO|modeling_utils.py:769] 2026-08-22 10:10:39,940 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/model.safetensors
+[INFO|modeling_utils.py:839] 2026-08-22 10:10:39,940 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 10:10:39,942 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": false
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 10:10:40,577 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95
+}
+
+[INFO|2026-08-22 10:10:40] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled.
+[INFO|2026-08-22 10:10:40] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-22 10:10:40] llamafactory.model.adapter:144 >> Upcasting trainable params to float32.
+[INFO|2026-08-22 10:10:40] llamafactory.model.adapter:144 >> Fine-tuning method: LoRA
+[INFO|2026-08-22 10:10:40] llamafactory.model.model_utils.misc:144 >> Found linear modules: o_proj,q_proj,v_proj,k_proj,up_proj,down_proj,gate_proj
+[INFO|2026-08-22 10:10:41] llamafactory.model.loader:144 >> trainable params: 20,185,088 || all params: 616,235,008 || trainable%: 3.2756
+[INFO|configuration_utils.py:780] 2026-08-22 10:10:41,509 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 10:10:41,511 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 10:10:44,324 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 10:10:44,571 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 10:10:44,573 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 10:10:46,596 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 10:10:46,598 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 10:10:46] llamafactory.model.model_utils.kv_cache:144 >> KV cache is enabled for faster generation.
+[INFO|modeling_utils.py:769] 2026-08-22 10:10:46,872 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model.safetensors.index.json
+[INFO|modeling_utils.py:839] 2026-08-22 10:10:46,872 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 10:10:46,873 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": true
+}
+
+
Loading weights: 0%| | 0/398 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 10:10:48,848 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.7,
+ "top_k": 20,
+ "top_p": 0.8
+}
+
+[INFO|2026-08-22 10:10:48] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-22 10:10:49] llamafactory.model.adapter:144 >> Merged 1 adapter(s).
+[INFO|2026-08-22 10:10:49] llamafactory.model.adapter:144 >> Loaded adapter(s): results/qwen3/teacher_lora
+[INFO|2026-08-22 10:10:49] llamafactory.model.loader:144 >> all params: 4,022,468,096
+[INFO|2026-08-22 10:10:49] llamafactory.train.trainer_utils:144 >> Created reference model from Qwen/Qwen3-4B-Instruct-2507
+
Loading weights: 0%| | 0/398 [00:00> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}.
+Gradient accumulation steps mismatch: GradientAccumulationPlugin has 1, DeepSpeed config has 8. Using DeepSpeed's value.
+[INFO|trainer.py:1469] 2026-08-22 10:10:54,921 >> ***** Running training *****
+[INFO|trainer.py:1470] 2026-08-22 10:10:54,921 >> Num examples = 10,027
+[INFO|trainer.py:1471] 2026-08-22 10:10:54,921 >> Num Epochs = 5
+[INFO|trainer.py:1472] 2026-08-22 10:10:54,921 >> Num update steps per epoch = 314
+[INFO|trainer.py:1473] 2026-08-22 10:10:54,921 >> Instantaneous batch size per device = 2
+[INFO|trainer.py:1476] 2026-08-22 10:10:54,921 >> Total train batch size (w. parallel, distributed & accumulation) = 32
+[INFO|trainer.py:1477] 2026-08-22 10:10:54,922 >> Gradient Accumulation steps = 8
+[INFO|trainer.py:1478] 2026-08-22 10:10:54,922 >> Total optimization steps = 1,570
+[INFO|trainer.py:1479] 2026-08-22 10:10:54,925 >> Number of trainable parameters = 20,185,088
+
0%| | 0/1570 [00:00>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 10:30:46,128 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 10:30:46,128 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/fkl/checkpoint-314
+[INFO|configuration_utils.py:780] 2026-08-22 10:40:58,635 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 10:40:58,638 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 10:40:58,674 >> chat template saved in results/qwen3/fkl/checkpoint-314/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 10:40:58,674 >> tokenizer config file saved in results/qwen3/fkl/checkpoint-314/tokenizer_config.json
+
20%|██ | 315/1570 [30:07<65:01:39, 186.53s/it]
20%|██ | 316/1570 [30:10<45:47:05, 131.44s/it]
20%|██ | 317/1570 [30:13<32:19:10, 92.86s/it]
20%|██ | 318/1570 [30:16<22:54:37, 65.88s/it]
20%|██ | 319/1570 [30:19<16:19:14, 46.97s/it]
20%|██ | 320/1570 [30:21<11:42:30, 33.72s/it]
{'loss': '0.01401', 'grad_norm': '1.464', 'learning_rate': '9.016e-05', 'epoch': '1.019'}
+
20%|██ | 320/1570 [30:21<11:42:30, 33.72s/it]
20%|██ | 321/1570 [30:24<8:29:10, 24.46s/it]
21%|██ | 322/1570 [30:27<6:13:23, 17.95s/it]
21%|██ | 323/1570 [30:30<4:38:53, 13.42s/it]
21%|██ | 324/1570 [30:33<3:32:48, 10.25s/it]
21%|██ | 325/1570 [30:36<2:46:33, 8.03s/it]
21%|██ | 326/1570 [30:38<2:13:47, 6.45s/it]
21%|██ | 327/1570 [30:41<1:51:08, 5.37s/it]
21%|██ | 328/1570 [30:44<1:35:14, 4.60s/it]
21%|██ | 329/1570 [30:47<1:24:11, 4.07s/it]
21%|██ | 330/1570 [30:50<1:16:50, 3.72s/it]
21%|██ | 331/1570 [30:53<1:11:24, 3.46s/it]
21%|██ | 332/1570 [30:55<1:07:28, 3.27s/it]
21%|██ | 333/1570 [30:58<1:04:46, 3.14s/it]
21%|██▏ | 334/1570 [31:01<1:02:55, 3.05s/it]
21%|██▏ | 335/1570 [31:04<1:01:44, 3.00s/it]
21%|██▏ | 336/1570 [31:07<1:00:39, 2.95s/it]
21%|██▏ | 337/1570 [31:10<59:59, 2.92s/it]
22%|██▏ | 338/1570 [31:12<59:30, 2.90s/it]
22%|██▏ | 339/1570 [31:15<59:11, 2.89s/it]
22%|██▏ | 340/1570 [31:18<59:23, 2.90s/it]
{'loss': '0.006688', 'grad_norm': '0.08159', 'learning_rate': '8.894e-05', 'epoch': '1.083'}
+
22%|██▏ | 340/1570 [31:18<59:23, 2.90s/it]
22%|██▏ | 341/1570 [31:21<59:18, 2.90s/it]
22%|██▏ | 342/1570 [31:24<59:11, 2.89s/it]
22%|██▏ | 343/1570 [31:27<58:50, 2.88s/it]
22%|██▏ | 344/1570 [31:30<58:24, 2.86s/it]
22%|██▏ | 345/1570 [31:33<58:16, 2.85s/it]
22%|██▏ | 346/1570 [31:35<58:03, 2.85s/it]
22%|██▏ | 347/1570 [31:38<58:20, 2.86s/it]
22%|██▏ | 348/1570 [31:41<58:12, 2.86s/it]
22%|██▏ | 349/1570 [31:44<58:08, 2.86s/it]
22%|██▏ | 350/1570 [31:47<57:49, 2.84s/it]
22%|██▏ | 351/1570 [31:50<57:31, 2.83s/it]
22%|██▏ | 352/1570 [31:52<57:22, 2.83s/it]
22%|██▏ | 353/1570 [31:55<57:30, 2.84s/it]
23%|██▎ | 354/1570 [31:58<57:46, 2.85s/it]
23%|██▎ | 355/1570 [32:01<57:50, 2.86s/it]
23%|██▎ | 356/1570 [32:04<57:44, 2.85s/it]
23%|██▎ | 357/1570 [32:07<57:54, 2.86s/it]
23%|██▎ | 358/1570 [32:10<57:53, 2.87s/it]
23%|██▎ | 359/1570 [32:12<57:52, 2.87s/it]
23%|██▎ | 360/1570 [32:15<57:35, 2.86s/it]
{'loss': '0.006568', 'grad_norm': '0.1202', 'learning_rate': '8.766e-05', 'epoch': '1.147'}
+
23%|██▎ | 360/1570 [32:15<57:35, 2.86s/it]
23%|██▎ | 361/1570 [32:18<57:44, 2.87s/it]
23%|██▎ | 362/1570 [32:21<57:28, 2.86s/it]
23%|██▎ | 363/1570 [32:24<57:25, 2.85s/it]
23%|██▎ | 364/1570 [32:27<57:32, 2.86s/it]
23%|██▎ | 365/1570 [32:30<57:31, 2.86s/it]
23%|██▎ | 366/1570 [32:32<57:32, 2.87s/it]
23%|██▎ | 367/1570 [32:35<57:17, 2.86s/it]
23%|██▎ | 368/1570 [32:38<57:14, 2.86s/it]
24%|██▎ | 369/1570 [32:41<57:23, 2.87s/it]
24%|██▎ | 370/1570 [32:44<57:24, 2.87s/it]
24%|██▎ | 371/1570 [32:47<57:20, 2.87s/it]
24%|██▎ | 372/1570 [32:50<57:13, 2.87s/it]
24%|██▍ | 373/1570 [32:53<57:14, 2.87s/it]
24%|██▍ | 374/1570 [32:55<57:06, 2.87s/it]
24%|██▍ | 375/1570 [32:58<57:11, 2.87s/it]
24%|██▍ | 376/1570 [33:01<57:37, 2.90s/it]
24%|██▍ | 377/1570 [33:04<57:18, 2.88s/it]
24%|██▍ | 378/1570 [33:07<57:03, 2.87s/it]
24%|██▍ | 379/1570 [33:10<56:50, 2.86s/it]
24%|██▍ | 380/1570 [33:13<56:25, 2.85s/it]
{'loss': '0.004217', 'grad_norm': '0.2054', 'learning_rate': '8.631e-05', 'epoch': '1.211'}
+
24%|██▍ | 380/1570 [33:13<56:25, 2.85s/it]
24%|██▍ | 381/1570 [33:15<56:20, 2.84s/it]
24%|██▍ | 382/1570 [33:18<56:16, 2.84s/it]
24%|██▍ | 383/1570 [33:21<56:20, 2.85s/it]
24%|██▍ | 384/1570 [33:24<56:08, 2.84s/it]
25%|██▍ | 385/1570 [33:27<55:59, 2.84s/it]
25%|██▍ | 386/1570 [33:30<55:52, 2.83s/it]
25%|██▍ | 387/1570 [33:32<56:01, 2.84s/it]
25%|██▍ | 388/1570 [33:35<55:57, 2.84s/it]
25%|██▍ | 389/1570 [33:38<55:56, 2.84s/it]
25%|██▍ | 390/1570 [33:41<55:44, 2.83s/it]
25%|██▍ | 391/1570 [33:44<55:47, 2.84s/it]
25%|██▍ | 392/1570 [33:47<55:53, 2.85s/it]
25%|██▌ | 393/1570 [33:50<55:45, 2.84s/it]
25%|██▌ | 394/1570 [33:52<55:55, 2.85s/it]
25%|██▌ | 395/1570 [33:55<55:48, 2.85s/it]
25%|██▌ | 396/1570 [33:58<55:45, 2.85s/it]
25%|██▌ | 397/1570 [34:01<55:46, 2.85s/it]
25%|██▌ | 398/1570 [34:04<55:31, 2.84s/it]
25%|██▌ | 399/1570 [34:07<55:17, 2.83s/it]
25%|██▌ | 400/1570 [34:09<55:12, 2.83s/it]
{'loss': '0.01048', 'grad_norm': '0.1577', 'learning_rate': '8.491e-05', 'epoch': '1.274'}
+
25%|██▌ | 400/1570 [34:09<55:12, 2.83s/it]
26%|██▌ | 401/1570 [34:12<55:15, 2.84s/it]
26%|██▌ | 402/1570 [34:15<55:00, 2.83s/it]
26%|██▌ | 403/1570 [34:18<55:12, 2.84s/it]
26%|██▌ | 404/1570 [34:21<55:12, 2.84s/it]
26%|██▌ | 405/1570 [34:24<55:20, 2.85s/it]
26%|██▌ | 406/1570 [34:27<55:30, 2.86s/it]
26%|██▌ | 407/1570 [34:29<55:31, 2.86s/it]
26%|██▌ | 408/1570 [34:32<55:13, 2.85s/it]
26%|██▌ | 409/1570 [34:35<55:17, 2.86s/it]
26%|██▌ | 410/1570 [34:38<54:59, 2.84s/it]
26%|██▌ | 411/1570 [34:41<54:51, 2.84s/it]
26%|██▌ | 412/1570 [34:44<54:47, 2.84s/it]
26%|██▋ | 413/1570 [34:46<54:46, 2.84s/it]
26%|██▋ | 414/1570 [34:49<54:39, 2.84s/it]
26%|██▋ | 415/1570 [34:52<54:19, 2.82s/it]
26%|██▋ | 416/1570 [34:55<54:29, 2.83s/it]
27%|██▋ | 417/1570 [34:58<54:12, 2.82s/it]
27%|██▋ | 418/1570 [35:01<54:22, 2.83s/it]
27%|██▋ | 419/1570 [35:03<54:16, 2.83s/it]
27%|██▋ | 420/1570 [35:06<54:16, 2.83s/it]
{'loss': '0.009177', 'grad_norm': '0.1818', 'learning_rate': '8.345e-05', 'epoch': '1.338'}
+
27%|██▋ | 420/1570 [35:06<54:16, 2.83s/it]
27%|██▋ | 421/1570 [35:09<54:25, 2.84s/it]
27%|██▋ | 422/1570 [35:12<54:11, 2.83s/it]
27%|██▋ | 423/1570 [35:15<54:12, 2.84s/it]
27%|██▋ | 424/1570 [35:18<54:07, 2.83s/it]
27%|██▋ | 425/1570 [35:20<54:03, 2.83s/it]
27%|██▋ | 426/1570 [35:23<53:56, 2.83s/it]
27%|██▋ | 427/1570 [35:26<53:50, 2.83s/it]
27%|██▋ | 428/1570 [35:29<53:40, 2.82s/it]
27%|██▋ | 429/1570 [35:32<53:52, 2.83s/it]
27%|██▋ | 430/1570 [35:35<53:54, 2.84s/it]
27%|██▋ | 431/1570 [35:37<53:56, 2.84s/it]
28%|██▊ | 432/1570 [35:40<53:49, 2.84s/it]
28%|██▊ | 433/1570 [35:43<53:39, 2.83s/it]
28%|██▊ | 434/1570 [35:46<53:29, 2.83s/it]
28%|██▊ | 435/1570 [35:49<53:32, 2.83s/it]
28%|██▊ | 436/1570 [35:52<53:32, 2.83s/it]
28%|██▊ | 437/1570 [35:54<53:30, 2.83s/it]
28%|██▊ | 438/1570 [35:57<53:18, 2.83s/it]
28%|██▊ | 439/1570 [36:00<53:04, 2.82s/it]
28%|██▊ | 440/1570 [36:03<53:10, 2.82s/it]
{'loss': '0.004672', 'grad_norm': '0.04398', 'learning_rate': '8.194e-05', 'epoch': '1.402'}
+
28%|██▊ | 440/1570 [36:03<53:10, 2.82s/it]
28%|██▊ | 441/1570 [36:06<53:09, 2.82s/it]
28%|██▊ | 442/1570 [36:08<53:06, 2.82s/it]
28%|██▊ | 443/1570 [36:11<53:01, 2.82s/it]
28%|██▊ | 444/1570 [36:14<52:58, 2.82s/it]
28%|██▊ | 445/1570 [36:17<52:48, 2.82s/it]
28%|██▊ | 446/1570 [36:20<52:54, 2.82s/it]
28%|██▊ | 447/1570 [36:23<52:54, 2.83s/it]
29%|██▊ | 448/1570 [36:25<52:38, 2.82s/it]
29%|██▊ | 449/1570 [36:28<53:02, 2.84s/it]
29%|██▊ | 450/1570 [36:31<52:57, 2.84s/it]
29%|██▊ | 451/1570 [36:34<52:50, 2.83s/it]
29%|██▉ | 452/1570 [36:37<52:56, 2.84s/it]
29%|██▉ | 453/1570 [36:40<52:59, 2.85s/it]
29%|██▉ | 454/1570 [36:42<52:46, 2.84s/it]
29%|██▉ | 455/1570 [36:45<52:38, 2.83s/it]
29%|██▉ | 456/1570 [36:48<52:36, 2.83s/it]
29%|██▉ | 457/1570 [36:51<52:21, 2.82s/it]
29%|██▉ | 458/1570 [36:54<52:14, 2.82s/it]
29%|██▉ | 459/1570 [36:57<52:12, 2.82s/it]
29%|██▉ | 460/1570 [36:59<52:02, 2.81s/it]
{'loss': '0.003688', 'grad_norm': '0.2848', 'learning_rate': '8.037e-05', 'epoch': '1.466'}
+
29%|██▉ | 460/1570 [36:59<52:02, 2.81s/it]
29%|██▉ | 461/1570 [37:02<52:04, 2.82s/it]
29%|██▉ | 462/1570 [37:05<52:11, 2.83s/it]
29%|██▉ | 463/1570 [37:08<52:10, 2.83s/it]
30%|██▉ | 464/1570 [37:11<52:05, 2.83s/it]
30%|██▉ | 465/1570 [37:13<51:54, 2.82s/it]
30%|██▉ | 466/1570 [37:16<51:47, 2.81s/it]
30%|██▉ | 467/1570 [37:19<51:56, 2.83s/it]
30%|██▉ | 468/1570 [37:22<51:53, 2.83s/it]
30%|██▉ | 469/1570 [37:25<51:52, 2.83s/it]
30%|██▉ | 470/1570 [37:28<51:36, 2.81s/it]
30%|███ | 471/1570 [37:30<51:42, 2.82s/it]
30%|███ | 472/1570 [37:33<51:44, 2.83s/it]
30%|███ | 473/1570 [37:36<51:41, 2.83s/it]
30%|███ | 474/1570 [37:39<51:18, 2.81s/it]
30%|███ | 475/1570 [37:42<51:23, 2.82s/it]
30%|███ | 476/1570 [37:44<51:32, 2.83s/it]
30%|███ | 477/1570 [37:47<51:29, 2.83s/it]
30%|███ | 478/1570 [37:50<51:25, 2.83s/it]
31%|███ | 479/1570 [37:53<51:25, 2.83s/it]
31%|███ | 480/1570 [37:56<51:22, 2.83s/it]
{'loss': '0.002212', 'grad_norm': '0.4655', 'learning_rate': '7.876e-05', 'epoch': '1.53'}
+
31%|███ | 480/1570 [37:56<51:22, 2.83s/it]
31%|███ | 481/1570 [37:59<51:24, 2.83s/it]
31%|███ | 482/1570 [38:01<51:21, 2.83s/it]
31%|███ | 483/1570 [38:04<51:05, 2.82s/it]
31%|███ | 484/1570 [38:07<50:54, 2.81s/it]
31%|███ | 485/1570 [38:10<51:03, 2.82s/it]
31%|███ | 486/1570 [38:13<51:00, 2.82s/it]
31%|███ | 487/1570 [38:16<51:03, 2.83s/it]
31%|███ | 488/1570 [38:18<50:54, 2.82s/it]
31%|███ | 489/1570 [38:21<50:48, 2.82s/it]
31%|███ | 490/1570 [38:24<50:54, 2.83s/it]
31%|███▏ | 491/1570 [38:27<50:49, 2.83s/it]
31%|███▏ | 492/1570 [38:30<51:05, 2.84s/it]
31%|███▏ | 493/1570 [38:33<51:08, 2.85s/it]
31%|███▏ | 494/1570 [38:35<50:55, 2.84s/it]
32%|███▏ | 495/1570 [38:38<50:52, 2.84s/it]
32%|███▏ | 496/1570 [38:41<50:25, 2.82s/it]
32%|███▏ | 497/1570 [38:44<50:19, 2.81s/it]
32%|███▏ | 498/1570 [38:47<50:33, 2.83s/it]
32%|███▏ | 499/1570 [38:50<50:31, 2.83s/it]
32%|███▏ | 500/1570 [38:52<50:18, 2.82s/it]
{'loss': '0.006767', 'grad_norm': '0.05735', 'learning_rate': '7.71e-05', 'epoch': '1.594'}
+
32%|███▏ | 500/1570 [38:52<50:18, 2.82s/it]
32%|███▏ | 501/1570 [38:55<50:03, 2.81s/it]
32%|███▏ | 502/1570 [38:58<50:14, 2.82s/it]
32%|███▏ | 503/1570 [39:01<50:15, 2.83s/it]
32%|███▏ | 504/1570 [39:04<50:15, 2.83s/it]
32%|███▏ | 505/1570 [39:06<50:14, 2.83s/it]
32%|███▏ | 506/1570 [39:09<49:55, 2.82s/it]
32%|███▏ | 507/1570 [39:12<49:48, 2.81s/it]
32%|███▏ | 508/1570 [39:15<49:38, 2.80s/it]
32%|███▏ | 509/1570 [39:18<49:31, 2.80s/it]
32%|███▏ | 510/1570 [39:20<49:27, 2.80s/it]
33%|███▎ | 511/1570 [39:23<49:24, 2.80s/it]
33%|███▎ | 512/1570 [39:26<49:29, 2.81s/it]
33%|███▎ | 513/1570 [39:29<49:30, 2.81s/it]
33%|███▎ | 514/1570 [39:32<49:36, 2.82s/it]
33%|███▎ | 515/1570 [39:35<49:19, 2.81s/it]
33%|███▎ | 516/1570 [39:37<49:16, 2.81s/it]
33%|███▎ | 517/1570 [39:40<48:57, 2.79s/it]
33%|███▎ | 518/1570 [39:43<48:41, 2.78s/it]
33%|███▎ | 519/1570 [39:46<48:51, 2.79s/it]
33%|███▎ | 520/1570 [39:48<48:44, 2.79s/it]
{'loss': '0.007654', 'grad_norm': '0.03887', 'learning_rate': '7.54e-05', 'epoch': '1.657'}
+
33%|███▎ | 520/1570 [39:48<48:44, 2.79s/it]
33%|███▎ | 521/1570 [39:51<48:51, 2.79s/it]
33%|███▎ | 522/1570 [39:54<48:55, 2.80s/it]
33%|███▎ | 523/1570 [39:57<48:47, 2.80s/it]
33%|███▎ | 524/1570 [40:00<48:54, 2.81s/it]
33%|███▎ | 525/1570 [40:02<48:49, 2.80s/it]
34%|███▎ | 526/1570 [40:05<48:46, 2.80s/it]
34%|███▎ | 527/1570 [40:08<48:39, 2.80s/it]
34%|███▎ | 528/1570 [40:11<48:25, 2.79s/it]
34%|███▎ | 529/1570 [40:14<48:35, 2.80s/it]
34%|███▍ | 530/1570 [40:16<48:36, 2.80s/it]
34%|███▍ | 531/1570 [40:19<48:41, 2.81s/it]
34%|███▍ | 532/1570 [40:22<48:42, 2.82s/it]
34%|███▍ | 533/1570 [40:25<48:42, 2.82s/it]
34%|███▍ | 534/1570 [40:28<49:04, 2.84s/it]
34%|███▍ | 535/1570 [40:31<48:53, 2.83s/it]
34%|███▍ | 536/1570 [40:33<48:56, 2.84s/it]
34%|███▍ | 537/1570 [40:36<48:46, 2.83s/it]
34%|███▍ | 538/1570 [40:39<48:55, 2.84s/it]
34%|███▍ | 539/1570 [40:42<48:41, 2.83s/it]
34%|███▍ | 540/1570 [40:45<48:36, 2.83s/it]
{'loss': '0.00647', 'grad_norm': '0.05847', 'learning_rate': '7.366e-05', 'epoch': '1.721'}
+
34%|███▍ | 540/1570 [40:45<48:36, 2.83s/it]
34%|███▍ | 541/1570 [40:48<48:34, 2.83s/it]
35%|███▍ | 542/1570 [40:50<48:27, 2.83s/it]
35%|███▍ | 543/1570 [40:53<48:27, 2.83s/it]
35%|███▍ | 544/1570 [40:56<48:22, 2.83s/it]
35%|███▍ | 545/1570 [40:59<48:21, 2.83s/it]
35%|███▍ | 546/1570 [41:02<48:15, 2.83s/it]
35%|███▍ | 547/1570 [41:05<48:21, 2.84s/it]
35%|███▍ | 548/1570 [41:07<48:09, 2.83s/it]
35%|███▍ | 549/1570 [41:10<47:58, 2.82s/it]
35%|███▌ | 550/1570 [41:13<48:01, 2.82s/it]
35%|███▌ | 551/1570 [41:16<48:23, 2.85s/it]
35%|███▌ | 552/1570 [41:19<48:23, 2.85s/it]
35%|███▌ | 553/1570 [41:22<48:17, 2.85s/it]
35%|███▌ | 554/1570 [41:25<48:11, 2.85s/it]
35%|███▌ | 555/1570 [41:27<47:59, 2.84s/it]
35%|███▌ | 556/1570 [41:30<47:57, 2.84s/it]
35%|███▌ | 557/1570 [41:33<47:47, 2.83s/it]
36%|███▌ | 558/1570 [41:36<47:37, 2.82s/it]
36%|███▌ | 559/1570 [41:39<47:32, 2.82s/it]
36%|███▌ | 560/1570 [41:41<47:37, 2.83s/it]
{'loss': '0.003531', 'grad_norm': '0.1613', 'learning_rate': '7.188e-05', 'epoch': '1.785'}
+
36%|███▌ | 560/1570 [41:41<47:37, 2.83s/it]
36%|███▌ | 561/1570 [41:44<47:40, 2.83s/it]
36%|███▌ | 562/1570 [41:47<47:19, 2.82s/it]
36%|███▌ | 563/1570 [41:50<47:11, 2.81s/it]
36%|███▌ | 564/1570 [41:53<47:05, 2.81s/it]
36%|███▌ | 565/1570 [41:56<47:16, 2.82s/it]
36%|███▌ | 566/1570 [41:58<47:06, 2.82s/it]
36%|███▌ | 567/1570 [42:01<47:08, 2.82s/it]
36%|███▌ | 568/1570 [42:04<47:08, 2.82s/it]
36%|███▌ | 569/1570 [42:07<47:12, 2.83s/it]
36%|███▋ | 570/1570 [42:10<46:58, 2.82s/it]
36%|███▋ | 571/1570 [42:12<46:51, 2.81s/it]
36%|███▋ | 572/1570 [42:15<46:46, 2.81s/it]
36%|███▋ | 573/1570 [42:18<46:57, 2.83s/it]
37%|███▋ | 574/1570 [42:21<46:50, 2.82s/it]
37%|███▋ | 575/1570 [42:24<46:44, 2.82s/it]
37%|███▋ | 576/1570 [42:27<46:35, 2.81s/it]
37%|███▋ | 577/1570 [42:29<46:50, 2.83s/it]
37%|███▋ | 578/1570 [42:32<46:33, 2.82s/it]
37%|███▋ | 579/1570 [42:35<46:24, 2.81s/it]
37%|███▋ | 580/1570 [42:38<46:36, 2.82s/it]
{'loss': '0.008149', 'grad_norm': '2.057', 'learning_rate': '7.006e-05', 'epoch': '1.849'}
+
37%|███▋ | 580/1570 [42:38<46:36, 2.82s/it]
37%|███▋ | 581/1570 [42:41<46:15, 2.81s/it]
37%|███▋ | 582/1570 [42:43<46:10, 2.80s/it]
37%|███▋ | 583/1570 [42:46<46:08, 2.80s/it]
37%|███▋ | 584/1570 [42:49<46:11, 2.81s/it]
37%|███▋ | 585/1570 [42:52<46:05, 2.81s/it]
37%|███▋ | 586/1570 [42:55<45:54, 2.80s/it]
37%|███▋ | 587/1570 [42:57<45:41, 2.79s/it]
37%|███▋ | 588/1570 [43:00<45:25, 2.78s/it]
38%|███▊ | 589/1570 [43:03<45:35, 2.79s/it]
38%|███▊ | 590/1570 [43:06<45:34, 2.79s/it]
38%|███▊ | 591/1570 [43:09<45:37, 2.80s/it]
38%|███▊ | 592/1570 [43:11<45:42, 2.80s/it]
38%|███▊ | 593/1570 [43:14<45:45, 2.81s/it]
38%|███▊ | 594/1570 [43:17<45:40, 2.81s/it]
38%|███▊ | 595/1570 [43:20<45:43, 2.81s/it]
38%|███▊ | 596/1570 [43:23<45:51, 2.83s/it]
38%|███▊ | 597/1570 [43:26<45:53, 2.83s/it]
38%|███▊ | 598/1570 [43:28<45:42, 2.82s/it]
38%|███▊ | 599/1570 [43:31<45:44, 2.83s/it]
38%|███▊ | 600/1570 [43:34<45:32, 2.82s/it]
{'loss': '0.006319', 'grad_norm': '1.818', 'learning_rate': '6.821e-05', 'epoch': '1.913'}
+
38%|███▊ | 600/1570 [43:34<45:32, 2.82s/it]
38%|███▊ | 601/1570 [43:37<45:39, 2.83s/it]
38%|███▊ | 602/1570 [43:40<45:37, 2.83s/it]
38%|███▊ | 603/1570 [43:43<45:46, 2.84s/it]
38%|███▊ | 604/1570 [43:45<45:49, 2.85s/it]
39%|███▊ | 605/1570 [43:48<45:43, 2.84s/it]
39%|███▊ | 606/1570 [43:51<45:40, 2.84s/it]
39%|███▊ | 607/1570 [43:54<45:31, 2.84s/it]
39%|███▊ | 608/1570 [43:57<45:28, 2.84s/it]
39%|███▉ | 609/1570 [44:00<45:21, 2.83s/it]
39%|███▉ | 610/1570 [44:02<45:30, 2.84s/it]
39%|███▉ | 611/1570 [44:05<45:25, 2.84s/it]
39%|███▉ | 612/1570 [44:08<45:15, 2.83s/it]
39%|███▉ | 613/1570 [44:11<44:54, 2.82s/it]
39%|███▉ | 614/1570 [44:14<44:40, 2.80s/it]
39%|███▉ | 615/1570 [44:16<44:35, 2.80s/it]
39%|███▉ | 616/1570 [44:19<44:42, 2.81s/it]
39%|███▉ | 617/1570 [44:22<44:40, 2.81s/it]
39%|███▉ | 618/1570 [44:25<44:26, 2.80s/it]
39%|███▉ | 619/1570 [44:28<44:31, 2.81s/it]
39%|███▉ | 620/1570 [44:30<44:27, 2.81s/it]
{'loss': '0.003846', 'grad_norm': '0.0236', 'learning_rate': '6.634e-05', 'epoch': '1.976'}
+
39%|███▉ | 620/1570 [44:30<44:27, 2.81s/it]
40%|███▉ | 621/1570 [44:33<44:31, 2.81s/it]
40%|███▉ | 622/1570 [44:36<44:17, 2.80s/it]
40%|███▉ | 623/1570 [44:39<44:20, 2.81s/it]
40%|███▉ | 624/1570 [44:42<44:38, 2.83s/it]
40%|███▉ | 625/1570 [44:45<44:35, 2.83s/it]
40%|███▉ | 626/1570 [44:47<44:24, 2.82s/it]
40%|███▉ | 627/1570 [44:50<44:17, 2.82s/it]
40%|████ | 628/1570 [44:51<36:13, 2.31s/it][INFO|trainer.py:2668] 2026-08-22 10:55:46,756 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 10:55:46,756 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 10:55:46,756 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/fkl/checkpoint-628
+[INFO|configuration_utils.py:780] 2026-08-22 11:05:59,343 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 11:05:59,345 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 11:05:59,393 >> chat template saved in results/qwen3/fkl/checkpoint-628/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 11:05:59,393 >> tokenizer config file saved in results/qwen3/fkl/checkpoint-628/tokenizer_config.json
+
40%|████ | 629/1570 [55:09<48:49:12, 186.77s/it]
40%|████ | 630/1570 [55:12<34:22:43, 131.66s/it]
40%|████ | 631/1570 [55:14<24:15:35, 93.01s/it]
40%|████ | 632/1570 [55:17<17:10:55, 65.94s/it]
40%|████ | 633/1570 [55:20<12:14:01, 47.00s/it]
40%|████ | 634/1570 [55:23<8:46:28, 33.75s/it]
40%|████ | 635/1570 [55:26<6:21:50, 24.50s/it]
41%|████ | 636/1570 [55:29<4:40:17, 18.01s/it]
41%|████ | 637/1570 [55:31<3:29:05, 13.45s/it]
41%|████ | 638/1570 [55:34<2:39:18, 10.26s/it]
41%|████ | 639/1570 [55:37<2:04:21, 8.01s/it]
41%|████ | 640/1570 [55:40<1:40:19, 6.47s/it]
{'loss': '0.003787', 'grad_norm': '0.07723', 'learning_rate': '6.444e-05', 'epoch': '2.038'}
+
41%|████ | 640/1570 [55:40<1:40:19, 6.47s/it]
41%|████ | 641/1570 [55:43<1:23:15, 5.38s/it]
41%|████ | 642/1570 [55:46<1:11:21, 4.61s/it]
41%|████ | 643/1570 [55:48<1:03:01, 4.08s/it]
41%|████ | 644/1570 [55:51<57:37, 3.73s/it]
41%|████ | 645/1570 [55:54<53:33, 3.47s/it]
41%|████ | 646/1570 [55:57<50:30, 3.28s/it]
41%|████ | 647/1570 [56:00<48:26, 3.15s/it]
41%|████▏ | 648/1570 [56:03<46:47, 3.05s/it]
41%|████▏ | 649/1570 [56:05<45:42, 2.98s/it]
41%|████▏ | 650/1570 [56:08<44:53, 2.93s/it]
41%|████▏ | 651/1570 [56:11<44:26, 2.90s/it]
42%|████▏ | 652/1570 [56:15<47:31, 3.11s/it]
42%|████▏ | 653/1570 [56:18<48:51, 3.20s/it]
42%|████▏ | 654/1570 [56:22<50:30, 3.31s/it]
42%|████▏ | 655/1570 [56:25<51:23, 3.37s/it]
42%|████▏ | 656/1570 [56:29<51:14, 3.36s/it]
42%|████▏ | 657/1570 [56:32<52:41, 3.46s/it]
42%|████▏ | 658/1570 [56:36<51:59, 3.42s/it]
42%|████▏ | 659/1570 [56:39<52:11, 3.44s/it]
42%|████▏ | 660/1570 [56:43<52:31, 3.46s/it]
{'loss': '0.006162', 'grad_norm': '0.4329', 'learning_rate': '6.251e-05', 'epoch': '2.102'}
+
42%|████▏ | 660/1570 [56:43<52:31, 3.46s/it]
42%|████▏ | 661/1570 [56:46<53:03, 3.50s/it]
42%|████▏ | 662/1570 [56:50<53:34, 3.54s/it]
42%|████▏ | 663/1570 [56:53<53:42, 3.55s/it]
42%|████▏ | 664/1570 [56:57<53:04, 3.51s/it]
42%|████▏ | 665/1570 [57:00<52:59, 3.51s/it]
42%|████▏ | 666/1570 [57:04<53:23, 3.54s/it]
42%|████▏ | 667/1570 [57:08<54:05, 3.59s/it]
43%|████▎ | 668/1570 [57:11<53:54, 3.59s/it]
43%|████▎ | 669/1570 [57:15<53:57, 3.59s/it]
43%|████▎ | 670/1570 [57:18<53:52, 3.59s/it]
43%|████▎ | 671/1570 [57:22<53:18, 3.56s/it]
43%|████▎ | 672/1570 [57:25<53:07, 3.55s/it]
43%|████▎ | 673/1570 [57:29<53:04, 3.55s/it]
43%|████▎ | 674/1570 [57:33<53:54, 3.61s/it]
43%|████▎ | 675/1570 [57:36<54:01, 3.62s/it]
43%|████▎ | 676/1570 [57:40<53:39, 3.60s/it]
43%|████▎ | 677/1570 [57:44<54:41, 3.67s/it]
43%|████▎ | 678/1570 [57:48<55:54, 3.76s/it]
43%|████▎ | 679/1570 [57:52<56:44, 3.82s/it]
43%|████▎ | 680/1570 [57:55<53:31, 3.61s/it]
{'loss': '0.00182', 'grad_norm': '0.1526', 'learning_rate': '6.057e-05', 'epoch': '2.166'}
+
43%|████▎ | 680/1570 [57:55<53:31, 3.61s/it]
43%|████▎ | 681/1570 [57:58<50:10, 3.39s/it]
43%|████▎ | 682/1570 [58:01<47:50, 3.23s/it]
44%|████▎ | 683/1570 [58:03<45:47, 3.10s/it]
44%|████▎ | 684/1570 [58:06<44:36, 3.02s/it]
44%|████▎ | 685/1570 [58:09<44:02, 2.99s/it]
44%|████▎ | 686/1570 [58:12<43:22, 2.94s/it]
44%|████▍ | 687/1570 [58:15<42:53, 2.92s/it]
44%|████▍ | 688/1570 [58:18<42:32, 2.89s/it]
44%|████▍ | 689/1570 [58:20<42:08, 2.87s/it]
44%|████▍ | 690/1570 [58:23<41:47, 2.85s/it]
44%|████▍ | 691/1570 [58:26<41:36, 2.84s/it]
44%|████▍ | 692/1570 [58:29<41:27, 2.83s/it]
44%|████▍ | 693/1570 [58:32<41:23, 2.83s/it]
44%|████▍ | 694/1570 [58:35<41:48, 2.86s/it]
44%|████▍ | 695/1570 [58:37<41:35, 2.85s/it]
44%|████▍ | 696/1570 [58:40<41:12, 2.83s/it]
44%|████▍ | 697/1570 [58:43<40:55, 2.81s/it]
44%|████▍ | 698/1570 [58:46<40:42, 2.80s/it]
45%|████▍ | 699/1570 [58:49<40:47, 2.81s/it]
45%|████▍ | 700/1570 [58:51<40:46, 2.81s/it]
{'loss': '0.00231', 'grad_norm': '0.03114', 'learning_rate': '5.86e-05', 'epoch': '2.23'}
+
45%|████▍ | 700/1570 [58:51<40:46, 2.81s/it]
45%|████▍ | 701/1570 [58:54<40:46, 2.82s/it]
45%|████▍ | 702/1570 [58:57<41:22, 2.86s/it]
45%|████▍ | 703/1570 [59:00<40:58, 2.84s/it]
45%|████▍ | 704/1570 [59:03<40:53, 2.83s/it]
45%|████▍ | 705/1570 [59:06<40:42, 2.82s/it]
45%|████▍ | 706/1570 [59:08<40:42, 2.83s/it]
45%|████▌ | 707/1570 [59:11<40:24, 2.81s/it]
45%|████▌ | 708/1570 [59:14<40:19, 2.81s/it]
45%|████▌ | 709/1570 [59:17<40:50, 2.85s/it]
45%|████▌ | 710/1570 [59:20<40:21, 2.82s/it]
45%|████▌ | 711/1570 [59:22<40:16, 2.81s/it]
45%|████▌ | 712/1570 [59:25<40:16, 2.82s/it]
45%|████▌ | 713/1570 [59:28<40:21, 2.83s/it]
45%|████▌ | 714/1570 [59:32<42:50, 3.00s/it]
46%|████▌ | 715/1570 [59:35<45:29, 3.19s/it]
46%|████▌ | 716/1570 [59:39<47:00, 3.30s/it]
46%|████▌ | 717/1570 [59:42<48:02, 3.38s/it]
46%|████▌ | 718/1570 [59:46<48:18, 3.40s/it]
46%|████▌ | 719/1570 [59:49<48:07, 3.39s/it]
46%|████▌ | 720/1570 [59:52<47:51, 3.38s/it]
{'loss': '0.001631', 'grad_norm': '0.0184', 'learning_rate': '5.663e-05', 'epoch': '2.294'}
+
46%|████▌ | 720/1570 [59:52<47:51, 3.38s/it]
46%|████▌ | 721/1570 [59:56<47:46, 3.38s/it]
46%|████▌ | 722/1570 [59:59<46:44, 3.31s/it]
46%|████▌ | 723/1570 [1:00:02<47:16, 3.35s/it]
46%|████▌ | 724/1570 [1:00:06<46:37, 3.31s/it]
46%|████▌ | 725/1570 [1:00:09<48:12, 3.42s/it]
46%|████▌ | 726/1570 [1:00:13<47:13, 3.36s/it]
46%|████▋ | 727/1570 [1:00:16<47:38, 3.39s/it]
46%|████▋ | 728/1570 [1:00:19<46:39, 3.32s/it]
46%|████▋ | 729/1570 [1:00:23<47:41, 3.40s/it]
46%|████▋ | 730/1570 [1:00:26<48:04, 3.43s/it]
47%|████▋ | 731/1570 [1:00:30<47:25, 3.39s/it]
47%|████▋ | 732/1570 [1:00:33<47:45, 3.42s/it]
47%|████▋ | 733/1570 [1:00:37<48:57, 3.51s/it]
47%|████▋ | 734/1570 [1:00:40<48:58, 3.51s/it]
47%|████▋ | 735/1570 [1:00:44<49:10, 3.53s/it]
47%|████▋ | 736/1570 [1:00:47<48:26, 3.48s/it]
47%|████▋ | 737/1570 [1:00:51<48:23, 3.48s/it]
47%|████▋ | 738/1570 [1:00:54<48:28, 3.50s/it]
47%|████▋ | 739/1570 [1:00:58<48:29, 3.50s/it]
47%|████▋ | 740/1570 [1:01:01<48:24, 3.50s/it]
{'loss': '0.002783', 'grad_norm': '0.03898', 'learning_rate': '5.464e-05', 'epoch': '2.357'}
+
47%|████▋ | 740/1570 [1:01:01<48:24, 3.50s/it]
47%|████▋ | 741/1570 [1:01:05<48:21, 3.50s/it]
47%|████▋ | 742/1570 [1:01:08<48:22, 3.51s/it]
47%|████▋ | 743/1570 [1:01:12<48:27, 3.52s/it]
47%|████▋ | 744/1570 [1:01:15<47:44, 3.47s/it]
47%|████▋ | 745/1570 [1:01:19<48:01, 3.49s/it]
48%|████▊ | 746/1570 [1:01:22<47:44, 3.48s/it]
48%|████▊ | 747/1570 [1:01:26<47:01, 3.43s/it]
48%|████▊ | 748/1570 [1:01:29<47:10, 3.44s/it]
48%|████▊ | 749/1570 [1:01:32<46:44, 3.42s/it]
48%|████▊ | 750/1570 [1:01:36<46:18, 3.39s/it]
48%|████▊ | 751/1570 [1:01:39<46:22, 3.40s/it]
48%|████▊ | 752/1570 [1:01:43<46:37, 3.42s/it]
48%|████▊ | 753/1570 [1:01:46<46:11, 3.39s/it]
48%|████▊ | 754/1570 [1:01:49<45:59, 3.38s/it]
48%|████▊ | 755/1570 [1:01:53<46:01, 3.39s/it]
48%|████▊ | 756/1570 [1:01:56<46:09, 3.40s/it]
48%|████▊ | 757/1570 [1:02:00<46:26, 3.43s/it]
48%|████▊ | 758/1570 [1:02:03<46:59, 3.47s/it]
48%|████▊ | 759/1570 [1:02:07<47:17, 3.50s/it]
48%|████▊ | 760/1570 [1:02:10<47:10, 3.49s/it]
{'loss': '0.001687', 'grad_norm': '0.008163', 'learning_rate': '5.265e-05', 'epoch': '2.421'}
+
48%|████▊ | 760/1570 [1:02:10<47:10, 3.49s/it]
48%|████▊ | 761/1570 [1:02:14<47:17, 3.51s/it]
49%|████▊ | 762/1570 [1:02:17<47:20, 3.52s/it]
49%|████▊ | 763/1570 [1:02:21<47:06, 3.50s/it]
49%|████▊ | 764/1570 [1:02:24<46:45, 3.48s/it]
49%|████▊ | 765/1570 [1:02:28<46:11, 3.44s/it]
49%|████▉ | 766/1570 [1:02:31<46:25, 3.46s/it]
49%|████▉ | 767/1570 [1:02:35<46:24, 3.47s/it]
49%|████▉ | 768/1570 [1:02:38<46:33, 3.48s/it]
49%|████▉ | 769/1570 [1:02:42<46:40, 3.50s/it]
49%|████▉ | 770/1570 [1:02:45<46:41, 3.50s/it]
49%|████▉ | 771/1570 [1:02:48<46:18, 3.48s/it]
49%|████▉ | 772/1570 [1:02:52<46:01, 3.46s/it]
49%|████▉ | 773/1570 [1:02:55<45:52, 3.45s/it]
49%|████▉ | 774/1570 [1:02:59<46:05, 3.47s/it]
49%|████▉ | 775/1570 [1:03:02<45:40, 3.45s/it]
49%|████▉ | 776/1570 [1:03:06<45:08, 3.41s/it]
49%|████▉ | 777/1570 [1:03:09<45:08, 3.41s/it]
50%|████▉ | 778/1570 [1:03:12<45:02, 3.41s/it]
50%|████▉ | 779/1570 [1:03:16<45:37, 3.46s/it]
50%|████▉ | 780/1570 [1:03:20<45:55, 3.49s/it]
{'loss': '0.001099', 'grad_norm': '0.05952', 'learning_rate': '5.065e-05', 'epoch': '2.485'}
+
50%|████▉ | 780/1570 [1:03:20<45:55, 3.49s/it]
50%|████▉ | 781/1570 [1:03:23<46:02, 3.50s/it]
50%|████▉ | 782/1570 [1:03:27<46:11, 3.52s/it]
50%|████▉ | 783/1570 [1:03:30<45:49, 3.49s/it]
50%|████▉ | 784/1570 [1:03:33<45:19, 3.46s/it]
50%|█████ | 785/1570 [1:03:37<45:44, 3.50s/it]
50%|█████ | 786/1570 [1:03:40<45:25, 3.48s/it]
50%|█████ | 787/1570 [1:03:44<45:10, 3.46s/it]
50%|█████ | 788/1570 [1:03:47<45:18, 3.48s/it]
50%|█████ | 789/1570 [1:03:51<45:21, 3.49s/it]
50%|█████ | 790/1570 [1:03:54<45:17, 3.48s/it]
50%|█████ | 791/1570 [1:03:58<45:23, 3.50s/it]
50%|█████ | 792/1570 [1:04:01<45:44, 3.53s/it]
51%|█████ | 793/1570 [1:04:05<45:43, 3.53s/it]
51%|█████ | 794/1570 [1:04:08<45:20, 3.51s/it]
51%|█████ | 795/1570 [1:04:12<45:17, 3.51s/it]
51%|█████ | 796/1570 [1:04:16<45:15, 3.51s/it]
51%|█████ | 797/1570 [1:04:19<45:26, 3.53s/it]
51%|█████ | 798/1570 [1:04:23<45:20, 3.52s/it]
51%|█████ | 799/1570 [1:04:26<44:37, 3.47s/it]
51%|█████ | 800/1570 [1:04:29<44:33, 3.47s/it]
{'loss': '0.002597', 'grad_norm': '0.08524', 'learning_rate': '4.865e-05', 'epoch': '2.549'}
+
51%|█████ | 800/1570 [1:04:29<44:33, 3.47s/it]
51%|█████ | 801/1570 [1:04:33<44:51, 3.50s/it]
51%|█████ | 802/1570 [1:04:37<45:03, 3.52s/it]
51%|█████ | 803/1570 [1:04:40<45:12, 3.54s/it]
51%|█████ | 804/1570 [1:04:44<45:09, 3.54s/it]
51%|█████▏ | 805/1570 [1:04:47<45:02, 3.53s/it]
51%|█████▏ | 806/1570 [1:04:50<43:31, 3.42s/it]
51%|█████▏ | 807/1570 [1:04:54<43:56, 3.46s/it]
51%|█████▏ | 808/1570 [1:04:57<44:13, 3.48s/it]
52%|█████▏ | 809/1570 [1:05:01<44:18, 3.49s/it]
52%|█████▏ | 810/1570 [1:05:04<43:59, 3.47s/it]
52%|█████▏ | 811/1570 [1:05:08<44:22, 3.51s/it]
52%|█████▏ | 812/1570 [1:05:11<44:26, 3.52s/it]
52%|█████▏ | 813/1570 [1:05:15<44:36, 3.54s/it]
52%|█████▏ | 814/1570 [1:05:18<43:45, 3.47s/it]
52%|█████▏ | 815/1570 [1:05:22<44:19, 3.52s/it]
52%|█████▏ | 816/1570 [1:05:25<42:47, 3.41s/it]
52%|█████▏ | 817/1570 [1:05:29<43:29, 3.47s/it]
52%|█████▏ | 818/1570 [1:05:32<43:41, 3.49s/it]
52%|█████▏ | 819/1570 [1:05:36<43:30, 3.48s/it]
52%|█████▏ | 820/1570 [1:05:39<43:33, 3.49s/it]
{'loss': '0.00206', 'grad_norm': '0.03939', 'learning_rate': '4.665e-05', 'epoch': '2.613'}
+
52%|█████▏ | 820/1570 [1:05:39<43:33, 3.49s/it]
52%|█████▏ | 821/1570 [1:05:43<43:05, 3.45s/it]
52%|█████▏ | 822/1570 [1:05:46<43:19, 3.48s/it]
52%|█████▏ | 823/1570 [1:05:49<42:23, 3.41s/it]
52%|█████▏ | 824/1570 [1:05:53<42:49, 3.44s/it]
53%|█████▎ | 825/1570 [1:05:56<41:30, 3.34s/it]
53%|█████▎ | 826/1570 [1:06:00<42:17, 3.41s/it]
53%|█████▎ | 827/1570 [1:06:03<42:27, 3.43s/it]
53%|█████▎ | 828/1570 [1:06:07<42:34, 3.44s/it]
53%|█████▎ | 829/1570 [1:06:10<43:26, 3.52s/it]
53%|█████▎ | 830/1570 [1:06:14<42:24, 3.44s/it]
53%|█████▎ | 831/1570 [1:06:17<43:15, 3.51s/it]
53%|█████▎ | 832/1570 [1:06:21<43:01, 3.50s/it]
53%|█████▎ | 833/1570 [1:06:24<43:09, 3.51s/it]
53%|█████▎ | 834/1570 [1:06:28<43:07, 3.52s/it]
53%|█████▎ | 835/1570 [1:06:31<43:11, 3.53s/it]
53%|█████▎ | 836/1570 [1:06:35<43:02, 3.52s/it]
53%|█████▎ | 837/1570 [1:06:39<44:04, 3.61s/it]
53%|█████▎ | 838/1570 [1:06:42<44:39, 3.66s/it]
53%|█████▎ | 839/1570 [1:06:46<45:47, 3.76s/it]
54%|█████▎ | 840/1570 [1:06:50<46:23, 3.81s/it]
{'loss': '0.002689', 'grad_norm': '0.1421', 'learning_rate': '4.466e-05', 'epoch': '2.677'}
+
54%|█████▎ | 840/1570 [1:06:50<46:23, 3.81s/it]
54%|█████▎ | 841/1570 [1:06:54<46:51, 3.86s/it]
54%|█████▎ | 842/1570 [1:06:58<47:12, 3.89s/it]
54%|█████▎ | 843/1570 [1:07:02<47:28, 3.92s/it]
54%|█████▍ | 844/1570 [1:07:06<46:55, 3.88s/it]
54%|█████▍ | 845/1570 [1:07:10<47:13, 3.91s/it]
54%|█████▍ | 846/1570 [1:07:14<47:37, 3.95s/it]
54%|█████▍ | 847/1570 [1:07:18<47:25, 3.94s/it]
54%|█████▍ | 848/1570 [1:07:22<47:28, 3.95s/it]
54%|█████▍ | 849/1570 [1:07:26<47:27, 3.95s/it]
54%|█████▍ | 850/1570 [1:07:30<46:48, 3.90s/it]
54%|█████▍ | 851/1570 [1:07:34<47:06, 3.93s/it]
54%|█████▍ | 852/1570 [1:07:38<47:17, 3.95s/it]
54%|█████▍ | 853/1570 [1:07:42<47:07, 3.94s/it]
54%|█████▍ | 854/1570 [1:07:46<47:06, 3.95s/it]
54%|█████▍ | 855/1570 [1:07:50<47:09, 3.96s/it]
55%|█████▍ | 856/1570 [1:07:53<46:34, 3.91s/it]
55%|█████▍ | 857/1570 [1:07:57<46:37, 3.92s/it]
55%|█████▍ | 858/1570 [1:08:01<46:45, 3.94s/it]
55%|█████▍ | 859/1570 [1:08:05<46:36, 3.93s/it]
55%|█████▍ | 860/1570 [1:08:09<46:33, 3.93s/it]
{'loss': '0.00225', 'grad_norm': '0.07379', 'learning_rate': '4.268e-05', 'epoch': '2.74'}
+
55%|█████▍ | 860/1570 [1:08:09<46:33, 3.93s/it]
55%|█████▍ | 861/1570 [1:08:13<46:22, 3.92s/it]
55%|█████▍ | 862/1570 [1:08:17<46:37, 3.95s/it]
55%|█████▍ | 863/1570 [1:08:21<46:20, 3.93s/it]
55%|█████▌ | 864/1570 [1:08:25<46:15, 3.93s/it]
55%|█████▌ | 865/1570 [1:08:29<46:14, 3.94s/it]
55%|█████▌ | 866/1570 [1:08:33<45:47, 3.90s/it]
55%|█████▌ | 867/1570 [1:08:37<45:53, 3.92s/it]
55%|█████▌ | 868/1570 [1:08:40<45:42, 3.91s/it]
55%|█████▌ | 869/1570 [1:08:44<45:56, 3.93s/it]
55%|█████▌ | 870/1570 [1:08:48<45:21, 3.89s/it]
55%|█████▌ | 871/1570 [1:08:52<45:13, 3.88s/it]
56%|█████▌ | 872/1570 [1:08:56<45:22, 3.90s/it]
56%|█████▌ | 873/1570 [1:09:00<44:55, 3.87s/it]
56%|█████▌ | 874/1570 [1:09:04<45:14, 3.90s/it]
56%|█████▌ | 875/1570 [1:09:08<44:50, 3.87s/it]
56%|█████▌ | 876/1570 [1:09:11<44:38, 3.86s/it]
56%|█████▌ | 877/1570 [1:09:15<43:25, 3.76s/it]
56%|█████▌ | 878/1570 [1:09:19<44:00, 3.82s/it]
56%|█████▌ | 879/1570 [1:09:23<43:57, 3.82s/it]
56%|█████▌ | 880/1570 [1:09:27<44:22, 3.86s/it]
{'loss': '0.001481', 'grad_norm': '0.1657', 'learning_rate': '4.071e-05', 'epoch': '2.804'}
+
56%|█████▌ | 880/1570 [1:09:27<44:22, 3.86s/it]
56%|█████▌ | 881/1570 [1:09:31<44:32, 3.88s/it]
56%|█████▌ | 882/1570 [1:09:34<44:03, 3.84s/it]
56%|█████▌ | 883/1570 [1:09:38<44:28, 3.88s/it]
56%|█████▋ | 884/1570 [1:09:42<43:47, 3.83s/it]
56%|█████▋ | 885/1570 [1:09:46<43:39, 3.82s/it]
56%|█████▋ | 886/1570 [1:09:50<43:11, 3.79s/it]
56%|█████▋ | 887/1570 [1:09:53<42:36, 3.74s/it]
57%|█████▋ | 888/1570 [1:09:57<41:53, 3.69s/it]
57%|█████▋ | 889/1570 [1:10:00<41:19, 3.64s/it]
57%|█████▋ | 890/1570 [1:10:04<41:26, 3.66s/it]
57%|█████▋ | 891/1570 [1:10:08<41:25, 3.66s/it]
57%|█████▋ | 892/1570 [1:10:11<40:46, 3.61s/it]
57%|█████▋ | 893/1570 [1:10:15<40:15, 3.57s/it]
57%|█████▋ | 894/1570 [1:10:18<40:02, 3.55s/it]
57%|█████▋ | 895/1570 [1:10:22<39:47, 3.54s/it]
57%|█████▋ | 896/1570 [1:10:25<39:35, 3.52s/it]
57%|█████▋ | 897/1570 [1:10:29<40:37, 3.62s/it]
57%|█████▋ | 898/1570 [1:10:33<41:11, 3.68s/it]
57%|█████▋ | 899/1570 [1:10:37<42:01, 3.76s/it]
57%|█████▋ | 900/1570 [1:10:41<42:45, 3.83s/it]
{'loss': '0.002208', 'grad_norm': '0.1267', 'learning_rate': '3.875e-05', 'epoch': '2.868'}
+
57%|█████▋ | 900/1570 [1:10:41<42:45, 3.83s/it]
57%|█████▋ | 901/1570 [1:10:45<43:15, 3.88s/it]
57%|█████▋ | 902/1570 [1:10:49<43:38, 3.92s/it]
58%|█████▊ | 903/1570 [1:10:53<43:38, 3.93s/it]
58%|█████▊ | 904/1570 [1:10:56<43:06, 3.88s/it]
58%|█████▊ | 905/1570 [1:11:00<43:22, 3.91s/it]
58%|█████▊ | 906/1570 [1:11:04<43:24, 3.92s/it]
58%|█████▊ | 907/1570 [1:11:08<43:20, 3.92s/it]
58%|█████▊ | 908/1570 [1:11:12<43:22, 3.93s/it]
58%|█████▊ | 909/1570 [1:11:16<43:14, 3.92s/it]
58%|█████▊ | 910/1570 [1:11:20<42:33, 3.87s/it]
58%|█████▊ | 911/1570 [1:11:24<42:47, 3.90s/it]
58%|█████▊ | 912/1570 [1:11:28<43:04, 3.93s/it]
58%|█████▊ | 913/1570 [1:11:31<39:40, 3.62s/it]
58%|█████▊ | 914/1570 [1:11:34<36:58, 3.38s/it]
58%|█████▊ | 915/1570 [1:11:36<35:08, 3.22s/it]
58%|█████▊ | 916/1570 [1:11:39<33:51, 3.11s/it]
58%|█████▊ | 917/1570 [1:11:42<32:46, 3.01s/it]
58%|█████▊ | 918/1570 [1:11:45<32:01, 2.95s/it]
59%|█████▊ | 919/1570 [1:11:48<31:45, 2.93s/it]
59%|█████▊ | 920/1570 [1:11:51<31:30, 2.91s/it]
{'loss': '0.002069', 'grad_norm': '0.005265', 'learning_rate': '3.682e-05', 'epoch': '2.932'}
+
59%|█████▊ | 920/1570 [1:11:51<31:30, 2.91s/it]
59%|█████▊ | 921/1570 [1:11:53<31:13, 2.89s/it]
59%|█████▊ | 922/1570 [1:11:56<30:57, 2.87s/it]
59%|█████▉ | 923/1570 [1:11:59<30:46, 2.85s/it]
59%|█████▉ | 924/1570 [1:12:02<30:35, 2.84s/it]
59%|█████▉ | 925/1570 [1:12:05<31:05, 2.89s/it]
59%|█████▉ | 926/1570 [1:12:08<30:45, 2.87s/it]
59%|█████▉ | 927/1570 [1:12:11<30:29, 2.85s/it]
59%|█████▉ | 928/1570 [1:12:13<30:20, 2.84s/it]
59%|█████▉ | 929/1570 [1:12:16<30:25, 2.85s/it]
59%|█████▉ | 930/1570 [1:12:19<30:49, 2.89s/it]
59%|█████▉ | 931/1570 [1:12:22<30:48, 2.89s/it]
59%|█████▉ | 932/1570 [1:12:25<30:48, 2.90s/it]
59%|█████▉ | 933/1570 [1:12:28<30:58, 2.92s/it]
59%|█████▉ | 934/1570 [1:12:31<31:04, 2.93s/it]
60%|█████▉ | 935/1570 [1:12:34<30:41, 2.90s/it]
60%|█████▉ | 936/1570 [1:12:37<31:08, 2.95s/it]
60%|█████▉ | 937/1570 [1:12:40<30:42, 2.91s/it]
60%|█████▉ | 938/1570 [1:12:43<31:00, 2.94s/it]
60%|█████▉ | 939/1570 [1:12:46<30:36, 2.91s/it]
60%|█████▉ | 940/1570 [1:12:49<30:56, 2.95s/it]
{'loss': '0.001283', 'grad_norm': '0.107', 'learning_rate': '3.49e-05', 'epoch': '2.996'}
+
60%|█████▉ | 940/1570 [1:12:49<30:56, 2.95s/it]
60%|█████▉ | 941/1570 [1:12:51<30:33, 2.91s/it]
60%|██████ | 942/1570 [1:12:53<24:54, 2.38s/it][INFO|trainer.py:2668] 2026-08-22 11:23:47,945 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 11:23:47,945 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 11:23:47,945 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/fkl/checkpoint-942
+[INFO|configuration_utils.py:780] 2026-08-22 11:34:09,111 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 11:34:09,115 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 11:34:09,150 >> chat template saved in results/qwen3/fkl/checkpoint-942/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 11:34:09,150 >> tokenizer config file saved in results/qwen3/fkl/checkpoint-942/tokenizer_config.json
+
60%|██████ | 943/1570 [1:23:18<32:56:46, 189.16s/it]
60%|██████ | 944/1570 [1:23:21<23:11:07, 133.33s/it]
60%|██████ | 945/1570 [1:23:23<16:21:10, 94.19s/it]
60%|██████ | 946/1570 [1:23:26<11:35:11, 66.85s/it]
60%|██████ | 947/1570 [1:23:29<8:14:36, 47.63s/it]
60%|██████ | 948/1570 [1:23:32<5:55:19, 34.28s/it]
60%|██████ | 949/1570 [1:23:35<4:17:02, 24.83s/it]
61%|██████ | 950/1570 [1:23:38<3:09:21, 18.32s/it]
61%|██████ | 951/1570 [1:23:41<2:21:00, 13.67s/it]
61%|██████ | 952/1570 [1:23:44<1:47:59, 10.48s/it]
61%|██████ | 953/1570 [1:23:47<1:24:21, 8.20s/it]
61%|██████ | 954/1570 [1:23:50<1:07:46, 6.60s/it]
61%|██████ | 955/1570 [1:23:53<56:32, 5.52s/it]
61%|██████ | 956/1570 [1:23:56<48:26, 4.73s/it]
61%|██████ | 957/1570 [1:23:59<43:02, 4.21s/it]
61%|██████ | 958/1570 [1:24:02<39:16, 3.85s/it]
61%|██████ | 959/1570 [1:24:05<36:25, 3.58s/it]
61%|██████ | 960/1570 [1:24:08<35:21, 3.48s/it]
{'loss': '0.001346', 'grad_norm': '0.02306', 'learning_rate': '3.301e-05', 'epoch': '3.057'}
+
61%|██████ | 960/1570 [1:24:08<35:21, 3.48s/it]
61%|██████ | 961/1570 [1:24:11<34:39, 3.41s/it]
61%|██████▏ | 962/1570 [1:24:15<34:23, 3.39s/it]
61%|██████▏ | 963/1570 [1:24:18<34:16, 3.39s/it]
61%|██████▏ | 964/1570 [1:24:21<34:25, 3.41s/it]
61%|██████▏ | 965/1570 [1:24:25<34:43, 3.44s/it]
62%|██████▏ | 966/1570 [1:24:28<34:51, 3.46s/it]
62%|██████▏ | 967/1570 [1:24:32<34:26, 3.43s/it]
62%|██████▏ | 968/1570 [1:24:35<33:51, 3.37s/it]
62%|██████▏ | 969/1570 [1:24:38<33:31, 3.35s/it]
62%|██████▏ | 970/1570 [1:24:42<33:46, 3.38s/it]
62%|██████▏ | 971/1570 [1:24:45<34:07, 3.42s/it]
62%|██████▏ | 972/1570 [1:24:49<34:31, 3.46s/it]
62%|██████▏ | 973/1570 [1:24:52<34:48, 3.50s/it]
62%|██████▏ | 974/1570 [1:24:56<34:58, 3.52s/it]
62%|██████▏ | 975/1570 [1:25:00<35:01, 3.53s/it]
62%|██████▏ | 976/1570 [1:25:03<35:05, 3.55s/it]
62%|██████▏ | 977/1570 [1:25:07<35:50, 3.63s/it]
62%|██████▏ | 978/1570 [1:25:11<36:40, 3.72s/it]
62%|██████▏ | 979/1570 [1:25:15<37:15, 3.78s/it]
62%|██████▏ | 980/1570 [1:25:19<37:37, 3.83s/it]
{'loss': '0.001208', 'grad_norm': '0.006213', 'learning_rate': '3.114e-05', 'epoch': '3.121'}
+
62%|██████▏ | 980/1570 [1:25:19<37:37, 3.83s/it]
62%|██████▏ | 981/1570 [1:25:23<37:53, 3.86s/it]
63%|██████▎ | 982/1570 [1:25:27<37:49, 3.86s/it]
63%|██████▎ | 983/1570 [1:25:30<37:26, 3.83s/it]
63%|██████▎ | 984/1570 [1:25:34<37:40, 3.86s/it]
63%|██████▎ | 985/1570 [1:25:38<37:53, 3.89s/it]
63%|██████▎ | 986/1570 [1:25:42<37:46, 3.88s/it]
63%|██████▎ | 987/1570 [1:25:46<37:53, 3.90s/it]
63%|██████▎ | 988/1570 [1:25:49<35:18, 3.64s/it]
63%|██████▎ | 989/1570 [1:25:52<32:52, 3.39s/it]
63%|██████▎ | 990/1570 [1:25:55<31:01, 3.21s/it]
63%|██████▎ | 991/1570 [1:25:57<29:52, 3.10s/it]
63%|██████▎ | 992/1570 [1:26:00<29:03, 3.02s/it]
63%|██████▎ | 993/1570 [1:26:03<28:36, 2.97s/it]
63%|██████▎ | 994/1570 [1:26:06<28:04, 2.92s/it]
63%|██████▎ | 995/1570 [1:26:09<27:45, 2.90s/it]
63%|██████▎ | 996/1570 [1:26:12<27:30, 2.88s/it]
64%|██████▎ | 997/1570 [1:26:14<27:11, 2.85s/it]
64%|██████▎ | 998/1570 [1:26:18<28:57, 3.04s/it]
64%|██████▎ | 999/1570 [1:26:21<29:56, 3.15s/it]
64%|██████▎ | 1000/1570 [1:26:25<30:49, 3.24s/it]
{'loss': '0.001188', 'grad_norm': '0.003079', 'learning_rate': '2.931e-05', 'epoch': '3.185'}
+
64%|██████▎ | 1000/1570 [1:26:25<30:49, 3.24s/it]
64%|██████▍ | 1001/1570 [1:26:28<31:21, 3.31s/it]
64%|██████▍ | 1002/1570 [1:26:32<31:58, 3.38s/it]
64%|██████▍ | 1003/1570 [1:26:35<32:01, 3.39s/it]
64%|██████▍ | 1004/1570 [1:26:39<32:12, 3.42s/it]
64%|██████▍ | 1005/1570 [1:26:42<32:22, 3.44s/it]
64%|██████▍ | 1006/1570 [1:26:46<32:13, 3.43s/it]
64%|██████▍ | 1007/1570 [1:26:49<32:26, 3.46s/it]
64%|██████▍ | 1008/1570 [1:26:52<32:09, 3.43s/it]
64%|██████▍ | 1009/1570 [1:26:56<32:10, 3.44s/it]
64%|██████▍ | 1010/1570 [1:26:59<31:30, 3.38s/it]
64%|██████▍ | 1011/1570 [1:27:03<31:52, 3.42s/it]
64%|██████▍ | 1012/1570 [1:27:06<31:56, 3.43s/it]
65%|██████▍ | 1013/1570 [1:27:10<32:02, 3.45s/it]
65%|██████▍ | 1014/1570 [1:27:13<32:11, 3.47s/it]
65%|██████▍ | 1015/1570 [1:27:17<31:56, 3.45s/it]
65%|██████▍ | 1016/1570 [1:27:20<32:05, 3.48s/it]
65%|██████▍ | 1017/1570 [1:27:24<31:53, 3.46s/it]
65%|██████▍ | 1018/1570 [1:27:27<31:44, 3.45s/it]
65%|██████▍ | 1019/1570 [1:27:30<31:35, 3.44s/it]
65%|██████▍ | 1020/1570 [1:27:34<31:35, 3.45s/it]
{'loss': '0.0008375', 'grad_norm': '0.02574', 'learning_rate': '2.751e-05', 'epoch': '3.249'}
+
65%|██████▍ | 1020/1570 [1:27:34<31:35, 3.45s/it]
65%|██████▌ | 1021/1570 [1:27:37<31:18, 3.42s/it]
65%|██████▌ | 1022/1570 [1:27:40<30:37, 3.35s/it]
65%|██████▌ | 1023/1570 [1:27:44<30:25, 3.34s/it]
65%|██████▌ | 1024/1570 [1:27:47<30:12, 3.32s/it]
65%|██████▌ | 1025/1570 [1:27:50<28:50, 3.18s/it]
65%|██████▌ | 1026/1570 [1:27:53<27:54, 3.08s/it]
65%|██████▌ | 1027/1570 [1:27:56<27:12, 3.01s/it]
65%|██████▌ | 1028/1570 [1:27:58<26:38, 2.95s/it]
66%|██████▌ | 1029/1570 [1:28:01<26:14, 2.91s/it]
66%|██████▌ | 1030/1570 [1:28:04<25:53, 2.88s/it]
66%|██████▌ | 1031/1570 [1:28:07<25:39, 2.86s/it]
66%|██████▌ | 1032/1570 [1:28:10<26:50, 2.99s/it]
66%|██████▌ | 1033/1570 [1:28:14<29:30, 3.30s/it]
66%|██████▌ | 1034/1570 [1:28:18<31:21, 3.51s/it]
66%|██████▌ | 1035/1570 [1:28:22<32:27, 3.64s/it]
66%|██████▌ | 1036/1570 [1:28:26<33:07, 3.72s/it]
66%|██████▌ | 1037/1570 [1:28:30<33:44, 3.80s/it]
66%|██████▌ | 1038/1570 [1:28:34<33:57, 3.83s/it]
66%|██████▌ | 1039/1570 [1:28:38<34:11, 3.86s/it]
66%|██████▌ | 1040/1570 [1:28:42<34:16, 3.88s/it]
{'loss': '0.0008559', 'grad_norm': '0.03453', 'learning_rate': '2.574e-05', 'epoch': '3.313'}
+
66%|██████▌ | 1040/1570 [1:28:42<34:16, 3.88s/it]
66%|██████▋ | 1041/1570 [1:28:46<34:17, 3.89s/it]
66%|██████▋ | 1042/1570 [1:28:50<34:20, 3.90s/it]
66%|██████▋ | 1043/1570 [1:28:53<34:16, 3.90s/it]
66%|██████▋ | 1044/1570 [1:28:57<33:50, 3.86s/it]
67%|██████▋ | 1045/1570 [1:29:01<33:58, 3.88s/it]
67%|██████▋ | 1046/1570 [1:29:05<34:07, 3.91s/it]
67%|██████▋ | 1047/1570 [1:29:09<33:45, 3.87s/it]
67%|██████▋ | 1048/1570 [1:29:13<33:49, 3.89s/it]
67%|██████▋ | 1049/1570 [1:29:17<33:59, 3.91s/it]
67%|██████▋ | 1050/1570 [1:29:21<33:47, 3.90s/it]
67%|██████▋ | 1051/1570 [1:29:24<33:23, 3.86s/it]
67%|██████▋ | 1052/1570 [1:29:28<33:31, 3.88s/it]
67%|██████▋ | 1053/1570 [1:29:31<31:23, 3.64s/it]
67%|██████▋ | 1054/1570 [1:29:34<29:08, 3.39s/it]
67%|██████▋ | 1055/1570 [1:29:37<27:32, 3.21s/it]
67%|██████▋ | 1056/1570 [1:29:40<26:28, 3.09s/it]
67%|██████▋ | 1057/1570 [1:29:43<25:32, 2.99s/it]
67%|██████▋ | 1058/1570 [1:29:45<25:04, 2.94s/it]
67%|██████▋ | 1059/1570 [1:29:48<24:44, 2.90s/it]
68%|██████▊ | 1060/1570 [1:29:51<24:27, 2.88s/it]
{'loss': '0.001648', 'grad_norm': '0.001913', 'learning_rate': '2.402e-05', 'epoch': '3.377'}
+
68%|██████▊ | 1060/1570 [1:29:51<24:27, 2.88s/it]
68%|██████▊ | 1061/1570 [1:29:54<24:23, 2.88s/it]
68%|██████▊ | 1062/1570 [1:29:57<24:10, 2.85s/it]
68%|██████▊ | 1063/1570 [1:29:59<23:53, 2.83s/it]
68%|██████▊ | 1064/1570 [1:30:02<23:49, 2.83s/it]
68%|██████▊ | 1065/1570 [1:30:05<23:50, 2.83s/it]
68%|██████▊ | 1066/1570 [1:30:08<23:43, 2.82s/it]
68%|██████▊ | 1067/1570 [1:30:11<23:38, 2.82s/it]
68%|██████▊ | 1068/1570 [1:30:14<23:31, 2.81s/it]
68%|██████▊ | 1069/1570 [1:30:16<23:39, 2.83s/it]
68%|██████▊ | 1070/1570 [1:30:19<23:31, 2.82s/it]
68%|██████▊ | 1071/1570 [1:30:22<23:34, 2.84s/it]
68%|██████▊ | 1072/1570 [1:30:25<23:33, 2.84s/it]
68%|██████▊ | 1073/1570 [1:30:28<23:30, 2.84s/it]
68%|██████▊ | 1074/1570 [1:30:31<23:25, 2.83s/it]
68%|██████▊ | 1075/1570 [1:30:33<23:17, 2.82s/it]
69%|██████▊ | 1076/1570 [1:30:36<23:14, 2.82s/it]
69%|██████▊ | 1077/1570 [1:30:39<23:06, 2.81s/it]
69%|██████▊ | 1078/1570 [1:30:42<23:13, 2.83s/it]
69%|██████▊ | 1079/1570 [1:30:45<23:03, 2.82s/it]
69%|██████▉ | 1080/1570 [1:30:47<23:00, 2.82s/it]
{'loss': '0.002572', 'grad_norm': '0.07636', 'learning_rate': '2.233e-05', 'epoch': '3.44'}
+
69%|██████▉ | 1080/1570 [1:30:47<23:00, 2.82s/it]
69%|██████▉ | 1081/1570 [1:30:50<23:03, 2.83s/it]
69%|██████▉ | 1082/1570 [1:30:53<23:03, 2.84s/it]
69%|██████▉ | 1083/1570 [1:30:56<23:04, 2.84s/it]
69%|██████▉ | 1084/1570 [1:30:59<23:05, 2.85s/it]
69%|██████▉ | 1085/1570 [1:31:02<22:58, 2.84s/it]
69%|██████▉ | 1086/1570 [1:31:05<22:51, 2.83s/it]
69%|██████▉ | 1087/1570 [1:31:07<22:45, 2.83s/it]
69%|██████▉ | 1088/1570 [1:31:10<22:39, 2.82s/it]
69%|██████▉ | 1089/1570 [1:31:13<22:36, 2.82s/it]
69%|██████▉ | 1090/1570 [1:31:16<22:29, 2.81s/it]
69%|██████▉ | 1091/1570 [1:31:19<22:24, 2.81s/it]
70%|██████▉ | 1092/1570 [1:31:21<22:23, 2.81s/it]
70%|██████▉ | 1093/1570 [1:31:24<22:18, 2.81s/it]
70%|██████▉ | 1094/1570 [1:31:27<22:13, 2.80s/it]
70%|██████▉ | 1095/1570 [1:31:30<22:09, 2.80s/it]
70%|██████▉ | 1096/1570 [1:31:33<22:10, 2.81s/it]
70%|██████▉ | 1097/1570 [1:31:35<22:05, 2.80s/it]
70%|██████▉ | 1098/1570 [1:31:38<22:10, 2.82s/it]
70%|███████ | 1099/1570 [1:31:41<22:11, 2.83s/it]
70%|███████ | 1100/1570 [1:31:44<22:04, 2.82s/it]
{'loss': '0.000987', 'grad_norm': '0.01592', 'learning_rate': '2.069e-05', 'epoch': '3.504'}
+
70%|███████ | 1100/1570 [1:31:44<22:04, 2.82s/it]
70%|███████ | 1101/1570 [1:31:47<22:08, 2.83s/it]
70%|███████ | 1102/1570 [1:31:50<22:04, 2.83s/it]
70%|███████ | 1103/1570 [1:31:52<21:58, 2.82s/it]
70%|███████ | 1104/1570 [1:31:55<21:48, 2.81s/it]
70%|███████ | 1105/1570 [1:31:58<21:44, 2.81s/it]
70%|███████ | 1106/1570 [1:32:01<21:44, 2.81s/it]
71%|███████ | 1107/1570 [1:32:04<21:49, 2.83s/it]
71%|███████ | 1108/1570 [1:32:06<21:38, 2.81s/it]
71%|███████ | 1109/1570 [1:32:09<21:30, 2.80s/it]
71%|███████ | 1110/1570 [1:32:12<21:29, 2.80s/it]
71%|███████ | 1111/1570 [1:32:15<21:27, 2.81s/it]
71%|███████ | 1112/1570 [1:32:18<21:22, 2.80s/it]
71%|███████ | 1113/1570 [1:32:20<21:24, 2.81s/it]
71%|███████ | 1114/1570 [1:32:23<21:27, 2.82s/it]
71%|███████ | 1115/1570 [1:32:26<21:28, 2.83s/it]
71%|███████ | 1116/1570 [1:32:29<21:25, 2.83s/it]
71%|███████ | 1117/1570 [1:32:32<21:22, 2.83s/it]
71%|███████ | 1118/1570 [1:32:35<21:15, 2.82s/it]
71%|███████▏ | 1119/1570 [1:32:38<21:26, 2.85s/it]
71%|███████▏ | 1120/1570 [1:32:40<21:14, 2.83s/it]
{'loss': '0.001458', 'grad_norm': '0.01483', 'learning_rate': '1.91e-05', 'epoch': '3.568'}
+
71%|███████▏ | 1120/1570 [1:32:40<21:14, 2.83s/it]
71%|███████▏ | 1121/1570 [1:32:43<21:09, 2.83s/it]
71%|███████▏ | 1122/1570 [1:32:46<21:13, 2.84s/it]
72%|███████▏ | 1123/1570 [1:32:49<21:09, 2.84s/it]
72%|███████▏ | 1124/1570 [1:32:52<21:05, 2.84s/it]
72%|███████▏ | 1125/1570 [1:32:54<20:57, 2.83s/it]
72%|███████▏ | 1126/1570 [1:32:57<21:04, 2.85s/it]
72%|███████▏ | 1127/1570 [1:33:00<20:51, 2.82s/it]
72%|███████▏ | 1128/1570 [1:33:03<21:01, 2.85s/it]
72%|███████▏ | 1129/1570 [1:33:06<20:58, 2.85s/it]
72%|███████▏ | 1130/1570 [1:33:09<21:02, 2.87s/it]
72%|███████▏ | 1131/1570 [1:33:12<20:52, 2.85s/it]
72%|███████▏ | 1132/1570 [1:33:15<20:54, 2.86s/it]
72%|███████▏ | 1133/1570 [1:33:17<20:44, 2.85s/it]
72%|███████▏ | 1134/1570 [1:33:20<20:45, 2.86s/it]
72%|███████▏ | 1135/1570 [1:33:23<20:45, 2.86s/it]
72%|███████▏ | 1136/1570 [1:33:26<20:44, 2.87s/it]
72%|███████▏ | 1137/1570 [1:33:29<20:38, 2.86s/it]
72%|███████▏ | 1138/1570 [1:33:32<20:42, 2.88s/it]
73%|███████▎ | 1139/1570 [1:33:35<20:40, 2.88s/it]
73%|███████▎ | 1140/1570 [1:33:38<20:58, 2.93s/it]
{'loss': '0.0008233', 'grad_norm': '0.02079', 'learning_rate': '1.755e-05', 'epoch': '3.632'}
+
73%|███████▎ | 1140/1570 [1:33:38<20:58, 2.93s/it]
73%|███████▎ | 1141/1570 [1:33:41<21:20, 2.99s/it]
73%|███████▎ | 1142/1570 [1:33:44<21:28, 3.01s/it]
73%|███████▎ | 1143/1570 [1:33:47<21:00, 2.95s/it]
73%|███████▎ | 1144/1570 [1:33:50<20:46, 2.93s/it]
73%|███████▎ | 1145/1570 [1:33:52<20:33, 2.90s/it]
73%|███████▎ | 1146/1570 [1:33:55<20:19, 2.88s/it]
73%|███████▎ | 1147/1570 [1:33:58<20:13, 2.87s/it]
73%|███████▎ | 1148/1570 [1:34:01<20:04, 2.85s/it]
73%|███████▎ | 1149/1570 [1:34:04<19:58, 2.85s/it]
73%|███████▎ | 1150/1570 [1:34:06<19:47, 2.83s/it]
73%|███████▎ | 1151/1570 [1:34:09<19:38, 2.81s/it]
73%|███████▎ | 1152/1570 [1:34:12<19:30, 2.80s/it]
73%|███████▎ | 1153/1570 [1:34:15<19:32, 2.81s/it]
74%|███████▎ | 1154/1570 [1:34:18<19:28, 2.81s/it]
74%|███████▎ | 1155/1570 [1:34:21<19:30, 2.82s/it]
74%|███████▎ | 1156/1570 [1:34:23<19:21, 2.81s/it]
74%|███████▎ | 1157/1570 [1:34:26<19:16, 2.80s/it]
74%|███████▍ | 1158/1570 [1:34:29<19:17, 2.81s/it]
74%|███████▍ | 1159/1570 [1:34:32<19:13, 2.81s/it]
74%|███████▍ | 1160/1570 [1:34:35<19:12, 2.81s/it]
{'loss': '0.001963', 'grad_norm': '0.005068', 'learning_rate': '1.606e-05', 'epoch': '3.696'}
+
74%|███████▍ | 1160/1570 [1:34:35<19:12, 2.81s/it]
74%|███████▍ | 1161/1570 [1:34:37<19:11, 2.81s/it]
74%|███████▍ | 1162/1570 [1:34:40<19:10, 2.82s/it]
74%|███████▍ | 1163/1570 [1:34:43<19:06, 2.82s/it]
74%|███████▍ | 1164/1570 [1:34:46<19:00, 2.81s/it]
74%|███████▍ | 1165/1570 [1:34:49<19:08, 2.83s/it]
74%|███████▍ | 1166/1570 [1:34:52<19:04, 2.83s/it]
74%|███████▍ | 1167/1570 [1:34:54<18:52, 2.81s/it]
74%|███████▍ | 1168/1570 [1:34:57<18:50, 2.81s/it]
74%|███████▍ | 1169/1570 [1:35:00<18:47, 2.81s/it]
75%|███████▍ | 1170/1570 [1:35:03<18:46, 2.82s/it]
75%|███████▍ | 1171/1570 [1:35:06<18:47, 2.83s/it]
75%|███████▍ | 1172/1570 [1:35:08<18:43, 2.82s/it]
75%|███████▍ | 1173/1570 [1:35:11<18:40, 2.82s/it]
75%|███████▍ | 1174/1570 [1:35:14<18:43, 2.84s/it]
75%|███████▍ | 1175/1570 [1:35:17<18:38, 2.83s/it]
75%|███████▍ | 1176/1570 [1:35:20<18:33, 2.82s/it]
75%|███████▍ | 1177/1570 [1:35:23<18:35, 2.84s/it]
75%|███████▌ | 1178/1570 [1:35:25<18:24, 2.82s/it]
75%|███████▌ | 1179/1570 [1:35:28<18:23, 2.82s/it]
75%|███████▌ | 1180/1570 [1:35:31<18:18, 2.82s/it]
{'loss': '0.002155', 'grad_norm': '0.01215', 'learning_rate': '1.462e-05', 'epoch': '3.759'}
+
75%|███████▌ | 1180/1570 [1:35:31<18:18, 2.82s/it]
75%|███████▌ | 1181/1570 [1:35:34<18:23, 2.84s/it]
75%|███████▌ | 1182/1570 [1:35:37<18:28, 2.86s/it]
75%|███████▌ | 1183/1570 [1:35:40<18:23, 2.85s/it]
75%|███████▌ | 1184/1570 [1:35:43<18:25, 2.86s/it]
75%|███████▌ | 1185/1570 [1:35:45<18:15, 2.84s/it]
76%|███████▌ | 1186/1570 [1:35:48<18:20, 2.87s/it]
76%|███████▌ | 1187/1570 [1:35:51<18:13, 2.86s/it]
76%|███████▌ | 1188/1570 [1:35:54<18:20, 2.88s/it]
76%|███████▌ | 1189/1570 [1:35:57<18:12, 2.87s/it]
76%|███████▌ | 1190/1570 [1:36:00<18:17, 2.89s/it]
76%|███████▌ | 1191/1570 [1:36:03<18:03, 2.86s/it]
76%|███████▌ | 1192/1570 [1:36:05<18:01, 2.86s/it]
76%|███████▌ | 1193/1570 [1:36:08<18:02, 2.87s/it]
76%|███████▌ | 1194/1570 [1:36:11<17:58, 2.87s/it]
76%|███████▌ | 1195/1570 [1:36:14<17:55, 2.87s/it]
76%|███████▌ | 1196/1570 [1:36:17<17:56, 2.88s/it]
76%|███████▌ | 1197/1570 [1:36:20<17:59, 2.89s/it]
76%|███████▋ | 1198/1570 [1:36:23<17:55, 2.89s/it]
76%|███████▋ | 1199/1570 [1:36:26<17:54, 2.90s/it]
76%|███████▋ | 1200/1570 [1:36:29<17:51, 2.90s/it]
{'loss': '0.001103', 'grad_norm': '0.01149', 'learning_rate': '1.324e-05', 'epoch': '3.823'}
+
76%|███████▋ | 1200/1570 [1:36:29<17:51, 2.90s/it]
76%|███████▋ | 1201/1570 [1:36:31<17:46, 2.89s/it]
77%|███████▋ | 1202/1570 [1:36:34<17:43, 2.89s/it]
77%|███████▋ | 1203/1570 [1:36:37<17:32, 2.87s/it]
77%|███████▋ | 1204/1570 [1:36:40<17:30, 2.87s/it]
77%|███████▋ | 1205/1570 [1:36:43<17:22, 2.86s/it]
77%|███████▋ | 1206/1570 [1:36:46<17:16, 2.85s/it]
77%|███████▋ | 1207/1570 [1:36:49<17:13, 2.85s/it]
77%|███████▋ | 1208/1570 [1:36:51<17:12, 2.85s/it]
77%|███████▋ | 1209/1570 [1:36:54<17:09, 2.85s/it]
77%|███████▋ | 1210/1570 [1:36:57<17:06, 2.85s/it]
77%|███████▋ | 1211/1570 [1:37:00<17:04, 2.85s/it]
77%|███████▋ | 1212/1570 [1:37:03<16:58, 2.85s/it]
77%|███████▋ | 1213/1570 [1:37:06<16:49, 2.83s/it]
77%|███████▋ | 1214/1570 [1:37:08<16:48, 2.83s/it]
77%|███████▋ | 1215/1570 [1:37:11<16:45, 2.83s/it]
77%|███████▋ | 1216/1570 [1:37:14<16:46, 2.84s/it]
78%|███████▊ | 1217/1570 [1:37:17<16:44, 2.84s/it]
78%|███████▊ | 1218/1570 [1:37:20<16:40, 2.84s/it]
78%|███████▊ | 1219/1570 [1:37:23<16:39, 2.85s/it]
78%|███████▊ | 1220/1570 [1:37:25<16:31, 2.83s/it]
{'loss': '0.00132', 'grad_norm': '0.0656', 'learning_rate': '1.192e-05', 'epoch': '3.887'}
+
78%|███████▊ | 1220/1570 [1:37:25<16:31, 2.83s/it]
78%|███████▊ | 1221/1570 [1:37:28<16:30, 2.84s/it]
78%|███████▊ | 1222/1570 [1:37:31<16:25, 2.83s/it]
78%|███████▊ | 1223/1570 [1:37:34<16:21, 2.83s/it]
78%|███████▊ | 1224/1570 [1:37:37<16:21, 2.84s/it]
78%|███████▊ | 1225/1570 [1:37:40<16:17, 2.83s/it]
78%|███████▊ | 1226/1570 [1:37:42<16:15, 2.83s/it]
78%|███████▊ | 1227/1570 [1:37:45<16:14, 2.84s/it]
78%|███████▊ | 1228/1570 [1:37:48<16:11, 2.84s/it]
78%|███████▊ | 1229/1570 [1:37:51<16:07, 2.84s/it]
78%|███████▊ | 1230/1570 [1:37:54<16:14, 2.87s/it]
78%|███████▊ | 1231/1570 [1:37:57<16:08, 2.86s/it]
78%|███████▊ | 1232/1570 [1:38:00<17:16, 3.07s/it]
79%|███████▊ | 1233/1570 [1:38:04<18:23, 3.27s/it]
79%|███████▊ | 1234/1570 [1:38:08<19:21, 3.46s/it]
79%|███████▊ | 1235/1570 [1:38:12<20:00, 3.58s/it]
79%|███████▊ | 1236/1570 [1:38:16<20:30, 3.68s/it]
79%|███████▉ | 1237/1570 [1:38:20<20:45, 3.74s/it]
79%|███████▉ | 1238/1570 [1:38:24<20:56, 3.78s/it]
79%|███████▉ | 1239/1570 [1:38:27<21:04, 3.82s/it]
79%|███████▉ | 1240/1570 [1:38:31<20:31, 3.73s/it]
{'loss': '0.0009617', 'grad_norm': '0.01427', 'learning_rate': '1.066e-05', 'epoch': '3.951'}
+
79%|███████▉ | 1240/1570 [1:38:31<20:31, 3.73s/it]
79%|███████▉ | 1241/1570 [1:38:34<19:11, 3.50s/it]
79%|███████▉ | 1242/1570 [1:38:37<17:56, 3.28s/it]
79%|███████▉ | 1243/1570 [1:38:40<17:17, 3.17s/it]
79%|███████▉ | 1244/1570 [1:38:42<16:39, 3.07s/it]
79%|███████▉ | 1245/1570 [1:38:45<16:21, 3.02s/it]
79%|███████▉ | 1246/1570 [1:38:48<16:02, 2.97s/it]
79%|███████▉ | 1247/1570 [1:38:51<15:58, 2.97s/it]
79%|███████▉ | 1248/1570 [1:38:54<15:42, 2.93s/it]
80%|███████▉ | 1249/1570 [1:38:57<15:30, 2.90s/it]
80%|███████▉ | 1250/1570 [1:39:00<15:26, 2.90s/it]
80%|███████▉ | 1251/1570 [1:39:02<15:13, 2.86s/it]
80%|███████▉ | 1252/1570 [1:39:05<15:15, 2.88s/it]
80%|███████▉ | 1253/1570 [1:39:08<15:02, 2.85s/it]
80%|███████▉ | 1254/1570 [1:39:11<15:05, 2.86s/it]
80%|███████▉ | 1255/1570 [1:39:14<15:00, 2.86s/it]
80%|████████ | 1256/1570 [1:39:15<12:28, 2.38s/it][INFO|trainer.py:2668] 2026-08-22 11:50:10,620 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 11:50:10,621 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 11:50:10,621 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/fkl/checkpoint-1256
+[INFO|configuration_utils.py:780] 2026-08-22 12:01:40,507 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 12:01:40,509 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 12:01:40,544 >> chat template saved in results/qwen3/fkl/checkpoint-1256/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 12:01:40,544 >> tokenizer config file saved in results/qwen3/fkl/checkpoint-1256/tokenizer_config.json
+
80%|████████ | 1257/1570 [1:50:49<18:14:19, 209.78s/it]
80%|████████ | 1258/1570 [1:50:52<12:47:57, 147.68s/it]
80%|████████ | 1259/1570 [1:50:54<9:00:12, 104.22s/it]
80%|████████ | 1260/1570 [1:50:57<6:21:19, 73.81s/it]
{'loss': '0.001441', 'grad_norm': '0.02666', 'learning_rate': '9.464e-06', 'epoch': '4.013'}
+
80%|████████ | 1260/1570 [1:50:57<6:21:19, 73.81s/it]
80%|████████ | 1261/1570 [1:51:00<4:30:26, 52.51s/it]
80%|████████ | 1262/1570 [1:51:03<3:13:03, 37.61s/it]
80%|████████ | 1263/1570 [1:51:06<2:19:03, 27.18s/it]
81%|████████ | 1264/1570 [1:51:09<1:41:23, 19.88s/it]
81%|████████ | 1265/1570 [1:51:11<1:15:02, 14.76s/it]
81%|████████ | 1266/1570 [1:51:14<56:37, 11.18s/it]
81%|████████ | 1267/1570 [1:51:17<43:49, 8.68s/it]
81%|████████ | 1268/1570 [1:51:20<34:51, 6.93s/it]
81%|████████ | 1269/1570 [1:51:23<28:32, 5.69s/it]
81%|████████ | 1270/1570 [1:51:26<24:08, 4.83s/it]
81%|████████ | 1271/1570 [1:51:28<21:06, 4.24s/it]
81%|████████ | 1272/1570 [1:51:31<18:58, 3.82s/it]
81%|████████ | 1273/1570 [1:51:34<17:25, 3.52s/it]
81%|████████ | 1274/1570 [1:51:37<16:21, 3.31s/it]
81%|████████ | 1275/1570 [1:51:40<15:39, 3.18s/it]
81%|████████▏ | 1276/1570 [1:51:43<15:07, 3.09s/it]
81%|████████▏ | 1277/1570 [1:51:46<14:44, 3.02s/it]
81%|████████▏ | 1278/1570 [1:51:48<14:23, 2.96s/it]
81%|████████▏ | 1279/1570 [1:51:51<14:17, 2.95s/it]
82%|████████▏ | 1280/1570 [1:51:54<14:03, 2.91s/it]
{'loss': '0.0007118', 'grad_norm': '0.05525', 'learning_rate': '8.332e-06', 'epoch': '4.077'}
+
82%|████████▏ | 1280/1570 [1:51:54<14:03, 2.91s/it]
82%|████████▏ | 1281/1570 [1:51:57<13:51, 2.88s/it]
82%|████████▏ | 1282/1570 [1:52:00<13:43, 2.86s/it]
82%|████████▏ | 1283/1570 [1:52:03<13:36, 2.84s/it]
82%|████████▏ | 1284/1570 [1:52:05<13:32, 2.84s/it]
82%|████████▏ | 1285/1570 [1:52:08<13:25, 2.83s/it]
82%|████████▏ | 1286/1570 [1:52:11<13:19, 2.82s/it]
82%|████████▏ | 1287/1570 [1:52:14<13:17, 2.82s/it]
82%|████████▏ | 1288/1570 [1:52:17<13:12, 2.81s/it]
82%|████████▏ | 1289/1570 [1:52:19<13:06, 2.80s/it]
82%|████████▏ | 1290/1570 [1:52:22<13:07, 2.81s/it]
82%|████████▏ | 1291/1570 [1:52:25<13:04, 2.81s/it]
82%|████████▏ | 1292/1570 [1:52:28<13:03, 2.82s/it]
82%|████████▏ | 1293/1570 [1:52:31<13:01, 2.82s/it]
82%|████████▏ | 1294/1570 [1:52:33<12:58, 2.82s/it]
82%|████████▏ | 1295/1570 [1:52:36<12:57, 2.83s/it]
83%|████████▎ | 1296/1570 [1:52:39<12:53, 2.82s/it]
83%|████████▎ | 1297/1570 [1:52:42<12:53, 2.83s/it]
83%|████████▎ | 1298/1570 [1:52:45<12:47, 2.82s/it]
83%|████████▎ | 1299/1570 [1:52:48<12:44, 2.82s/it]
83%|████████▎ | 1300/1570 [1:52:50<12:40, 2.82s/it]
{'loss': '0.001056', 'grad_norm': '0.003827', 'learning_rate': '7.266e-06', 'epoch': '4.14'}
+
83%|████████▎ | 1300/1570 [1:52:50<12:40, 2.82s/it]
83%|████████▎ | 1301/1570 [1:52:53<12:39, 2.82s/it]
83%|████████▎ | 1302/1570 [1:52:56<12:35, 2.82s/it]
83%|████████▎ | 1303/1570 [1:52:59<12:33, 2.82s/it]
83%|████████▎ | 1304/1570 [1:53:02<12:29, 2.82s/it]
83%|████████▎ | 1305/1570 [1:53:05<12:28, 2.82s/it]
83%|████████▎ | 1306/1570 [1:53:07<12:25, 2.82s/it]
83%|████████▎ | 1307/1570 [1:53:10<12:26, 2.84s/it]
83%|████████▎ | 1308/1570 [1:53:13<12:21, 2.83s/it]
83%|████████▎ | 1309/1570 [1:53:16<12:18, 2.83s/it]
83%|████████▎ | 1310/1570 [1:53:19<12:16, 2.83s/it]
84%|████████▎ | 1311/1570 [1:53:22<12:13, 2.83s/it]
84%|████████▎ | 1312/1570 [1:53:24<12:07, 2.82s/it]
84%|████████▎ | 1313/1570 [1:53:27<12:02, 2.81s/it]
84%|████████▎ | 1314/1570 [1:53:30<11:58, 2.81s/it]
84%|████████▍ | 1315/1570 [1:53:33<11:55, 2.80s/it]
84%|████████▍ | 1316/1570 [1:53:36<11:52, 2.80s/it]
84%|████████▍ | 1317/1570 [1:53:38<11:47, 2.79s/it]
84%|████████▍ | 1318/1570 [1:53:41<11:51, 2.82s/it]
84%|████████▍ | 1319/1570 [1:53:44<11:45, 2.81s/it]
84%|████████▍ | 1320/1570 [1:53:47<11:46, 2.83s/it]
{'loss': '0.0007781', 'grad_norm': '0.009402', 'learning_rate': '6.269e-06', 'epoch': '4.204'}
+
84%|████████▍ | 1320/1570 [1:53:47<11:46, 2.83s/it]
84%|████████▍ | 1321/1570 [1:53:50<11:41, 2.82s/it]
84%|████████▍ | 1322/1570 [1:53:53<11:46, 2.85s/it]
84%|████████▍ | 1323/1570 [1:53:55<11:40, 2.84s/it]
84%|████████▍ | 1324/1570 [1:53:58<11:43, 2.86s/it]
84%|████████▍ | 1325/1570 [1:54:01<11:40, 2.86s/it]
84%|████████▍ | 1326/1570 [1:54:04<11:44, 2.89s/it]
85%|████████▍ | 1327/1570 [1:54:07<11:34, 2.86s/it]
85%|████████▍ | 1328/1570 [1:54:10<11:30, 2.85s/it]
85%|████████▍ | 1329/1570 [1:54:13<11:26, 2.85s/it]
85%|████████▍ | 1330/1570 [1:54:15<11:21, 2.84s/it]
85%|████████▍ | 1331/1570 [1:54:18<11:13, 2.82s/it]
85%|████████▍ | 1332/1570 [1:54:21<11:17, 2.85s/it]
85%|████████▍ | 1333/1570 [1:54:24<11:19, 2.87s/it]
85%|████████▍ | 1334/1570 [1:54:27<11:22, 2.89s/it]
85%|████████▌ | 1335/1570 [1:54:30<11:33, 2.95s/it]
85%|████████▌ | 1336/1570 [1:54:33<11:39, 2.99s/it]
85%|████████▌ | 1337/1570 [1:54:36<11:29, 2.96s/it]
85%|████████▌ | 1338/1570 [1:54:39<11:14, 2.91s/it]
85%|████████▌ | 1339/1570 [1:54:42<11:04, 2.88s/it]
85%|████████▌ | 1340/1570 [1:54:44<10:56, 2.85s/it]
{'loss': '0.001051', 'grad_norm': '0.0375', 'learning_rate': '5.342e-06', 'epoch': '4.268'}
+
85%|████████▌ | 1340/1570 [1:54:44<10:56, 2.85s/it]
85%|████████▌ | 1341/1570 [1:54:47<10:49, 2.83s/it]
85%|████████▌ | 1342/1570 [1:54:50<10:45, 2.83s/it]
86%|████████▌ | 1343/1570 [1:54:53<10:41, 2.83s/it]
86%|████████▌ | 1344/1570 [1:54:56<10:36, 2.82s/it]
86%|████████▌ | 1345/1570 [1:54:58<10:34, 2.82s/it]
86%|████████▌ | 1346/1570 [1:55:01<10:31, 2.82s/it]
86%|████████▌ | 1347/1570 [1:55:04<10:26, 2.81s/it]
86%|████████▌ | 1348/1570 [1:55:07<10:21, 2.80s/it]
86%|████████▌ | 1349/1570 [1:55:10<10:18, 2.80s/it]
86%|████████▌ | 1350/1570 [1:55:12<10:15, 2.80s/it]
86%|████████▌ | 1351/1570 [1:55:15<10:14, 2.81s/it]
86%|████████▌ | 1352/1570 [1:55:18<10:12, 2.81s/it]
86%|████████▌ | 1353/1570 [1:55:21<10:10, 2.81s/it]
86%|████████▌ | 1354/1570 [1:55:24<10:08, 2.82s/it]
86%|████████▋ | 1355/1570 [1:55:26<10:03, 2.81s/it]
86%|████████▋ | 1356/1570 [1:55:29<09:58, 2.80s/it]
86%|████████▋ | 1357/1570 [1:55:32<09:56, 2.80s/it]
86%|████████▋ | 1358/1570 [1:55:35<09:51, 2.79s/it]
87%|████████▋ | 1359/1570 [1:55:38<09:50, 2.80s/it]
87%|████████▋ | 1360/1570 [1:55:40<09:51, 2.82s/it]
{'loss': '0.0007311', 'grad_norm': '0.008091', 'learning_rate': '4.486e-06', 'epoch': '4.332'}
+
87%|████████▋ | 1360/1570 [1:55:40<09:51, 2.82s/it]
87%|████████▋ | 1361/1570 [1:55:43<09:47, 2.81s/it]
87%|████████▋ | 1362/1570 [1:55:46<09:43, 2.80s/it]
87%|████████▋ | 1363/1570 [1:55:49<09:37, 2.79s/it]
87%|████████▋ | 1364/1570 [1:55:52<09:37, 2.80s/it]
87%|████████▋ | 1365/1570 [1:55:54<09:34, 2.80s/it]
87%|████████▋ | 1366/1570 [1:55:57<09:32, 2.81s/it]
87%|████████▋ | 1367/1570 [1:56:00<09:29, 2.81s/it]
87%|████████▋ | 1368/1570 [1:56:03<09:26, 2.81s/it]
87%|████████▋ | 1369/1570 [1:56:06<09:22, 2.80s/it]
87%|████████▋ | 1370/1570 [1:56:08<09:20, 2.80s/it]
87%|████████▋ | 1371/1570 [1:56:11<09:18, 2.81s/it]
87%|████████▋ | 1372/1570 [1:56:14<09:18, 2.82s/it]
87%|████████▋ | 1373/1570 [1:56:17<09:14, 2.81s/it]
88%|████████▊ | 1374/1570 [1:56:20<09:11, 2.81s/it]
88%|████████▊ | 1375/1570 [1:56:23<09:10, 2.82s/it]
88%|████████▊ | 1376/1570 [1:56:25<09:07, 2.82s/it]
88%|████████▊ | 1377/1570 [1:56:28<09:00, 2.80s/it]
88%|████████▊ | 1378/1570 [1:56:31<09:01, 2.82s/it]
88%|████████▊ | 1379/1570 [1:56:34<08:59, 2.83s/it]
88%|████████▊ | 1380/1570 [1:56:37<08:57, 2.83s/it]
{'loss': '0.0008125', 'grad_norm': '0.003737', 'learning_rate': '3.704e-06', 'epoch': '4.396'}
+
88%|████████▊ | 1380/1570 [1:56:37<08:57, 2.83s/it]
88%|████████▊ | 1381/1570 [1:56:40<09:21, 2.97s/it]
88%|████████▊ | 1382/1570 [1:56:43<09:08, 2.92s/it]
88%|████████▊ | 1383/1570 [1:56:46<08:58, 2.88s/it]
88%|████████▊ | 1384/1570 [1:56:48<08:50, 2.85s/it]
88%|████████▊ | 1385/1570 [1:56:51<08:46, 2.85s/it]
88%|████████▊ | 1386/1570 [1:56:54<08:38, 2.82s/it]
88%|████████▊ | 1387/1570 [1:56:57<08:38, 2.83s/it]
88%|████████▊ | 1388/1570 [1:57:00<08:35, 2.83s/it]
88%|████████▊ | 1389/1570 [1:57:03<08:32, 2.83s/it]
89%|████████▊ | 1390/1570 [1:57:06<08:50, 2.95s/it]
89%|████████▊ | 1391/1570 [1:57:09<08:42, 2.92s/it]
89%|████████▊ | 1392/1570 [1:57:11<08:32, 2.88s/it]
89%|████████▊ | 1393/1570 [1:57:14<08:27, 2.87s/it]
89%|████████▉ | 1394/1570 [1:57:17<08:23, 2.86s/it]
89%|████████▉ | 1395/1570 [1:57:20<08:18, 2.85s/it]
89%|████████▉ | 1396/1570 [1:57:23<08:14, 2.84s/it]
89%|████████▉ | 1397/1570 [1:57:25<08:08, 2.82s/it]
89%|████████▉ | 1398/1570 [1:57:28<08:06, 2.83s/it]
89%|████████▉ | 1399/1570 [1:57:31<08:02, 2.82s/it]
89%|████████▉ | 1400/1570 [1:57:34<07:59, 2.82s/it]
{'loss': '0.001586', 'grad_norm': '0.03763', 'learning_rate': '2.996e-06', 'epoch': '4.46'}
+
89%|████████▉ | 1400/1570 [1:57:34<07:59, 2.82s/it]
89%|████████▉ | 1401/1570 [1:57:37<07:53, 2.80s/it]
89%|████████▉ | 1402/1570 [1:57:40<07:52, 2.81s/it]
89%|████████▉ | 1403/1570 [1:57:42<07:48, 2.81s/it]
89%|████████▉ | 1404/1570 [1:57:45<07:48, 2.82s/it]
89%|████████▉ | 1405/1570 [1:57:48<07:45, 2.82s/it]
90%|████████▉ | 1406/1570 [1:57:51<07:41, 2.82s/it]
90%|████████▉ | 1407/1570 [1:57:54<07:38, 2.81s/it]
90%|████████▉ | 1408/1570 [1:57:56<07:35, 2.81s/it]
90%|████████▉ | 1409/1570 [1:57:59<07:33, 2.81s/it]
90%|████████▉ | 1410/1570 [1:58:02<07:31, 2.82s/it]
90%|████████▉ | 1411/1570 [1:58:05<07:27, 2.81s/it]
90%|████████▉ | 1412/1570 [1:58:08<07:25, 2.82s/it]
90%|█████████ | 1413/1570 [1:58:11<07:23, 2.82s/it]
90%|█████████ | 1414/1570 [1:58:13<07:19, 2.82s/it]
90%|█████████ | 1415/1570 [1:58:16<07:17, 2.82s/it]
90%|█████████ | 1416/1570 [1:58:19<07:13, 2.82s/it]
90%|█████████ | 1417/1570 [1:58:22<07:10, 2.81s/it]
90%|█████████ | 1418/1570 [1:58:25<07:06, 2.80s/it]
90%|█████████ | 1419/1570 [1:58:27<07:02, 2.80s/it]
90%|█████████ | 1420/1570 [1:58:30<07:01, 2.81s/it]
{'loss': '0.001064', 'grad_norm': '0.04097', 'learning_rate': '2.363e-06', 'epoch': '4.523'}
+
90%|█████████ | 1420/1570 [1:58:30<07:01, 2.81s/it]
91%|█████████ | 1421/1570 [1:58:33<06:57, 2.81s/it]
91%|█████████ | 1422/1570 [1:58:36<06:53, 2.80s/it]
91%|█████████ | 1423/1570 [1:58:39<06:51, 2.80s/it]
91%|█████████ | 1424/1570 [1:58:41<06:50, 2.81s/it]
91%|█████████ | 1425/1570 [1:58:44<06:48, 2.82s/it]
91%|█████████ | 1426/1570 [1:58:47<06:46, 2.82s/it]
91%|█████████ | 1427/1570 [1:58:50<06:41, 2.81s/it]
91%|█████████ | 1428/1570 [1:58:53<06:39, 2.81s/it]
91%|█████████ | 1429/1570 [1:58:55<06:36, 2.81s/it]
91%|█████████ | 1430/1570 [1:58:58<06:33, 2.81s/it]
91%|█████████ | 1431/1570 [1:59:01<06:30, 2.81s/it]
91%|█████████ | 1432/1570 [1:59:04<06:28, 2.82s/it]
91%|█████████▏| 1433/1570 [1:59:07<06:24, 2.81s/it]
91%|█████████▏| 1434/1570 [1:59:10<06:23, 2.82s/it]
91%|█████████▏| 1435/1570 [1:59:12<06:21, 2.83s/it]
91%|█████████▏| 1436/1570 [1:59:15<06:17, 2.82s/it]
92%|█████████▏| 1437/1570 [1:59:18<06:14, 2.82s/it]
92%|█████████▏| 1438/1570 [1:59:21<06:14, 2.83s/it]
92%|█████████▏| 1439/1570 [1:59:24<06:10, 2.83s/it]
92%|█████████▏| 1440/1570 [1:59:27<06:08, 2.83s/it]
{'loss': '0.0007133', 'grad_norm': '0.01522', 'learning_rate': '1.806e-06', 'epoch': '4.587'}
+
92%|█████████▏| 1440/1570 [1:59:27<06:08, 2.83s/it]
92%|█████████▏| 1441/1570 [1:59:29<06:04, 2.83s/it]
92%|█████████▏| 1442/1570 [1:59:32<05:58, 2.80s/it]
92%|█████████▏| 1443/1570 [1:59:35<05:56, 2.81s/it]
92%|█████████▏| 1444/1570 [1:59:38<05:54, 2.81s/it]
92%|█████████▏| 1445/1570 [1:59:41<05:51, 2.81s/it]
92%|█████████▏| 1446/1570 [1:59:43<05:48, 2.81s/it]
92%|█████████▏| 1447/1570 [1:59:46<05:45, 2.81s/it]
92%|█████████▏| 1448/1570 [1:59:49<05:42, 2.80s/it]
92%|█████████▏| 1449/1570 [1:59:52<05:39, 2.80s/it]
92%|█████████▏| 1450/1570 [1:59:55<05:37, 2.81s/it]
92%|█████████▏| 1451/1570 [1:59:57<05:36, 2.83s/it]
92%|█████████▏| 1452/1570 [2:00:00<05:33, 2.83s/it]
93%|█████████▎| 1453/1570 [2:00:03<05:31, 2.84s/it]
93%|█████████▎| 1454/1570 [2:00:06<05:28, 2.84s/it]
93%|█████████▎| 1455/1570 [2:00:09<05:24, 2.82s/it]
93%|█████████▎| 1456/1570 [2:00:12<05:20, 2.81s/it]
93%|█████████▎| 1457/1570 [2:00:14<05:16, 2.80s/it]
93%|█████████▎| 1458/1570 [2:00:17<05:15, 2.82s/it]
93%|█████████▎| 1459/1570 [2:00:20<05:12, 2.81s/it]
93%|█████████▎| 1460/1570 [2:00:23<05:09, 2.81s/it]
{'loss': '0.0007402', 'grad_norm': '0.01911', 'learning_rate': '1.327e-06', 'epoch': '4.651'}
+
93%|█████████▎| 1460/1570 [2:00:23<05:09, 2.81s/it]
93%|█████████▎| 1461/1570 [2:00:26<05:05, 2.80s/it]
93%|█████████▎| 1462/1570 [2:00:28<05:02, 2.80s/it]
93%|█████████▎| 1463/1570 [2:00:31<05:00, 2.81s/it]
93%|█████████▎| 1464/1570 [2:00:34<04:57, 2.81s/it]
93%|█████████▎| 1465/1570 [2:00:37<04:54, 2.81s/it]
93%|█████████▎| 1466/1570 [2:00:40<04:52, 2.81s/it]
93%|█████████▎| 1467/1570 [2:00:42<04:49, 2.81s/it]
94%|█████████▎| 1468/1570 [2:00:45<04:47, 2.82s/it]
94%|█████████▎| 1469/1570 [2:00:48<04:44, 2.81s/it]
94%|█████████▎| 1470/1570 [2:00:51<04:41, 2.81s/it]
94%|█████████▎| 1471/1570 [2:00:54<04:38, 2.81s/it]
94%|█████████▍| 1472/1570 [2:00:57<04:34, 2.80s/it]
94%|█████████▍| 1473/1570 [2:00:59<04:31, 2.80s/it]
94%|█████████▍| 1474/1570 [2:01:02<04:28, 2.80s/it]
94%|█████████▍| 1475/1570 [2:01:05<04:27, 2.82s/it]
94%|█████████▍| 1476/1570 [2:01:08<04:24, 2.81s/it]
94%|█████████▍| 1477/1570 [2:01:11<04:20, 2.81s/it]
94%|█████████▍| 1478/1570 [2:01:13<04:17, 2.80s/it]
94%|█████████▍| 1479/1570 [2:01:16<04:14, 2.79s/it]
94%|█████████▍| 1480/1570 [2:01:19<04:11, 2.80s/it]
{'loss': '0.0008888', 'grad_norm': '0.03158', 'learning_rate': '9.258e-07', 'epoch': '4.715'}
+
94%|█████████▍| 1480/1570 [2:01:19<04:11, 2.80s/it]
94%|█████████▍| 1481/1570 [2:01:22<04:09, 2.80s/it]
94%|█████████▍| 1482/1570 [2:01:24<04:04, 2.78s/it]
94%|█████████▍| 1483/1570 [2:01:27<04:02, 2.79s/it]
95%|█████████▍| 1484/1570 [2:01:30<04:00, 2.80s/it]
95%|█████████▍| 1485/1570 [2:01:33<03:56, 2.79s/it]
95%|█████████▍| 1486/1570 [2:01:36<03:54, 2.79s/it]
95%|█████████▍| 1487/1570 [2:01:38<03:51, 2.79s/it]
95%|█████████▍| 1488/1570 [2:01:41<03:50, 2.81s/it]
95%|█████████▍| 1489/1570 [2:01:44<03:47, 2.81s/it]
95%|█████████▍| 1490/1570 [2:01:47<03:44, 2.81s/it]
95%|█████████▍| 1491/1570 [2:01:50<03:41, 2.80s/it]
95%|█████████▌| 1492/1570 [2:01:53<03:38, 2.81s/it]
95%|█████████▌| 1493/1570 [2:01:55<03:35, 2.80s/it]
95%|█████████▌| 1494/1570 [2:01:58<03:33, 2.81s/it]
95%|█████████▌| 1495/1570 [2:02:01<03:30, 2.80s/it]
95%|█████████▌| 1496/1570 [2:02:04<03:28, 2.81s/it]
95%|█████████▌| 1497/1570 [2:02:07<03:25, 2.82s/it]
95%|█████████▌| 1498/1570 [2:02:09<03:22, 2.82s/it]
95%|█████████▌| 1499/1570 [2:02:12<03:19, 2.81s/it]
96%|█████████▌| 1500/1570 [2:02:15<03:16, 2.80s/it]
{'loss': '0.001249', 'grad_norm': '0.008442', 'learning_rate': '6.033e-07', 'epoch': '4.779'}
+
96%|█████████▌| 1500/1570 [2:02:15<03:16, 2.80s/it]
96%|█████████▌| 1501/1570 [2:02:18<03:13, 2.81s/it]
96%|█████████▌| 1502/1570 [2:02:21<03:10, 2.80s/it]
96%|█████████▌| 1503/1570 [2:02:23<03:08, 2.81s/it]
96%|█████████▌| 1504/1570 [2:02:26<03:05, 2.82s/it]
96%|█████████▌| 1505/1570 [2:02:29<03:02, 2.81s/it]
96%|█████████▌| 1506/1570 [2:02:32<03:00, 2.83s/it]
96%|█████████▌| 1507/1570 [2:02:35<02:57, 2.82s/it]
96%|█████████▌| 1508/1570 [2:02:38<02:55, 2.83s/it]
96%|█████████▌| 1509/1570 [2:02:40<02:52, 2.83s/it]
96%|█████████▌| 1510/1570 [2:02:43<02:50, 2.84s/it]
96%|█████████▌| 1511/1570 [2:02:46<02:47, 2.84s/it]
96%|█████████▋| 1512/1570 [2:02:49<02:44, 2.84s/it]
96%|█████████▋| 1513/1570 [2:02:52<02:41, 2.84s/it]
96%|█████████▋| 1514/1570 [2:02:55<02:38, 2.83s/it]
96%|█████████▋| 1515/1570 [2:02:57<02:35, 2.83s/it]
97%|█████████▋| 1516/1570 [2:03:00<02:31, 2.81s/it]
97%|█████████▋| 1517/1570 [2:03:03<02:29, 2.82s/it]
97%|█████████▋| 1518/1570 [2:03:06<02:27, 2.83s/it]
97%|█████████▋| 1519/1570 [2:03:09<02:24, 2.84s/it]
97%|█████████▋| 1520/1570 [2:03:12<02:21, 2.82s/it]
{'loss': '0.004219', 'grad_norm': '0.02742', 'learning_rate': '3.599e-07', 'epoch': '4.842'}
+
97%|█████████▋| 1520/1570 [2:03:12<02:21, 2.82s/it]
97%|█████████▋| 1521/1570 [2:03:14<02:18, 2.83s/it]
97%|█████████▋| 1522/1570 [2:03:17<02:16, 2.83s/it]
97%|█████████▋| 1523/1570 [2:03:20<02:12, 2.83s/it]
97%|█████████▋| 1524/1570 [2:03:23<02:10, 2.83s/it]
97%|█████████▋| 1525/1570 [2:03:26<02:07, 2.83s/it]
97%|█████████▋| 1526/1570 [2:03:29<02:04, 2.83s/it]
97%|█████████▋| 1527/1570 [2:03:31<02:01, 2.83s/it]
97%|█████████▋| 1528/1570 [2:03:34<01:58, 2.83s/it]
97%|█████████▋| 1529/1570 [2:03:37<01:55, 2.82s/it]
97%|█████████▋| 1530/1570 [2:03:40<01:53, 2.83s/it]
98%|█████████▊| 1531/1570 [2:03:43<01:50, 2.83s/it]
98%|█████████▊| 1532/1570 [2:03:45<01:47, 2.83s/it]
98%|█████████▊| 1533/1570 [2:03:48<01:44, 2.83s/it]
98%|█████████▊| 1534/1570 [2:03:51<01:41, 2.82s/it]
98%|█████████▊| 1535/1570 [2:03:54<01:39, 2.83s/it]
98%|█████████▊| 1536/1570 [2:03:57<01:36, 2.84s/it]
98%|█████████▊| 1537/1570 [2:04:00<01:33, 2.84s/it]
98%|█████████▊| 1538/1570 [2:04:03<01:30, 2.84s/it]
98%|█████████▊| 1539/1570 [2:04:05<01:27, 2.83s/it]
98%|█████████▊| 1540/1570 [2:04:08<01:24, 2.81s/it]
{'loss': '0.001393', 'grad_norm': '1.104', 'learning_rate': '1.961e-07', 'epoch': '4.906'}
+
98%|█████████▊| 1540/1570 [2:04:08<01:24, 2.81s/it]
98%|█████████▊| 1541/1570 [2:04:11<01:21, 2.82s/it]
98%|█████████▊| 1542/1570 [2:04:14<01:18, 2.81s/it]
98%|█████████▊| 1543/1570 [2:04:18<01:24, 3.11s/it]
98%|█████████▊| 1544/1570 [2:04:21<01:27, 3.35s/it]
98%|█████████▊| 1545/1570 [2:04:25<01:27, 3.48s/it]
98%|█████████▊| 1546/1570 [2:04:29<01:26, 3.61s/it]
99%|█████████▊| 1547/1570 [2:04:33<01:25, 3.72s/it]
99%|█████████▊| 1548/1570 [2:04:37<01:21, 3.72s/it]
99%|█████████▊| 1549/1570 [2:04:41<01:19, 3.77s/it]
99%|█████████▊| 1550/1570 [2:04:45<01:16, 3.81s/it]
99%|█████████▉| 1551/1570 [2:04:49<01:13, 3.86s/it]
99%|█████████▉| 1552/1570 [2:04:52<01:09, 3.86s/it]
99%|█████████▉| 1553/1570 [2:04:56<01:05, 3.86s/it]
99%|█████████▉| 1554/1570 [2:05:00<01:02, 3.89s/it]
99%|█████████▉| 1555/1570 [2:05:04<00:57, 3.86s/it]
99%|█████████▉| 1556/1570 [2:05:08<00:54, 3.89s/it]
99%|█████████▉| 1557/1570 [2:05:12<00:50, 3.87s/it]
99%|█████████▉| 1558/1570 [2:05:16<00:46, 3.84s/it]
99%|█████████▉| 1559/1570 [2:05:19<00:41, 3.79s/it]
99%|█████████▉| 1560/1570 [2:05:23<00:38, 3.84s/it]
{'loss': '0.0008278', 'grad_norm': '0.02524', 'learning_rate': '1.121e-07', 'epoch': '4.97'}
+
99%|█████████▉| 1560/1570 [2:05:23<00:38, 3.84s/it]
99%|█████████▉| 1561/1570 [2:05:27<00:34, 3.82s/it]
99%|█████████▉| 1562/1570 [2:05:31<00:30, 3.87s/it]
100%|█████████▉| 1563/1570 [2:05:35<00:27, 3.87s/it]
100%|█████████▉| 1564/1570 [2:05:39<00:23, 3.89s/it]
100%|█████████▉| 1565/1570 [2:05:43<00:19, 3.89s/it]
100%|█████████▉| 1566/1570 [2:05:47<00:15, 3.86s/it]
100%|█████████▉| 1567/1570 [2:05:50<00:11, 3.86s/it]
100%|█████████▉| 1568/1570 [2:05:54<00:07, 3.84s/it]
100%|█████████▉| 1569/1570 [2:05:58<00:03, 3.76s/it]
100%|██████████| 1570/1570 [2:05:59<00:00, 3.03s/it][INFO|trainer.py:2668] 2026-08-22 12:16:54,505 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 12:16:54,505 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 12:16:54,506 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/fkl/checkpoint-1570
+[INFO|configuration_utils.py:780] 2026-08-22 12:28:48,032 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 12:28:48,035 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 12:28:48,070 >> chat template saved in results/qwen3/fkl/checkpoint-1570/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 12:28:48,070 >> tokenizer config file saved in results/qwen3/fkl/checkpoint-1570/tokenizer_config.json
+[INFO|trainer.py:1823] 2026-08-22 12:28:48,853 >>
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
{'train_runtime': '8274', 'train_samples_per_second': '6.059', 'train_steps_per_second': '0.19', 'train_loss': '0.01522', 'epoch': '5'}
+
100%|██████████| 1570/1570 [2:17:53<00:00, 3.03s/it]
100%|██████████| 1570/1570 [2:17:53<00:00, 5.27s/it]
+[INFO|trainer.py:3815] 2026-08-22 12:28:49,340 >> Saving model checkpoint to results/qwen3/fkl
+[INFO|configuration_utils.py:780] 2026-08-22 12:28:49,840 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 12:28:49,843 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[transformers] The following generation flags are not valid and may be ignored: ['temperature', 'top_p']. Set `TRANSFORMERS_VERBOSITY=info` for more details.
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 12:28:49,877 >> chat template saved in results/qwen3/fkl/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 12:28:49,877 >> tokenizer config file saved in results/qwen3/fkl/tokenizer_config.json
+***** train metrics *****
+ epoch = 5.0
+ total_flos = 54682091GF
+ train_loss = 0.0152
+ train_runtime = 2:17:53.92
+ train_samples_per_second = 6.059
+ train_steps_per_second = 0.19
+[INFO|trainer.py:2668] 2026-08-22 12:28:49,998 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 12:28:49,998 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 12:28:49,998 >> Batch size = 16
+[WARNING|logging.py:340] 2026-08-22 12:28:50,108 >> The following generation flags are not valid and may be ignored: ['temperature', 'top_p'].
+[INFO|logging.py:355] 2026-08-22 12:28:50,108 >> - `temperature`: `do_sample` is not set to `True`. However, `temperature` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.
+- `top_p`: `do_sample` is not set to `True`. However, `top_p` is set to `0.7` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.
+If you're using a pretrained model, note that some of these attributes may be set through the model's `generation_config.json` file.
+[WARNING|utils.py:1642] 2026-08-22 12:28:50,111 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
0%| | 0/107 [00:00> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
2%|▏ | 2/107 [00:05<04:53, 2.80s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:29:01,557 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
3%|▎ | 3/107 [00:11<06:52, 3.96s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:29:07,145 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
4%|▎ | 4/107 [00:16<07:56, 4.63s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:29:12,901 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
5%|▍ | 5/107 [00:22<08:26, 4.96s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:29:18,490 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
6%|▌ | 6/107 [00:27<08:22, 4.97s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:29:23,481 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 7/107 [00:33<08:38, 5.18s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:29:29,105 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 8/107 [00:38<08:45, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:29:34,697 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
8%|▊ | 9/107 [00:44<09:07, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:29:40,900 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
9%|▉ | 10/107 [00:50<09:09, 5.67s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:29:46,749 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
10%|█ | 11/107 [00:55<08:38, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:29:51,547 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
11%|█ | 12/107 [01:01<08:45, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:29:57,367 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
12%|█▏ | 13/107 [01:07<08:43, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:30:03,032 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
13%|█▎ | 14/107 [01:12<08:33, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:30:08,444 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
14%|█▍ | 15/107 [01:18<08:31, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:30:14,105 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
15%|█▍ | 16/107 [01:23<08:28, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:30:19,732 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
16%|█▌ | 17/107 [01:29<08:30, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:30:25,625 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
17%|█▋ | 18/107 [01:35<08:27, 5.70s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:30:31,388 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
18%|█▊ | 19/107 [01:40<08:15, 5.63s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:30:36,859 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
19%|█▊ | 20/107 [01:46<08:10, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:30:42,516 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
20%|█▉ | 21/107 [01:51<07:57, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:30:47,867 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██ | 22/107 [01:57<08:01, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:30:53,778 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██▏ | 23/107 [02:03<07:55, 5.67s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:30:59,459 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
22%|██▏ | 24/107 [02:09<07:49, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:31:05,073 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
23%|██▎ | 25/107 [02:15<08:07, 5.95s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:31:11,704 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
24%|██▍ | 26/107 [02:21<08:03, 5.97s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:31:17,742 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
25%|██▌ | 27/107 [02:27<07:56, 5.95s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:31:23,643 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
26%|██▌ | 28/107 [02:33<07:44, 5.88s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:31:29,342 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
27%|██▋ | 29/107 [02:39<07:44, 5.95s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:31:35,480 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
28%|██▊ | 30/107 [02:45<07:32, 5.87s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:31:41,161 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
29%|██▉ | 31/107 [02:49<06:57, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:31:45,783 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
30%|██▉ | 32/107 [02:55<06:52, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:31:51,270 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
31%|███ | 33/107 [03:00<06:50, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:31:56,939 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
32%|███▏ | 34/107 [03:07<07:05, 5.83s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:32:03,413 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
33%|███▎ | 35/107 [03:13<06:55, 5.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:32:09,074 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
34%|███▎ | 36/107 [03:18<06:43, 5.69s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:32:14,553 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
35%|███▍ | 37/107 [03:23<06:18, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:32:19,319 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▌ | 38/107 [03:29<06:22, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:32:25,186 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▋ | 39/107 [03:34<06:10, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:32:30,386 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
37%|███▋ | 40/107 [03:40<06:07, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:32:35,981 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
38%|███▊ | 41/107 [03:45<06:06, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:32:41,704 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
39%|███▉ | 42/107 [03:52<06:15, 5.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:32:48,003 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
40%|████ | 43/107 [03:58<06:16, 5.88s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:32:54,106 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
41%|████ | 44/107 [04:03<06:02, 5.76s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:32:59,586 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
42%|████▏ | 45/107 [04:08<05:45, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:33:04,713 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
43%|████▎ | 46/107 [04:14<05:36, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:33:10,097 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
44%|████▍ | 47/107 [04:20<05:38, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:33:16,048 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
45%|████▍ | 48/107 [04:25<05:36, 5.71s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:33:21,915 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
46%|████▌ | 49/107 [04:31<05:27, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:33:27,438 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
47%|████▋ | 50/107 [04:36<05:18, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:33:32,876 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
48%|████▊ | 51/107 [04:42<05:15, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:33:38,621 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
49%|████▊ | 52/107 [04:48<05:05, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:33:43,972 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|████▉ | 53/107 [04:54<05:08, 5.71s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:33:50,061 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|█████ | 54/107 [04:59<04:55, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:33:55,339 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
51%|█████▏ | 55/107 [05:05<04:53, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:34:01,101 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
52%|█████▏ | 56/107 [05:11<04:51, 5.72s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:34:07,010 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
53%|█████▎ | 57/107 [05:16<04:48, 5.76s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:34:12,871 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
54%|█████▍ | 58/107 [05:22<04:38, 5.69s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:34:18,384 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
55%|█████▌ | 59/107 [05:27<04:30, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:34:23,900 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
56%|█████▌ | 60/107 [05:33<04:23, 5.60s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:34:29,411 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
57%|█████▋ | 61/107 [05:38<04:08, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:34:34,333 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
58%|█████▊ | 62/107 [05:43<04:03, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:34:39,808 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
59%|█████▉ | 63/107 [05:49<04:01, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:34:45,478 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
60%|█████▉ | 64/107 [05:54<03:54, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:34:50,830 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
61%|██████ | 65/107 [06:00<03:48, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:34:56,222 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
62%|██████▏ | 66/107 [06:05<03:43, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:35:01,712 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
63%|██████▎ | 67/107 [06:11<03:37, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:35:07,126 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▎ | 68/107 [06:16<03:26, 5.29s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:35:12,065 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▍ | 69/107 [06:21<03:21, 5.30s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:35:17,392 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
65%|██████▌ | 70/107 [06:27<03:25, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:35:23,524 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
66%|██████▋ | 71/107 [06:33<03:22, 5.63s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:35:29,350 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
67%|██████▋ | 72/107 [06:38<03:14, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:35:34,711 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
68%|██████▊ | 73/107 [06:44<03:09, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:35:40,309 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
69%|██████▉ | 74/107 [06:49<03:02, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:35:45,758 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
70%|███████ | 75/107 [06:55<02:55, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:35:51,143 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
71%|███████ | 76/107 [07:00<02:50, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:35:56,719 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
72%|███████▏ | 77/107 [07:06<02:45, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:36:02,296 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
73%|███████▎ | 78/107 [07:11<02:41, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:36:07,937 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
74%|███████▍ | 79/107 [07:17<02:35, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:36:13,492 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
75%|███████▍ | 80/107 [07:23<02:32, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:36:19,391 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
76%|███████▌ | 81/107 [07:28<02:25, 5.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:36:24,874 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
77%|███████▋ | 82/107 [07:34<02:18, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:36:30,234 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
78%|███████▊ | 83/107 [07:39<02:11, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:36:35,568 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▊ | 84/107 [07:44<02:04, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:36:40,845 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▉ | 85/107 [07:49<01:52, 5.11s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:36:45,239 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
80%|████████ | 86/107 [07:54<01:50, 5.27s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:36:50,894 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
81%|████████▏ | 87/107 [07:59<01:41, 5.08s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:36:55,533 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
82%|████████▏ | 88/107 [08:05<01:39, 5.22s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:37:01,061 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
83%|████████▎ | 89/107 [08:11<01:40, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:37:07,528 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
84%|████████▍ | 90/107 [08:17<01:34, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:37:13,076 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
85%|████████▌ | 91/107 [08:22<01:27, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:37:18,321 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
86%|████████▌ | 92/107 [08:27<01:21, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:37:23,573 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
87%|████████▋ | 93/107 [08:33<01:15, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:37:28,996 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
88%|████████▊ | 94/107 [08:38<01:11, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:37:34,664 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
89%|████████▉ | 95/107 [08:44<01:05, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:37:40,178 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
90%|████████▉ | 96/107 [08:49<01:00, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:37:45,549 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
91%|█████████ | 97/107 [08:55<00:55, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:37:51,203 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
92%|█████████▏| 98/107 [09:01<00:50, 5.62s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:37:57,073 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 99/107 [09:06<00:45, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:38:02,819 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 100/107 [09:12<00:39, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:38:08,236 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
94%|█████████▍| 101/107 [09:18<00:34, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:38:14,132 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
95%|█████████▌| 102/107 [09:23<00:28, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:38:19,794 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
96%|█████████▋| 103/107 [09:29<00:22, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:38:25,157 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
97%|█████████▋| 104/107 [09:34<00:16, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:38:30,745 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
98%|█████████▊| 105/107 [09:40<00:11, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:38:36,598 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
99%|█████████▉| 106/107 [09:46<00:05, 5.67s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 12:38:42,334 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
100%|██████████| 107/107 [09:52<00:00, 5.69s/it]
100%|██████████| 107/107 [09:53<00:00, 5.54s/it]
+***** eval metrics *****
+ epoch = 5.0
+ eval_generator_count = 1712.0
+ eval_generator_exact_match = 96.9042
+ eval_generator_rouge1 = 99.9327
+ eval_generator_rouge2 = 99.876
+ eval_generator_rougeL = 99.7659
+ eval_loss = 0.0035
+ eval_runtime = 0:09:59.14
+ eval_samples_per_second = 5.698
+ eval_selector_accuracy = 98.6565
+ eval_selector_count = 1712.0
+ eval_steps_per_second = 0.179
+[INFO|modelcard.py:263] 2026-08-22 12:38:49,144 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
diff --git a/qwen3/run_all_logs/20260822_031516/hpd.log b/qwen3/run_all_logs/20260822_031516/hpd.log
new file mode 100644
index 0000000000000000000000000000000000000000..16a93a6a61aa66ff5cceb069cd9f8c56080ee317
--- /dev/null
+++ b/qwen3/run_all_logs/20260822_031516/hpd.log
@@ -0,0 +1,2100 @@
+W0822 21:58:01.799000 1402930 torch/distributed/run.py:803]
+W0822 21:58:01.799000 1402930 torch/distributed/run.py:803] *****************************************
+W0822 21:58:01.799000 1402930 torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+W0822 21:58:01.799000 1402930 torch/distributed/run.py:803] *****************************************
+[INFO|2026-08-22 21:58:08] llamafactory.hparams.parser:651 >> Process rank: 1, world size: 2, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16
+[INFO|2026-08-22 21:58:08] llamafactory.hparams.parser:144 >> Set `ddp_find_unused_parameters` to False in DDP training since LoRA is enabled.
+[INFO|2026-08-22 21:58:08] llamafactory.hparams.parser:651 >> Process rank: 0, world size: 2, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16
+Starting hpd with LlamaFactory and DeepSpeed=True.
+[INFO|configuration_utils.py:780] 2026-08-22 21:58:10,016 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 21:58:10,020 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 21:58:13,171 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 21:58:13,174 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 21:58:13,414 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 21:58:13,417 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 21:58:14] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_train.jsonl...
+[INFO|2026-08-22 21:58:15] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_eval.jsonl...
+training example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 1947, 19029, 10592, 311, 35592, 304, 279, 17328, 1947, 1707, 429, 9482, 1573, 279, 1042, 220, 16, 21, 15, 15, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 9286, 38050, 756, 414, 330, 13193, 788, 341, 286, 330, 408, 14645, 788, 330, 68689, 756, 286, 330, 2468, 14645, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 37525, 756, 414, 330, 13193, 788, 5613, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 18098, 287, 756, 414, 330, 13193, 788, 341, 286, 330, 11141, 3575, 34043, 788, 330, 16468, 756, 286, 330, 37375, 14645, 788, 330, 68689, 698, 414, 456, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 9286, 38050, 756, 414, 330, 1313, 788, 330, 49087, 2354, 698, 262, 1153, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 37525, 756, 414, 330, 1313, 788, 330, 4648, 37525, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of art movements linked to paintings in the genre art style that ended before the year 1600?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "ArtMovement",
+ "properties": {
+ "end_year": "INTEGER",
+ "start_year": "INTEGER"
+ }
+ },
+ {
+ "label": "Genre",
+ "properties": {}
+ },
+ {
+ "label": "Painting",
+ "properties": {
+ "country_of_origin": "STRING",
+ "creation_year": "INTEGER"
+ }
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "ArtMovement",
+ "type": "associatedWith"
+ },
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "Genre",
+ "type": "hasGenre"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+eval example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 9893, 1483, 304, 2987, 279, 220, 17, 15, 17, 16, 55218, 18144, 73379, 476, 279, 23007, 24847, 7755, 9257, 36331, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 13193, 788, 341, 286, 330, 1028, 788, 330, 7097, 756, 286, 330, 31309, 788, 330, 22852, 756, 286, 330, 4082, 3575, 2259, 26286, 788, 330, 68689, 756, 286, 330, 4082, 3575, 1243, 89827, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 28214, 756, 414, 330, 13193, 788, 5613, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 5657, 788, 330, 28214, 756, 414, 330, 1313, 788, 330, 30210, 1047, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of weapons used in either the 2021 Baghdad suicide bombings or the Botroseya Church bombing?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "TerroristAttack",
+ "properties": {
+ "date": "DATE",
+ "locations": "LIST",
+ "number_of_deaths": "INTEGER",
+ "number_of_injuries": "INTEGER"
+ }
+ },
+ {
+ "label": "Weapon",
+ "properties": {}
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "TerroristAttack",
+ "target": "Weapon",
+ "type": "employs"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+[INFO|configuration_utils.py:780] 2026-08-22 21:58:19,356 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 21:58:19,359 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 21:58:19] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training.
+[INFO|modeling_utils.py:769] 2026-08-22 21:58:20,075 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/model.safetensors
+[INFO|modeling_utils.py:839] 2026-08-22 21:58:20,076 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 21:58:20,079 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": false
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 21:58:20,755 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95
+}
+
+[INFO|2026-08-22 21:58:20] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled.
+[INFO|2026-08-22 21:58:20] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-22 21:58:20] llamafactory.model.adapter:144 >> Upcasting trainable params to float32.
+[INFO|2026-08-22 21:58:20] llamafactory.model.adapter:144 >> Fine-tuning method: LoRA
+[INFO|2026-08-22 21:58:20] llamafactory.model.model_utils.misc:144 >> Found linear modules: v_proj,up_proj,q_proj,down_proj,k_proj,o_proj,gate_proj
+[INFO|2026-08-22 21:58:21] llamafactory.model.loader:144 >> trainable params: 20,185,088 || all params: 616,235,008 || trainable%: 3.2756
+[INFO|configuration_utils.py:780] 2026-08-22 21:58:21,667 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 21:58:21,672 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 21:58:24,443 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 21:58:24,679 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 21:58:24,684 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 21:58:26,706 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 21:58:26,709 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 21:58:26] llamafactory.model.model_utils.kv_cache:144 >> KV cache is enabled for faster generation.
+[INFO|modeling_utils.py:769] 2026-08-22 21:58:26,948 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model.safetensors.index.json
+[INFO|modeling_utils.py:839] 2026-08-22 21:58:26,948 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 21:58:26,949 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": true
+}
+
+
Loading weights: 0%| | 0/398 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 21:58:28,934 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.7,
+ "top_k": 20,
+ "top_p": 0.8
+}
+
+[INFO|2026-08-22 21:58:28] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-22 21:58:29] llamafactory.model.adapter:144 >> Merged 1 adapter(s).
+[INFO|2026-08-22 21:58:29] llamafactory.model.adapter:144 >> Loaded adapter(s): results/qwen3/teacher_lora
+[INFO|2026-08-22 21:58:29] llamafactory.model.loader:144 >> all params: 4,022,468,096
+[INFO|2026-08-22 21:58:29] llamafactory.train.trainer_utils:144 >> Created reference model from Qwen/Qwen3-4B-Instruct-2507
+
Loading weights: 0%| | 0/398 [00:00> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}.
+Gradient accumulation steps mismatch: GradientAccumulationPlugin has 1, DeepSpeed config has 8. Using DeepSpeed's value.
+[INFO|trainer.py:1469] 2026-08-22 21:58:34,865 >> ***** Running training *****
+[INFO|trainer.py:1470] 2026-08-22 21:58:34,865 >> Num examples = 10,027
+[INFO|trainer.py:1471] 2026-08-22 21:58:34,865 >> Num Epochs = 5
+[INFO|trainer.py:1472] 2026-08-22 21:58:34,865 >> Num update steps per epoch = 314
+[INFO|trainer.py:1473] 2026-08-22 21:58:34,865 >> Instantaneous batch size per device = 2
+[INFO|trainer.py:1476] 2026-08-22 21:58:34,865 >> Total train batch size (w. parallel, distributed & accumulation) = 32
+[INFO|trainer.py:1477] 2026-08-22 21:58:34,865 >> Gradient Accumulation steps = 8
+[INFO|trainer.py:1478] 2026-08-22 21:58:34,865 >> Total optimization steps = 1,570
+[INFO|trainer.py:1479] 2026-08-22 21:58:34,869 >> Number of trainable parameters = 20,185,088
+
0%| | 0/1570 [00:00>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 22:13:25,844 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 22:13:25,844 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/hpd/checkpoint-314
+[INFO|configuration_utils.py:780] 2026-08-22 22:23:22,261 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 22:23:22,266 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 22:23:22,301 >> chat template saved in results/qwen3/hpd/checkpoint-314/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 22:23:22,301 >> tokenizer config file saved in results/qwen3/hpd/checkpoint-314/tokenizer_config.json
+
20%|██ | 315/1570 [24:51<63:20:48, 181.71s/it]
20%|██ | 316/1570 [24:54<44:36:11, 128.05s/it]
20%|██ | 317/1570 [24:56<31:29:40, 90.49s/it]
20%|██ | 318/1570 [24:59<22:19:43, 64.20s/it]
20%|██ | 319/1570 [25:02<15:55:00, 45.80s/it]
20%|██ | 320/1570 [25:05<11:25:34, 32.91s/it]
{'loss': '0.01457', 'grad_norm': '4.387', 'learning_rate': '9.016e-05', 'neg_student_nll_loss': '0.01188', 'ce_loss': '0.01188', 'hpd_loss': '-0.00197', 'sampled_diff_pct': '1.217', 'adv1_neg_pct': '0.6673', 'adv2_active_pct': '1.217', 'mask3_pct': '1.217', 'k1_gt_raw_mean': '0.006571', 'k1_gt_raw_min': '-0.001141', 'k1_gt_raw_max': '0.1499', 'k1_gt_raw_neg_pct': '0.6673', 'k1_sample_mean': '-0.7651', 'k1_sample_min': '-0.9706', 'k1_sample_max': '-0.5734', 'k1_sample_neg_pct': '36.46', 'epoch': '1.019'}
+
20%|██ | 320/1570 [25:05<11:25:34, 32.91s/it]
20%|██ | 321/1570 [25:08<8:17:17, 23.89s/it]
21%|██ | 322/1570 [25:11<6:05:12, 17.56s/it]
21%|██ | 323/1570 [25:13<4:33:00, 13.14s/it]
21%|██ | 324/1570 [25:16<3:28:52, 10.06s/it]
21%|██ | 325/1570 [25:19<2:43:46, 7.89s/it]
21%|██ | 326/1570 [25:22<2:12:02, 6.37s/it]
21%|██ | 327/1570 [25:25<1:50:05, 5.31s/it]
21%|██ | 328/1570 [25:28<1:34:34, 4.57s/it]
21%|██ | 329/1570 [25:31<1:27:49, 4.25s/it]
21%|██ | 330/1570 [25:35<1:23:44, 4.05s/it]
21%|██ | 331/1570 [25:38<1:20:41, 3.91s/it]
21%|██ | 332/1570 [25:42<1:18:40, 3.81s/it]
21%|██ | 333/1570 [25:45<1:16:22, 3.70s/it]
21%|██▏ | 334/1570 [25:49<1:15:42, 3.68s/it]
21%|██▏ | 335/1570 [25:53<1:15:09, 3.65s/it]
21%|██▏ | 336/1570 [25:56<1:14:24, 3.62s/it]
21%|██▏ | 337/1570 [26:00<1:13:38, 3.58s/it]
22%|██▏ | 338/1570 [26:03<1:12:49, 3.55s/it]
22%|██▏ | 339/1570 [26:07<1:12:53, 3.55s/it]
22%|██▏ | 340/1570 [26:10<1:12:12, 3.52s/it]
{'loss': '0.009684', 'grad_norm': '1.29', 'learning_rate': '8.894e-05', 'neg_student_nll_loss': '0.009724', 'ce_loss': '0.009724', 'hpd_loss': '0.009684', 'sampled_diff_pct': '0.4521', 'adv1_neg_pct': '1.147', 'adv2_active_pct': '0.4477', 'mask3_pct': '0.4477', 'k1_gt_raw_mean': '0.003205', 'k1_gt_raw_min': '-0.009925', 'k1_gt_raw_max': '0.1098', 'k1_gt_raw_neg_pct': '1.147', 'k1_sample_mean': '-0.8627', 'k1_sample_min': '-0.9573', 'k1_sample_max': '-0.7682', 'k1_sample_neg_pct': '28.96', 'epoch': '1.083'}
+
22%|██▏ | 340/1570 [26:10<1:12:12, 3.52s/it]
22%|██▏ | 341/1570 [26:14<1:12:28, 3.54s/it]
22%|██▏ | 342/1570 [26:17<1:12:13, 3.53s/it]
22%|██▏ | 343/1570 [26:21<1:12:11, 3.53s/it]
22%|██▏ | 344/1570 [26:24<1:11:54, 3.52s/it]
22%|██▏ | 345/1570 [26:28<1:12:14, 3.54s/it]
22%|██▏ | 346/1570 [26:31<1:12:00, 3.53s/it]
22%|██▏ | 347/1570 [26:35<1:10:43, 3.47s/it]
22%|██▏ | 348/1570 [26:38<1:09:59, 3.44s/it]
22%|██▏ | 349/1570 [26:41<1:10:19, 3.46s/it]
22%|██▏ | 350/1570 [26:45<1:10:10, 3.45s/it]
22%|██▏ | 351/1570 [26:48<1:10:06, 3.45s/it]
22%|██▏ | 352/1570 [26:52<1:09:27, 3.42s/it]
22%|██▏ | 353/1570 [26:55<1:10:15, 3.46s/it]
23%|██▎ | 354/1570 [26:59<1:10:28, 3.48s/it]
23%|██▎ | 355/1570 [27:02<1:10:37, 3.49s/it]
23%|██▎ | 356/1570 [27:06<1:10:46, 3.50s/it]
23%|██▎ | 357/1570 [27:09<1:11:13, 3.52s/it]
23%|██▎ | 358/1570 [27:13<1:10:52, 3.51s/it]
23%|██▎ | 359/1570 [27:16<1:10:55, 3.51s/it]
23%|██▎ | 360/1570 [27:20<1:10:26, 3.49s/it]
{'loss': '0.0125', 'grad_norm': '0.819', 'learning_rate': '8.766e-05', 'neg_student_nll_loss': '0.01364', 'ce_loss': '0.01364', 'hpd_loss': '0.0125', 'sampled_diff_pct': '0.7438', 'adv1_neg_pct': '1.657', 'adv2_active_pct': '0.7416', 'mask3_pct': '0.7416', 'k1_gt_raw_mean': '0.003038', 'k1_gt_raw_min': '-0.01136', 'k1_gt_raw_max': '0.1126', 'k1_gt_raw_neg_pct': '1.657', 'k1_sample_mean': '-1.38', 'k1_sample_min': '-1.649', 'k1_sample_max': '-1.135', 'k1_sample_neg_pct': '30', 'epoch': '1.147'}
+
23%|██▎ | 360/1570 [27:20<1:10:26, 3.49s/it]
23%|██▎ | 361/1570 [27:23<1:09:57, 3.47s/it]
23%|██▎ | 362/1570 [27:27<1:08:39, 3.41s/it]
23%|██▎ | 363/1570 [27:30<1:09:36, 3.46s/it]
23%|██▎ | 364/1570 [27:34<1:09:28, 3.46s/it]
23%|██▎ | 365/1570 [27:37<1:09:23, 3.45s/it]
23%|██▎ | 366/1570 [27:41<1:09:48, 3.48s/it]
23%|██▎ | 367/1570 [27:44<1:10:01, 3.49s/it]
23%|██▎ | 368/1570 [27:48<1:09:56, 3.49s/it]
24%|██▎ | 369/1570 [27:51<1:10:23, 3.52s/it]
24%|██▎ | 370/1570 [27:55<1:10:48, 3.54s/it]
24%|██▎ | 371/1570 [27:58<1:10:58, 3.55s/it]
24%|██▎ | 372/1570 [28:02<1:10:54, 3.55s/it]
24%|██▍ | 373/1570 [28:05<1:10:43, 3.55s/it]
24%|██▍ | 374/1570 [28:09<1:10:44, 3.55s/it]
24%|██▍ | 375/1570 [28:12<1:09:19, 3.48s/it]
24%|██▍ | 376/1570 [28:16<1:09:56, 3.51s/it]
24%|██▍ | 377/1570 [28:19<1:10:07, 3.53s/it]
24%|██▍ | 378/1570 [28:23<1:10:37, 3.56s/it]
24%|██▍ | 379/1570 [28:27<1:10:40, 3.56s/it]
24%|██▍ | 380/1570 [28:30<1:09:55, 3.53s/it]
{'loss': '0.008744', 'grad_norm': '0.5893', 'learning_rate': '8.631e-05', 'neg_student_nll_loss': '0.007902', 'ce_loss': '0.007902', 'hpd_loss': '0.008744', 'sampled_diff_pct': '0.3568', 'adv1_neg_pct': '1.351', 'adv2_active_pct': '0.3445', 'mask3_pct': '0.3445', 'k1_gt_raw_mean': '0.002256', 'k1_gt_raw_min': '-0.006403', 'k1_gt_raw_max': '0.0942', 'k1_gt_raw_neg_pct': '1.351', 'k1_sample_mean': '-0.7342', 'k1_sample_min': '-0.8995', 'k1_sample_max': '-0.5839', 'k1_sample_neg_pct': '23.75', 'epoch': '1.211'}
+
24%|██▍ | 380/1570 [28:30<1:09:55, 3.53s/it]
24%|██▍ | 381/1570 [28:34<1:10:01, 3.53s/it]
24%|██▍ | 382/1570 [28:37<1:09:52, 3.53s/it]
24%|██▍ | 383/1570 [28:41<1:10:03, 3.54s/it]
24%|██▍ | 384/1570 [28:44<1:09:36, 3.52s/it]
25%|██▍ | 385/1570 [28:48<1:09:45, 3.53s/it]
25%|██▍ | 386/1570 [28:51<1:09:54, 3.54s/it]
25%|██▍ | 387/1570 [28:55<1:08:56, 3.50s/it]
25%|██▍ | 388/1570 [28:58<1:09:22, 3.52s/it]
25%|██▍ | 389/1570 [29:02<1:09:29, 3.53s/it]
25%|██▍ | 390/1570 [29:05<1:09:00, 3.51s/it]
25%|██▍ | 391/1570 [29:09<1:09:23, 3.53s/it]
25%|██▍ | 392/1570 [29:12<1:09:43, 3.55s/it]
25%|██▌ | 393/1570 [29:16<1:09:52, 3.56s/it]
25%|██▌ | 394/1570 [29:19<1:08:42, 3.51s/it]
25%|██▌ | 395/1570 [29:23<1:09:03, 3.53s/it]
25%|██▌ | 396/1570 [29:27<1:09:27, 3.55s/it]
25%|██▌ | 397/1570 [29:30<1:09:14, 3.54s/it]
25%|██▌ | 398/1570 [29:34<1:08:25, 3.50s/it]
25%|██▌ | 399/1570 [29:37<1:05:56, 3.38s/it]
25%|██▌ | 400/1570 [29:39<1:02:44, 3.22s/it]
{'loss': '0.00669', 'grad_norm': '1.293', 'learning_rate': '8.491e-05', 'neg_student_nll_loss': '0.008193', 'ce_loss': '0.008193', 'hpd_loss': '0.00669', 'sampled_diff_pct': '0.345', 'adv1_neg_pct': '1.414', 'adv2_active_pct': '0.3374', 'mask3_pct': '0.3345', 'k1_gt_raw_mean': '0.002954', 'k1_gt_raw_min': '-0.006222', 'k1_gt_raw_max': '0.1067', 'k1_gt_raw_neg_pct': '1.414', 'k1_sample_mean': '-0.7585', 'k1_sample_min': '-0.8608', 'k1_sample_max': '-0.6661', 'k1_sample_neg_pct': '25.76', 'epoch': '1.274'}
+
25%|██▌ | 400/1570 [29:39<1:02:44, 3.22s/it]
26%|██▌ | 401/1570 [29:42<1:00:50, 3.12s/it]
26%|██▌ | 402/1570 [29:45<59:02, 3.03s/it]
26%|██▌ | 403/1570 [29:48<57:57, 2.98s/it]
26%|██▌ | 404/1570 [29:51<57:03, 2.94s/it]
26%|██▌ | 405/1570 [29:54<56:23, 2.90s/it]
26%|██▌ | 406/1570 [29:57<56:07, 2.89s/it]
26%|██▌ | 407/1570 [29:59<55:51, 2.88s/it]
26%|██▌ | 408/1570 [30:02<55:19, 2.86s/it]
26%|██▌ | 409/1570 [30:05<55:25, 2.86s/it]
26%|██▌ | 410/1570 [30:08<55:22, 2.86s/it]
26%|██▌ | 411/1570 [30:11<55:10, 2.86s/it]
26%|██▌ | 412/1570 [30:14<55:05, 2.85s/it]
26%|██▋ | 413/1570 [30:16<54:49, 2.84s/it]
26%|██▋ | 414/1570 [30:19<55:15, 2.87s/it]
26%|██▋ | 415/1570 [30:22<54:59, 2.86s/it]
26%|██▋ | 416/1570 [30:25<54:52, 2.85s/it]
27%|██▋ | 417/1570 [30:28<54:47, 2.85s/it]
27%|██▋ | 418/1570 [30:31<54:35, 2.84s/it]
27%|██▋ | 419/1570 [30:34<54:17, 2.83s/it]
27%|██▋ | 420/1570 [30:36<54:28, 2.84s/it]
{'loss': '0.01081', 'grad_norm': '0.8164', 'learning_rate': '8.345e-05', 'neg_student_nll_loss': '0.01111', 'ce_loss': '0.01111', 'hpd_loss': '0.01081', 'sampled_diff_pct': '0.4343', 'adv1_neg_pct': '2.122', 'adv2_active_pct': '0.4313', 'mask3_pct': '0.4313', 'k1_gt_raw_mean': '0.003569', 'k1_gt_raw_min': '-0.00447', 'k1_gt_raw_max': '0.1057', 'k1_gt_raw_neg_pct': '2.122', 'k1_sample_mean': '-0.7544', 'k1_sample_min': '-0.827', 'k1_sample_max': '-0.6987', 'k1_sample_neg_pct': '25.28', 'epoch': '1.338'}
+
27%|██▋ | 420/1570 [30:36<54:28, 2.84s/it]
27%|██▋ | 421/1570 [30:39<54:38, 2.85s/it]
27%|██▋ | 422/1570 [30:42<54:15, 2.84s/it]
27%|██▋ | 423/1570 [30:45<54:10, 2.83s/it]
27%|██▋ | 424/1570 [30:48<54:10, 2.84s/it]
27%|██▋ | 425/1570 [30:51<54:16, 2.84s/it]
27%|██▋ | 426/1570 [30:53<53:53, 2.83s/it]
27%|██▋ | 427/1570 [30:56<54:06, 2.84s/it]
27%|██▋ | 428/1570 [30:59<53:48, 2.83s/it]
27%|██▋ | 429/1570 [31:02<53:46, 2.83s/it]
27%|██▋ | 430/1570 [31:05<53:57, 2.84s/it]
27%|██▋ | 431/1570 [31:08<53:48, 2.83s/it]
28%|██▊ | 432/1570 [31:10<54:02, 2.85s/it]
28%|██▊ | 433/1570 [31:13<54:06, 2.86s/it]
28%|██▊ | 434/1570 [31:16<53:50, 2.84s/it]
28%|██▊ | 435/1570 [31:19<53:59, 2.85s/it]
28%|██▊ | 436/1570 [31:22<53:48, 2.85s/it]
28%|██▊ | 437/1570 [31:25<54:01, 2.86s/it]
28%|██▊ | 438/1570 [31:28<53:49, 2.85s/it]
28%|██▊ | 439/1570 [31:30<53:42, 2.85s/it]
28%|██▊ | 440/1570 [31:33<53:44, 2.85s/it]
{'loss': '0.005844', 'grad_norm': '0.9594', 'learning_rate': '8.194e-05', 'neg_student_nll_loss': '0.007173', 'ce_loss': '0.007173', 'hpd_loss': '0.005844', 'sampled_diff_pct': '0.3067', 'adv1_neg_pct': '2', 'adv2_active_pct': '0.2968', 'mask3_pct': '0.2968', 'k1_gt_raw_mean': '0.00214', 'k1_gt_raw_min': '-0.005929', 'k1_gt_raw_max': '0.08694', 'k1_gt_raw_neg_pct': '2', 'k1_sample_mean': '-0.898', 'k1_sample_min': '-0.9926', 'k1_sample_max': '-0.8035', 'k1_sample_neg_pct': '20.78', 'epoch': '1.402'}
+
28%|██▊ | 440/1570 [31:33<53:44, 2.85s/it]
28%|██▊ | 441/1570 [31:36<53:28, 2.84s/it]
28%|██▊ | 442/1570 [31:39<53:21, 2.84s/it]
28%|██▊ | 443/1570 [31:42<53:05, 2.83s/it]
28%|██▊ | 444/1570 [31:45<53:00, 2.82s/it]
28%|██▊ | 445/1570 [31:47<52:53, 2.82s/it]
28%|██▊ | 446/1570 [31:50<52:55, 2.83s/it]
28%|██▊ | 447/1570 [31:53<52:54, 2.83s/it]
29%|██▊ | 448/1570 [31:56<52:51, 2.83s/it]
29%|██▊ | 449/1570 [31:59<53:08, 2.84s/it]
29%|██▊ | 450/1570 [32:02<53:05, 2.84s/it]
29%|██▊ | 451/1570 [32:04<53:12, 2.85s/it]
29%|██▉ | 452/1570 [32:07<53:05, 2.85s/it]
29%|██▉ | 453/1570 [32:10<52:56, 2.84s/it]
29%|██▉ | 454/1570 [32:13<52:55, 2.85s/it]
29%|██▉ | 455/1570 [32:16<53:01, 2.85s/it]
29%|██▉ | 456/1570 [32:19<53:07, 2.86s/it]
29%|██▉ | 457/1570 [32:22<52:54, 2.85s/it]
29%|██▉ | 458/1570 [32:24<52:54, 2.86s/it]
29%|██▉ | 459/1570 [32:27<52:50, 2.85s/it]
29%|██▉ | 460/1570 [32:30<52:17, 2.83s/it]
{'loss': '0.009073', 'grad_norm': '1.359', 'learning_rate': '8.037e-05', 'neg_student_nll_loss': '0.007787', 'ce_loss': '0.007787', 'hpd_loss': '0.009073', 'sampled_diff_pct': '0.2836', 'adv1_neg_pct': '1.945', 'adv2_active_pct': '0.2836', 'mask3_pct': '0.2791', 'k1_gt_raw_mean': '0.001622', 'k1_gt_raw_min': '-0.002223', 'k1_gt_raw_max': '0.07425', 'k1_gt_raw_neg_pct': '1.945', 'k1_sample_mean': '-0.6861', 'k1_sample_min': '-0.73', 'k1_sample_max': '-0.6454', 'k1_sample_neg_pct': '17.19', 'epoch': '1.466'}
+
29%|██▉ | 460/1570 [32:30<52:17, 2.83s/it]
29%|██▉ | 461/1570 [32:33<52:11, 2.82s/it]
29%|██▉ | 462/1570 [32:36<56:19, 3.05s/it]
29%|██▉ | 463/1570 [32:40<59:06, 3.20s/it]
30%|██▉ | 464/1570 [32:43<1:00:27, 3.28s/it]
30%|██▉ | 465/1570 [32:47<1:01:44, 3.35s/it]
30%|██▉ | 466/1570 [32:51<1:02:39, 3.41s/it]
30%|██▉ | 467/1570 [32:54<1:03:27, 3.45s/it]
30%|██▉ | 468/1570 [32:58<1:04:03, 3.49s/it]
30%|██▉ | 469/1570 [33:01<1:03:08, 3.44s/it]
30%|██▉ | 470/1570 [33:05<1:03:51, 3.48s/it]
30%|███ | 471/1570 [33:08<1:04:02, 3.50s/it]
30%|███ | 472/1570 [33:12<1:04:07, 3.50s/it]
30%|███ | 473/1570 [33:15<1:03:56, 3.50s/it]
30%|███ | 474/1570 [33:19<1:03:54, 3.50s/it]
30%|███ | 475/1570 [33:22<1:03:59, 3.51s/it]
30%|███ | 476/1570 [33:26<1:03:46, 3.50s/it]
30%|███ | 477/1570 [33:29<1:03:59, 3.51s/it]
30%|███ | 478/1570 [33:33<1:04:05, 3.52s/it]
31%|███ | 479/1570 [33:36<1:04:12, 3.53s/it]
31%|███ | 480/1570 [33:40<1:04:31, 3.55s/it]
{'loss': '0.001571', 'grad_norm': '0.2024', 'learning_rate': '7.876e-05', 'neg_student_nll_loss': '0.005176', 'ce_loss': '0.005176', 'hpd_loss': '0.001571', 'sampled_diff_pct': '0.4208', 'adv1_neg_pct': '1.84', 'adv2_active_pct': '0.4187', 'mask3_pct': '0.4187', 'k1_gt_raw_mean': '0.002221', 'k1_gt_raw_min': '-0.008055', 'k1_gt_raw_max': '0.07685', 'k1_gt_raw_neg_pct': '1.84', 'k1_sample_mean': '-0.4974', 'k1_sample_min': '-0.5228', 'k1_sample_max': '-0.472', 'k1_sample_neg_pct': '17.5', 'epoch': '1.53'}
+
31%|███ | 480/1570 [33:40<1:04:31, 3.55s/it]
31%|███ | 481/1570 [33:43<1:03:54, 3.52s/it]
31%|███ | 482/1570 [33:47<1:04:16, 3.54s/it]
31%|███ | 483/1570 [33:50<1:04:11, 3.54s/it]
31%|███ | 484/1570 [33:54<1:03:39, 3.52s/it]
31%|███ | 485/1570 [33:57<1:04:01, 3.54s/it]
31%|███ | 486/1570 [34:01<1:03:06, 3.49s/it]
31%|███ | 487/1570 [34:04<1:02:55, 3.49s/it]
31%|███ | 488/1570 [34:08<1:02:36, 3.47s/it]
31%|███ | 489/1570 [34:11<1:01:50, 3.43s/it]
31%|███ | 490/1570 [34:14<58:49, 3.27s/it]
31%|███▏ | 491/1570 [34:17<56:40, 3.15s/it]
31%|███▏ | 492/1570 [34:20<55:26, 3.09s/it]
31%|███▏ | 493/1570 [34:23<54:17, 3.02s/it]
31%|███▏ | 494/1570 [34:26<53:10, 2.97s/it]
32%|███▏ | 495/1570 [34:28<52:27, 2.93s/it]
32%|███▏ | 496/1570 [34:31<51:52, 2.90s/it]
32%|███▏ | 497/1570 [34:34<51:36, 2.89s/it]
32%|███▏ | 498/1570 [34:37<51:30, 2.88s/it]
32%|███▏ | 499/1570 [34:40<51:23, 2.88s/it]
32%|███▏ | 500/1570 [34:43<51:06, 2.87s/it]
{'loss': '0.003286', 'grad_norm': '0.03088', 'learning_rate': '7.71e-05', 'neg_student_nll_loss': '0.005859', 'ce_loss': '0.005859', 'hpd_loss': '0.003286', 'sampled_diff_pct': '0.3419', 'adv1_neg_pct': '2.532', 'adv2_active_pct': '0.3381', 'mask3_pct': '0.3381', 'k1_gt_raw_mean': '0.002065', 'k1_gt_raw_min': '-0.003', 'k1_gt_raw_max': '0.05826', 'k1_gt_raw_neg_pct': '2.532', 'k1_sample_mean': '-0.3973', 'k1_sample_min': '-0.4579', 'k1_sample_max': '-0.3367', 'k1_sample_neg_pct': '14.06', 'epoch': '1.594'}
+
32%|███▏ | 500/1570 [34:43<51:06, 2.87s/it]
32%|███▏ | 501/1570 [34:45<51:00, 2.86s/it]
32%|███▏ | 502/1570 [34:48<51:07, 2.87s/it]
32%|███▏ | 503/1570 [34:51<51:11, 2.88s/it]
32%|███▏ | 504/1570 [34:54<51:11, 2.88s/it]
32%|███▏ | 505/1570 [34:57<51:35, 2.91s/it]
32%|███▏ | 506/1570 [35:00<51:00, 2.88s/it]
32%|███▏ | 507/1570 [35:03<51:44, 2.92s/it]
32%|███▏ | 508/1570 [35:06<51:20, 2.90s/it]
32%|███▏ | 509/1570 [35:09<50:54, 2.88s/it]
32%|███▏ | 510/1570 [35:12<52:20, 2.96s/it]
33%|███▎ | 511/1570 [35:15<55:44, 3.16s/it]
33%|███▎ | 512/1570 [35:19<58:07, 3.30s/it]
33%|███▎ | 513/1570 [35:23<59:46, 3.39s/it]
33%|███▎ | 514/1570 [35:26<1:00:28, 3.44s/it]
33%|███▎ | 515/1570 [35:30<1:01:09, 3.48s/it]
33%|███▎ | 516/1570 [35:33<1:01:48, 3.52s/it]
33%|███▎ | 517/1570 [35:37<1:02:00, 3.53s/it]
33%|███▎ | 518/1570 [35:41<1:02:23, 3.56s/it]
33%|███▎ | 519/1570 [35:44<1:02:03, 3.54s/it]
33%|███▎ | 520/1570 [35:48<1:02:26, 3.57s/it]
{'loss': '0.01286', 'grad_norm': '0.1429', 'learning_rate': '7.54e-05', 'neg_student_nll_loss': '0.01001', 'ce_loss': '0.01001', 'hpd_loss': '0.01286', 'sampled_diff_pct': '0.371', 'adv1_neg_pct': '2.702', 'adv2_active_pct': '0.3644', 'mask3_pct': '0.3644', 'k1_gt_raw_mean': '0.001742', 'k1_gt_raw_min': '-0.004031', 'k1_gt_raw_max': '0.06845', 'k1_gt_raw_neg_pct': '2.702', 'k1_sample_mean': '-0.5421', 'k1_sample_min': '-0.6135', 'k1_sample_max': '-0.4803', 'k1_sample_neg_pct': '17.03', 'epoch': '1.657'}
+
33%|███▎ | 520/1570 [35:48<1:02:26, 3.57s/it]
33%|███▎ | 521/1570 [35:51<1:02:34, 3.58s/it]
33%|███▎ | 522/1570 [35:55<1:02:45, 3.59s/it]
33%|███▎ | 523/1570 [35:58<1:02:30, 3.58s/it]
33%|███▎ | 524/1570 [36:02<1:02:17, 3.57s/it]
33%|███▎ | 525/1570 [36:06<1:02:31, 3.59s/it]
34%|███▎ | 526/1570 [36:09<1:02:33, 3.60s/it]
34%|███▎ | 527/1570 [36:12<58:35, 3.37s/it]
34%|███▎ | 528/1570 [36:15<55:40, 3.21s/it]
34%|███▎ | 529/1570 [36:18<53:38, 3.09s/it]
34%|███▍ | 530/1570 [36:21<52:08, 3.01s/it]
34%|███▍ | 531/1570 [36:23<51:20, 2.96s/it]
34%|███▍ | 532/1570 [36:26<50:31, 2.92s/it]
34%|███▍ | 533/1570 [36:29<50:07, 2.90s/it]
34%|███▍ | 534/1570 [36:32<49:55, 2.89s/it]
34%|███▍ | 535/1570 [36:35<49:29, 2.87s/it]
34%|███▍ | 536/1570 [36:38<49:15, 2.86s/it]
34%|███▍ | 537/1570 [36:40<48:51, 2.84s/it]
34%|███▍ | 538/1570 [36:43<48:50, 2.84s/it]
34%|███▍ | 539/1570 [36:46<48:35, 2.83s/it]
34%|███▍ | 540/1570 [36:49<48:35, 2.83s/it]
{'loss': '0.009246', 'grad_norm': '0.6681', 'learning_rate': '7.366e-05', 'neg_student_nll_loss': '0.01003', 'ce_loss': '0.01003', 'hpd_loss': '0.009246', 'sampled_diff_pct': '0.521', 'adv1_neg_pct': '2.249', 'adv2_active_pct': '0.5109', 'mask3_pct': '0.5109', 'k1_gt_raw_mean': '0.002854', 'k1_gt_raw_min': '-0.00446', 'k1_gt_raw_max': '0.08858', 'k1_gt_raw_neg_pct': '2.249', 'k1_sample_mean': '-0.8169', 'k1_sample_min': '-0.8906', 'k1_sample_max': '-0.7409', 'k1_sample_neg_pct': '22.66', 'epoch': '1.721'}
+
34%|███▍ | 540/1570 [36:49<48:35, 2.83s/it]
34%|███▍ | 541/1570 [36:52<48:42, 2.84s/it]
35%|███▍ | 542/1570 [36:55<48:39, 2.84s/it]
35%|███▍ | 543/1570 [36:57<48:32, 2.84s/it]
35%|███▍ | 544/1570 [37:00<48:26, 2.83s/it]
35%|███▍ | 545/1570 [37:03<48:28, 2.84s/it]
35%|███▍ | 546/1570 [37:06<48:24, 2.84s/it]
35%|███▍ | 547/1570 [37:09<48:20, 2.83s/it]
35%|███▍ | 548/1570 [37:12<48:06, 2.82s/it]
35%|███▍ | 549/1570 [37:14<47:45, 2.81s/it]
35%|███▌ | 550/1570 [37:17<47:45, 2.81s/it]
35%|███▌ | 551/1570 [37:20<48:07, 2.83s/it]
35%|███▌ | 552/1570 [37:23<48:03, 2.83s/it]
35%|███▌ | 553/1570 [37:26<47:53, 2.83s/it]
35%|███▌ | 554/1570 [37:29<47:56, 2.83s/it]
35%|███▌ | 555/1570 [37:31<47:58, 2.84s/it]
35%|███▌ | 556/1570 [37:34<47:57, 2.84s/it]
35%|███▌ | 557/1570 [37:37<47:48, 2.83s/it]
36%|███▌ | 558/1570 [37:40<47:45, 2.83s/it]
36%|███▌ | 559/1570 [37:43<47:37, 2.83s/it]
36%|███▌ | 560/1570 [37:45<47:28, 2.82s/it]
{'loss': '0.006253', 'grad_norm': '0.08244', 'learning_rate': '7.188e-05', 'neg_student_nll_loss': '0.005005', 'ce_loss': '0.005005', 'hpd_loss': '0.006253', 'sampled_diff_pct': '0.2273', 'adv1_neg_pct': '2.464', 'adv2_active_pct': '0.2212', 'mask3_pct': '0.2212', 'k1_gt_raw_mean': '0.001176', 'k1_gt_raw_min': '-0.005645', 'k1_gt_raw_max': '0.05803', 'k1_gt_raw_neg_pct': '2.464', 'k1_sample_mean': '-0.2807', 'k1_sample_min': '-0.3224', 'k1_sample_max': '-0.2404', 'k1_sample_neg_pct': '14.92', 'epoch': '1.785'}
+
36%|███▌ | 560/1570 [37:45<47:28, 2.82s/it]
36%|███▌ | 561/1570 [37:48<47:31, 2.83s/it]
36%|███▌ | 562/1570 [37:51<47:23, 2.82s/it]
36%|███▌ | 563/1570 [37:54<47:17, 2.82s/it]
36%|███▌ | 564/1570 [37:57<47:21, 2.82s/it]
36%|███▌ | 565/1570 [38:00<47:46, 2.85s/it]
36%|███▌ | 566/1570 [38:03<47:38, 2.85s/it]
36%|███▌ | 567/1570 [38:05<47:24, 2.84s/it]
36%|███▌ | 568/1570 [38:08<47:19, 2.83s/it]
36%|███▌ | 569/1570 [38:11<47:17, 2.83s/it]
36%|███▋ | 570/1570 [38:14<47:13, 2.83s/it]
36%|███▋ | 571/1570 [38:17<47:00, 2.82s/it]
36%|███▋ | 572/1570 [38:19<46:51, 2.82s/it]
36%|███▋ | 573/1570 [38:22<47:11, 2.84s/it]
37%|███▋ | 574/1570 [38:25<47:01, 2.83s/it]
37%|███▋ | 575/1570 [38:28<47:03, 2.84s/it]
37%|███▋ | 576/1570 [38:31<46:41, 2.82s/it]
37%|███▋ | 577/1570 [38:34<46:52, 2.83s/it]
37%|███▋ | 578/1570 [38:36<46:44, 2.83s/it]
37%|███▋ | 579/1570 [38:39<46:46, 2.83s/it]
37%|███▋ | 580/1570 [38:42<47:05, 2.85s/it]
{'loss': '0.009691', 'grad_norm': '0.08691', 'learning_rate': '7.006e-05', 'neg_student_nll_loss': '0.00684', 'ce_loss': '0.00684', 'hpd_loss': '0.009691', 'sampled_diff_pct': '0.2427', 'adv1_neg_pct': '3.338', 'adv2_active_pct': '0.2326', 'mask3_pct': '0.2326', 'k1_gt_raw_mean': '0.001634', 'k1_gt_raw_min': '-0.009443', 'k1_gt_raw_max': '0.05798', 'k1_gt_raw_neg_pct': '3.338', 'k1_sample_mean': '-0.3186', 'k1_sample_min': '-0.3385', 'k1_sample_max': '-0.2987', 'k1_sample_neg_pct': '10.62', 'epoch': '1.849'}
+
37%|███▋ | 580/1570 [38:42<47:05, 2.85s/it]
37%|███▋ | 581/1570 [38:45<46:51, 2.84s/it]
37%|███▋ | 582/1570 [38:48<46:46, 2.84s/it]
37%|███▋ | 583/1570 [38:51<46:45, 2.84s/it]
37%|███▋ | 584/1570 [38:54<46:53, 2.85s/it]
37%|███▋ | 585/1570 [38:56<46:57, 2.86s/it]
37%|███▋ | 586/1570 [38:59<46:48, 2.85s/it]
37%|███▋ | 587/1570 [39:02<46:29, 2.84s/it]
37%|███▋ | 588/1570 [39:05<46:07, 2.82s/it]
38%|███▊ | 589/1570 [39:08<46:17, 2.83s/it]
38%|███▊ | 590/1570 [39:11<46:22, 2.84s/it]
38%|███▊ | 591/1570 [39:13<46:27, 2.85s/it]
38%|███▊ | 592/1570 [39:16<46:29, 2.85s/it]
38%|███▊ | 593/1570 [39:19<46:16, 2.84s/it]
38%|███▊ | 594/1570 [39:22<46:09, 2.84s/it]
38%|███▊ | 595/1570 [39:25<46:01, 2.83s/it]
38%|███▊ | 596/1570 [39:28<46:16, 2.85s/it]
38%|███▊ | 597/1570 [39:31<46:20, 2.86s/it]
38%|███▊ | 598/1570 [39:33<46:16, 2.86s/it]
38%|███▊ | 599/1570 [39:36<46:17, 2.86s/it]
38%|███▊ | 600/1570 [39:39<46:00, 2.85s/it]
{'loss': '0.0005976', 'grad_norm': '0.2512', 'learning_rate': '6.821e-05', 'neg_student_nll_loss': '0.002632', 'ce_loss': '0.002632', 'hpd_loss': '0.0005976', 'sampled_diff_pct': '0.2187', 'adv1_neg_pct': '4.041', 'adv2_active_pct': '0.208', 'mask3_pct': '0.208', 'k1_gt_raw_mean': '0.001406', 'k1_gt_raw_min': '-0.002837', 'k1_gt_raw_max': '0.04939', 'k1_gt_raw_neg_pct': '4.041', 'k1_sample_mean': '-0.2559', 'k1_sample_min': '-0.2803', 'k1_sample_max': '-0.2366', 'k1_sample_neg_pct': '10.21', 'epoch': '1.913'}
+
38%|███▊ | 600/1570 [39:39<46:00, 2.85s/it]
38%|███▊ | 601/1570 [39:42<45:55, 2.84s/it]
38%|███▊ | 602/1570 [39:45<45:45, 2.84s/it]
38%|███▊ | 603/1570 [39:48<45:48, 2.84s/it]
38%|███▊ | 604/1570 [39:50<45:51, 2.85s/it]
39%|███▊ | 605/1570 [39:53<45:44, 2.84s/it]
39%|███▊ | 606/1570 [39:56<45:53, 2.86s/it]
39%|███▊ | 607/1570 [39:59<45:50, 2.86s/it]
39%|███▊ | 608/1570 [40:02<45:42, 2.85s/it]
39%|███▉ | 609/1570 [40:05<45:23, 2.83s/it]
39%|███▉ | 610/1570 [40:07<45:17, 2.83s/it]
39%|███▉ | 611/1570 [40:10<45:23, 2.84s/it]
39%|███▉ | 612/1570 [40:13<45:26, 2.85s/it]
39%|███▉ | 613/1570 [40:16<45:07, 2.83s/it]
39%|███▉ | 614/1570 [40:19<45:00, 2.82s/it]
39%|███▉ | 615/1570 [40:22<44:53, 2.82s/it]
39%|███▉ | 616/1570 [40:24<45:00, 2.83s/it]
39%|███▉ | 617/1570 [40:27<44:59, 2.83s/it]
39%|███▉ | 618/1570 [40:30<44:50, 2.83s/it]
39%|███▉ | 619/1570 [40:33<45:02, 2.84s/it]
39%|███▉ | 620/1570 [40:36<45:01, 2.84s/it]
{'loss': '0.00215', 'grad_norm': '0.3381', 'learning_rate': '6.634e-05', 'neg_student_nll_loss': '0.003203', 'ce_loss': '0.003203', 'hpd_loss': '0.00215', 'sampled_diff_pct': '0.1893', 'adv1_neg_pct': '4.098', 'adv2_active_pct': '0.1849', 'mask3_pct': '0.1849', 'k1_gt_raw_mean': '0.000772', 'k1_gt_raw_min': '-0.003699', 'k1_gt_raw_max': '0.04364', 'k1_gt_raw_neg_pct': '4.098', 'k1_sample_mean': '-0.2737', 'k1_sample_min': '-0.3079', 'k1_sample_max': '-0.2396', 'k1_sample_neg_pct': '13.44', 'epoch': '1.976'}
+
39%|███▉ | 620/1570 [40:36<45:01, 2.84s/it]
40%|███▉ | 621/1570 [40:39<45:02, 2.85s/it]
40%|███▉ | 622/1570 [40:41<44:40, 2.83s/it]
40%|███▉ | 623/1570 [40:44<44:41, 2.83s/it]
40%|███▉ | 624/1570 [40:47<44:48, 2.84s/it]
40%|███▉ | 625/1570 [40:50<44:49, 2.85s/it]
40%|███▉ | 626/1570 [40:53<44:36, 2.83s/it]
40%|███▉ | 627/1570 [40:56<44:31, 2.83s/it]
40%|████ | 628/1570 [40:57<36:30, 2.33s/it][INFO|trainer.py:2668] 2026-08-22 22:39:32,198 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 22:39:32,199 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 22:39:32,199 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/hpd/checkpoint-628
+[INFO|configuration_utils.py:780] 2026-08-22 22:49:29,538 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 22:49:29,540 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 22:49:29,575 >> chat template saved in results/qwen3/hpd/checkpoint-628/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 22:49:29,576 >> tokenizer config file saved in results/qwen3/hpd/checkpoint-628/tokenizer_config.json
+
40%|████ | 629/1570 [50:58<47:34:08, 181.99s/it]
40%|████ | 630/1570 [51:01<33:29:07, 128.24s/it]
40%|████ | 631/1570 [51:04<23:38:07, 90.62s/it]
40%|████ | 632/1570 [51:06<16:44:50, 64.28s/it]
40%|████ | 633/1570 [51:09<11:55:59, 45.85s/it]
40%|████ | 634/1570 [51:12<8:33:59, 32.95s/it]
40%|████ | 635/1570 [51:15<6:13:01, 23.94s/it]
41%|████ | 636/1570 [51:18<4:34:07, 17.61s/it]
41%|████ | 637/1570 [51:21<3:24:55, 13.18s/it]
41%|████ | 638/1570 [51:24<2:36:33, 10.08s/it]
41%|████ | 639/1570 [51:26<2:02:29, 7.89s/it]
41%|████ | 640/1570 [51:29<1:39:01, 6.39s/it]
{'loss': '0.002021', 'grad_norm': '0.6891', 'learning_rate': '6.444e-05', 'neg_student_nll_loss': '0.004337', 'ce_loss': '0.004337', 'hpd_loss': '0.00261', 'sampled_diff_pct': '0.1496', 'adv1_neg_pct': '3.476', 'adv2_active_pct': '0.1299', 'mask3_pct': '0.1299', 'k1_gt_raw_mean': '0.0007897', 'k1_gt_raw_min': '-0.005202', 'k1_gt_raw_max': '0.03807', 'k1_gt_raw_neg_pct': '3.476', 'k1_sample_mean': '-0.2449', 'k1_sample_min': '-0.2522', 'k1_sample_max': '-0.2407', 'k1_sample_neg_pct': '9.798', 'epoch': '2.038'}
+
41%|████ | 640/1570 [51:29<1:39:01, 6.39s/it]
41%|████ | 641/1570 [51:32<1:22:25, 5.32s/it]
41%|████ | 642/1570 [51:35<1:10:51, 4.58s/it]
41%|████ | 643/1570 [51:38<1:02:45, 4.06s/it]
41%|████ | 644/1570 [51:41<56:58, 3.69s/it]
41%|████ | 645/1570 [51:44<53:04, 3.44s/it]
41%|████ | 646/1570 [51:46<50:16, 3.27s/it]
41%|████ | 647/1570 [51:49<48:19, 3.14s/it]
41%|████▏ | 648/1570 [51:52<46:45, 3.04s/it]
41%|████▏ | 649/1570 [51:55<45:45, 2.98s/it]
41%|████▏ | 650/1570 [51:58<44:56, 2.93s/it]
41%|████▏ | 651/1570 [52:01<44:31, 2.91s/it]
42%|████▏ | 652/1570 [52:03<44:03, 2.88s/it]
42%|████▏ | 653/1570 [52:06<44:00, 2.88s/it]
42%|████▏ | 654/1570 [52:09<43:24, 2.84s/it]
42%|████▏ | 655/1570 [52:12<43:14, 2.84s/it]
42%|████▏ | 656/1570 [52:15<43:11, 2.83s/it]
42%|████▏ | 657/1570 [52:17<42:59, 2.83s/it]
42%|████▏ | 658/1570 [52:20<42:54, 2.82s/it]
42%|████▏ | 659/1570 [52:23<42:52, 2.82s/it]
42%|████▏ | 660/1570 [52:26<42:52, 2.83s/it]
{'loss': '0.004152', 'grad_norm': '1.311', 'learning_rate': '6.251e-05', 'neg_student_nll_loss': '0.003459', 'ce_loss': '0.003459', 'hpd_loss': '0.004152', 'sampled_diff_pct': '0.1661', 'adv1_neg_pct': '3.822', 'adv2_active_pct': '0.1639', 'mask3_pct': '0.1639', 'k1_gt_raw_mean': '0.0008853', 'k1_gt_raw_min': '-0.006316', 'k1_gt_raw_max': '0.04582', 'k1_gt_raw_neg_pct': '3.822', 'k1_sample_mean': '-0.3023', 'k1_sample_min': '-0.3555', 'k1_sample_max': '-0.2509', 'k1_sample_neg_pct': '10.83', 'epoch': '2.102'}
+
42%|████▏ | 660/1570 [52:26<42:52, 2.83s/it]
42%|████▏ | 661/1570 [52:29<42:57, 2.84s/it]
42%|████▏ | 662/1570 [52:32<42:43, 2.82s/it]
42%|████▏ | 663/1570 [52:34<42:43, 2.83s/it]
42%|████▏ | 664/1570 [52:37<42:34, 2.82s/it]
42%|████▏ | 665/1570 [52:40<42:33, 2.82s/it]
42%|████▏ | 666/1570 [52:43<42:38, 2.83s/it]
42%|████▏ | 667/1570 [52:46<42:37, 2.83s/it]
43%|████▎ | 668/1570 [52:49<42:31, 2.83s/it]
43%|████▎ | 669/1570 [52:51<42:25, 2.82s/it]
43%|████▎ | 670/1570 [52:54<42:20, 2.82s/it]
43%|████▎ | 671/1570 [52:57<42:29, 2.84s/it]
43%|████▎ | 672/1570 [53:00<42:27, 2.84s/it]
43%|████▎ | 673/1570 [53:03<42:13, 2.82s/it]
43%|████▎ | 674/1570 [53:06<42:29, 2.84s/it]
43%|████▎ | 675/1570 [53:08<42:35, 2.86s/it]
43%|████▎ | 676/1570 [53:11<42:26, 2.85s/it]
43%|████▎ | 677/1570 [53:14<42:14, 2.84s/it]
43%|████▎ | 678/1570 [53:17<42:12, 2.84s/it]
43%|████▎ | 679/1570 [53:20<42:11, 2.84s/it]
43%|████▎ | 680/1570 [53:23<42:08, 2.84s/it]
{'loss': '0.004049', 'grad_norm': '0.2058', 'learning_rate': '6.057e-05', 'neg_student_nll_loss': '0.003789', 'ce_loss': '0.003789', 'hpd_loss': '0.004049', 'sampled_diff_pct': '0.1798', 'adv1_neg_pct': '4.673', 'adv2_active_pct': '0.1779', 'mask3_pct': '0.1779', 'k1_gt_raw_mean': '0.001093', 'k1_gt_raw_min': '-0.008512', 'k1_gt_raw_max': '0.04776', 'k1_gt_raw_neg_pct': '4.673', 'k1_sample_mean': '-0.3683', 'k1_sample_min': '-0.441', 'k1_sample_max': '-0.2957', 'k1_sample_neg_pct': '12.81', 'epoch': '2.166'}
+
43%|████▎ | 680/1570 [53:23<42:08, 2.84s/it]
43%|████▎ | 681/1570 [53:25<42:06, 2.84s/it]
43%|████▎ | 682/1570 [53:28<42:06, 2.85s/it]
44%|████▎ | 683/1570 [53:31<41:53, 2.83s/it]
44%|████▎ | 684/1570 [53:34<41:53, 2.84s/it]
44%|████▎ | 685/1570 [53:37<42:03, 2.85s/it]
44%|████▎ | 686/1570 [53:40<42:01, 2.85s/it]
44%|████▍ | 687/1570 [53:43<41:53, 2.85s/it]
44%|████▍ | 688/1570 [53:45<41:52, 2.85s/it]
44%|████▍ | 689/1570 [53:48<41:50, 2.85s/it]
44%|████▍ | 690/1570 [53:51<41:40, 2.84s/it]
44%|████▍ | 691/1570 [53:54<41:34, 2.84s/it]
44%|████▍ | 692/1570 [53:57<41:33, 2.84s/it]
44%|████▍ | 693/1570 [54:00<41:28, 2.84s/it]
44%|████▍ | 694/1570 [54:02<41:35, 2.85s/it]
44%|████▍ | 695/1570 [54:05<41:35, 2.85s/it]
44%|████▍ | 696/1570 [54:08<41:19, 2.84s/it]
44%|████▍ | 697/1570 [54:11<41:08, 2.83s/it]
44%|████▍ | 698/1570 [54:14<41:02, 2.82s/it]
45%|████▍ | 699/1570 [54:17<41:07, 2.83s/it]
45%|████▍ | 700/1570 [54:19<41:04, 2.83s/it]
{'loss': '0.01242', 'grad_norm': '0.05892', 'learning_rate': '5.86e-05', 'neg_student_nll_loss': '0.008372', 'ce_loss': '0.008372', 'hpd_loss': '0.01242', 'sampled_diff_pct': '0.3336', 'adv1_neg_pct': '3.504', 'adv2_active_pct': '0.3336', 'mask3_pct': '0.3336', 'k1_gt_raw_mean': '0.00146', 'k1_gt_raw_min': '-0.001376', 'k1_gt_raw_max': '0.04455', 'k1_gt_raw_neg_pct': '3.504', 'k1_sample_mean': '-0.4156', 'k1_sample_min': '-0.4171', 'k1_sample_max': '-0.414', 'k1_sample_neg_pct': '12.81', 'epoch': '2.23'}
+
45%|████▍ | 700/1570 [54:19<41:04, 2.83s/it]
45%|████▍ | 701/1570 [54:22<41:02, 2.83s/it]
45%|████▍ | 702/1570 [54:25<41:06, 2.84s/it]
45%|████▍ | 703/1570 [54:28<40:58, 2.84s/it]
45%|████▍ | 704/1570 [54:31<40:53, 2.83s/it]
45%|████▍ | 705/1570 [54:34<40:46, 2.83s/it]
45%|████▍ | 706/1570 [54:37<41:06, 2.85s/it]
45%|████▌ | 707/1570 [54:39<40:54, 2.84s/it]
45%|████▌ | 708/1570 [54:42<40:57, 2.85s/it]
45%|████▌ | 709/1570 [54:45<40:57, 2.85s/it]
45%|████▌ | 710/1570 [54:48<40:39, 2.84s/it]
45%|████▌ | 711/1570 [54:51<40:39, 2.84s/it]
45%|████▌ | 712/1570 [54:54<40:34, 2.84s/it]
45%|████▌ | 713/1570 [54:56<40:38, 2.85s/it]
45%|████▌ | 714/1570 [54:59<40:35, 2.84s/it]
46%|████▌ | 715/1570 [55:02<40:32, 2.84s/it]
46%|████▌ | 716/1570 [55:05<40:27, 2.84s/it]
46%|████▌ | 717/1570 [55:08<40:21, 2.84s/it]
46%|████▌ | 718/1570 [55:11<40:31, 2.85s/it]
46%|████▌ | 719/1570 [55:14<40:28, 2.85s/it]
46%|████▌ | 720/1570 [55:16<40:27, 2.86s/it]
{'loss': '0.001331', 'grad_norm': '0.0307', 'learning_rate': '5.663e-05', 'neg_student_nll_loss': '0.002733', 'ce_loss': '0.002733', 'hpd_loss': '0.001331', 'sampled_diff_pct': '0.2406', 'adv1_neg_pct': '2.956', 'adv2_active_pct': '0.2356', 'mask3_pct': '0.2356', 'k1_gt_raw_mean': '0.001189', 'k1_gt_raw_min': '-0.003353', 'k1_gt_raw_max': '0.04304', 'k1_gt_raw_neg_pct': '2.956', 'k1_sample_mean': '-0.1317', 'k1_sample_min': '-0.1389', 'k1_sample_max': '-0.1255', 'k1_sample_neg_pct': '12.19', 'epoch': '2.294'}
+
46%|████▌ | 720/1570 [55:16<40:27, 2.86s/it]
46%|████▌ | 721/1570 [55:19<40:14, 2.84s/it]
46%|████▌ | 722/1570 [55:22<40:07, 2.84s/it]
46%|████▌ | 723/1570 [55:25<40:03, 2.84s/it]
46%|████▌ | 724/1570 [55:28<40:08, 2.85s/it]
46%|████▌ | 725/1570 [55:31<40:15, 2.86s/it]
46%|████▌ | 726/1570 [55:33<40:00, 2.84s/it]
46%|████▋ | 727/1570 [55:36<39:57, 2.84s/it]
46%|████▋ | 728/1570 [55:39<39:58, 2.85s/it]
46%|████▋ | 729/1570 [55:42<39:47, 2.84s/it]
46%|████▋ | 730/1570 [55:45<39:34, 2.83s/it]
47%|████▋ | 731/1570 [55:48<39:27, 2.82s/it]
47%|████▋ | 732/1570 [55:50<39:37, 2.84s/it]
47%|████▋ | 733/1570 [55:53<39:34, 2.84s/it]
47%|████▋ | 734/1570 [55:56<39:29, 2.83s/it]
47%|████▋ | 735/1570 [55:59<39:17, 2.82s/it]
47%|████▋ | 736/1570 [56:02<39:17, 2.83s/it]
47%|████▋ | 737/1570 [56:05<39:08, 2.82s/it]
47%|████▋ | 738/1570 [56:07<39:12, 2.83s/it]
47%|████▋ | 739/1570 [56:10<39:04, 2.82s/it]
47%|████▋ | 740/1570 [56:13<39:00, 2.82s/it]
{'loss': '0.00289', 'grad_norm': '0.03651', 'learning_rate': '5.464e-05', 'neg_student_nll_loss': '0.003668', 'ce_loss': '0.003668', 'hpd_loss': '0.00289', 'sampled_diff_pct': '0.1667', 'adv1_neg_pct': '4.363', 'adv2_active_pct': '0.1499', 'mask3_pct': '0.1499', 'k1_gt_raw_mean': '0.0007138', 'k1_gt_raw_min': '-0.007671', 'k1_gt_raw_max': '0.03678', 'k1_gt_raw_neg_pct': '4.363', 'k1_sample_mean': '-0.1942', 'k1_sample_min': '-0.2256', 'k1_sample_max': '-0.1629', 'k1_sample_neg_pct': '9.062', 'epoch': '2.357'}
+
47%|████▋ | 740/1570 [56:13<39:00, 2.82s/it]
47%|████▋ | 741/1570 [56:16<39:05, 2.83s/it]
47%|████▋ | 742/1570 [56:19<39:12, 2.84s/it]
47%|████▋ | 743/1570 [56:22<39:14, 2.85s/it]
47%|████▋ | 744/1570 [56:24<39:22, 2.86s/it]
47%|████▋ | 745/1570 [56:27<39:06, 2.84s/it]
48%|████▊ | 746/1570 [56:30<38:46, 2.82s/it]
48%|████▊ | 747/1570 [56:33<38:44, 2.82s/it]
48%|████▊ | 748/1570 [56:36<38:39, 2.82s/it]
48%|████▊ | 749/1570 [56:39<38:49, 2.84s/it]
48%|████▊ | 750/1570 [56:41<38:30, 2.82s/it]
48%|████▊ | 751/1570 [56:44<38:35, 2.83s/it]
48%|████▊ | 752/1570 [56:47<38:32, 2.83s/it]
48%|████▊ | 753/1570 [56:50<38:36, 2.83s/it]
48%|████▊ | 754/1570 [56:53<38:35, 2.84s/it]
48%|████▊ | 755/1570 [56:56<38:33, 2.84s/it]
48%|████▊ | 756/1570 [56:58<38:28, 2.84s/it]
48%|████▊ | 757/1570 [57:01<38:24, 2.83s/it]
48%|████▊ | 758/1570 [57:04<38:21, 2.83s/it]
48%|████▊ | 759/1570 [57:07<38:21, 2.84s/it]
48%|████▊ | 760/1570 [57:10<38:14, 2.83s/it]
{'loss': '0.00182', 'grad_norm': '0.03706', 'learning_rate': '5.265e-05', 'neg_student_nll_loss': '0.001899', 'ce_loss': '0.001899', 'hpd_loss': '0.00182', 'sampled_diff_pct': '0.08057', 'adv1_neg_pct': '4.309', 'adv2_active_pct': '0.07393', 'mask3_pct': '0.07393', 'k1_gt_raw_mean': '0.0005116', 'k1_gt_raw_min': '-0.003718', 'k1_gt_raw_max': '0.03066', 'k1_gt_raw_neg_pct': '4.309', 'k1_sample_mean': '-0.1201', 'k1_sample_min': '-0.1215', 'k1_sample_max': '-0.1187', 'k1_sample_neg_pct': '7.031', 'epoch': '2.421'}
+
48%|████▊ | 760/1570 [57:10<38:14, 2.83s/it]
48%|████▊ | 761/1570 [57:13<38:15, 2.84s/it]
49%|████▊ | 762/1570 [57:15<38:15, 2.84s/it]
49%|████▊ | 763/1570 [57:18<38:02, 2.83s/it]
49%|████▊ | 764/1570 [57:21<38:05, 2.84s/it]
49%|████▊ | 765/1570 [57:24<38:00, 2.83s/it]
49%|████▉ | 766/1570 [57:27<37:56, 2.83s/it]
49%|████▉ | 767/1570 [57:30<37:51, 2.83s/it]
49%|████▉ | 768/1570 [57:32<37:47, 2.83s/it]
49%|████▉ | 769/1570 [57:35<37:38, 2.82s/it]
49%|████▉ | 770/1570 [57:38<37:45, 2.83s/it]
49%|████▉ | 771/1570 [57:41<37:42, 2.83s/it]
49%|████▉ | 772/1570 [57:44<37:41, 2.83s/it]
49%|████▉ | 773/1570 [57:46<37:32, 2.83s/it]
49%|████▉ | 774/1570 [57:49<37:25, 2.82s/it]
49%|████▉ | 775/1570 [57:52<37:35, 2.84s/it]
49%|████▉ | 776/1570 [57:55<37:40, 2.85s/it]
49%|████▉ | 777/1570 [57:58<37:33, 2.84s/it]
50%|████▉ | 778/1570 [58:01<37:27, 2.84s/it]
50%|████▉ | 779/1570 [58:04<37:20, 2.83s/it]
50%|████▉ | 780/1570 [58:06<37:04, 2.82s/it]
{'loss': '0.0008434', 'grad_norm': '0.06339', 'learning_rate': '5.065e-05', 'neg_student_nll_loss': '0.0009786', 'ce_loss': '0.0009786', 'hpd_loss': '0.0008434', 'sampled_diff_pct': '0.09829', 'adv1_neg_pct': '4.628', 'adv2_active_pct': '0.08584', 'mask3_pct': '0.08584', 'k1_gt_raw_mean': '0.0004272', 'k1_gt_raw_min': '-0.006446', 'k1_gt_raw_max': '0.02713', 'k1_gt_raw_neg_pct': '4.628', 'k1_sample_mean': '-0.1102', 'k1_sample_min': '-0.1173', 'k1_sample_max': '-0.103', 'k1_sample_neg_pct': '8.906', 'epoch': '2.485'}
+
50%|████▉ | 780/1570 [58:06<37:04, 2.82s/it]
50%|████▉ | 781/1570 [58:09<37:01, 2.82s/it]
50%|████▉ | 782/1570 [58:12<37:05, 2.82s/it]
50%|████▉ | 783/1570 [58:15<37:08, 2.83s/it]
50%|████▉ | 784/1570 [58:18<37:05, 2.83s/it]
50%|█████ | 785/1570 [58:20<36:59, 2.83s/it]
50%|█████ | 786/1570 [58:23<36:55, 2.83s/it]
50%|█████ | 787/1570 [58:26<36:50, 2.82s/it]
50%|█████ | 788/1570 [58:29<36:39, 2.81s/it]
50%|█████ | 789/1570 [58:32<36:42, 2.82s/it]
50%|█████ | 790/1570 [58:35<36:45, 2.83s/it]
50%|█████ | 791/1570 [58:37<36:47, 2.83s/it]
50%|█████ | 792/1570 [58:40<36:45, 2.83s/it]
51%|█████ | 793/1570 [58:43<36:42, 2.83s/it]
51%|█████ | 794/1570 [58:46<36:37, 2.83s/it]
51%|█████ | 795/1570 [58:49<36:28, 2.82s/it]
51%|█████ | 796/1570 [58:52<36:31, 2.83s/it]
51%|█████ | 797/1570 [58:54<36:27, 2.83s/it]
51%|█████ | 798/1570 [58:57<36:21, 2.83s/it]
51%|█████ | 799/1570 [59:00<36:20, 2.83s/it]
51%|█████ | 800/1570 [59:03<36:37, 2.85s/it]
{'loss': '0.005209', 'grad_norm': '0.2113', 'learning_rate': '4.865e-05', 'neg_student_nll_loss': '0.004278', 'ce_loss': '0.004278', 'hpd_loss': '0.005209', 'sampled_diff_pct': '0.1585', 'adv1_neg_pct': '5.204', 'adv2_active_pct': '0.153', 'mask3_pct': '0.153', 'k1_gt_raw_mean': '0.0005127', 'k1_gt_raw_min': '-0.002086', 'k1_gt_raw_max': '0.03059', 'k1_gt_raw_neg_pct': '5.204', 'k1_sample_mean': '-0.08607', 'k1_sample_min': '-0.1001', 'k1_sample_max': '-0.07243', 'k1_sample_neg_pct': '10', 'epoch': '2.549'}
+
51%|█████ | 800/1570 [59:03<36:37, 2.85s/it]
51%|█████ | 801/1570 [59:06<36:31, 2.85s/it]
51%|█████ | 802/1570 [59:09<36:21, 2.84s/it]
51%|█████ | 803/1570 [59:11<36:22, 2.85s/it]
51%|█████ | 804/1570 [59:14<36:15, 2.84s/it]
51%|█████▏ | 805/1570 [59:17<36:05, 2.83s/it]
51%|█████▏ | 806/1570 [59:20<36:02, 2.83s/it]
51%|█████▏ | 807/1570 [59:23<35:57, 2.83s/it]
51%|█████▏ | 808/1570 [59:26<35:46, 2.82s/it]
52%|█████▏ | 809/1570 [59:28<35:49, 2.82s/it]
52%|█████▏ | 810/1570 [59:31<35:45, 2.82s/it]
52%|█████▏ | 811/1570 [59:34<35:37, 2.82s/it]
52%|█████▏ | 812/1570 [59:37<35:37, 2.82s/it]
52%|█████▏ | 813/1570 [59:40<35:33, 2.82s/it]
52%|█████▏ | 814/1570 [59:42<35:33, 2.82s/it]
52%|█████▏ | 815/1570 [59:45<35:29, 2.82s/it]
52%|█████▏ | 816/1570 [59:48<35:41, 2.84s/it]
52%|█████▏ | 817/1570 [59:51<35:39, 2.84s/it]
52%|█████▏ | 818/1570 [59:54<35:40, 2.85s/it]
52%|█████▏ | 819/1570 [59:57<35:39, 2.85s/it]
52%|█████▏ | 820/1570 [1:00:00<35:32, 2.84s/it]
{'loss': '0.002393', 'grad_norm': '0.3225', 'learning_rate': '4.665e-05', 'neg_student_nll_loss': '0.003248', 'ce_loss': '0.003248', 'hpd_loss': '0.002393', 'sampled_diff_pct': '0.1166', 'adv1_neg_pct': '5.268', 'adv2_active_pct': '0.107', 'mask3_pct': '0.104', 'k1_gt_raw_mean': '0.001083', 'k1_gt_raw_min': '-0.002741', 'k1_gt_raw_max': '0.03637', 'k1_gt_raw_neg_pct': '5.268', 'k1_sample_mean': '-0.0661', 'k1_sample_min': '-0.06948', 'k1_sample_max': '-0.06465', 'k1_sample_neg_pct': '6.827', 'epoch': '2.613'}
+
52%|█████▏ | 820/1570 [1:00:00<35:32, 2.84s/it]
52%|█████▏ | 821/1570 [1:00:02<35:37, 2.85s/it]
52%|█████▏ | 822/1570 [1:00:05<35:24, 2.84s/it]
52%|█████▏ | 823/1570 [1:00:08<35:19, 2.84s/it]
52%|█████▏ | 824/1570 [1:00:11<35:12, 2.83s/it]
53%|█████▎ | 825/1570 [1:00:14<35:05, 2.83s/it]
53%|█████▎ | 826/1570 [1:00:17<35:15, 2.84s/it]
53%|█████▎ | 827/1570 [1:00:19<35:07, 2.84s/it]
53%|█████▎ | 828/1570 [1:00:22<35:07, 2.84s/it]
53%|█████▎ | 829/1570 [1:00:25<34:59, 2.83s/it]
53%|█████▎ | 830/1570 [1:00:28<35:04, 2.84s/it]
53%|█████▎ | 831/1570 [1:00:31<34:58, 2.84s/it]
53%|█████▎ | 832/1570 [1:00:34<34:51, 2.83s/it]
53%|█████▎ | 833/1570 [1:00:36<34:56, 2.84s/it]
53%|█████▎ | 834/1570 [1:00:39<34:55, 2.85s/it]
53%|█████▎ | 835/1570 [1:00:42<34:56, 2.85s/it]
53%|█████▎ | 836/1570 [1:00:45<34:45, 2.84s/it]
53%|█████▎ | 837/1570 [1:00:48<34:40, 2.84s/it]
53%|█████▎ | 838/1570 [1:00:51<34:40, 2.84s/it]
53%|█████▎ | 839/1570 [1:00:54<34:32, 2.84s/it]
54%|█████▎ | 840/1570 [1:00:56<34:23, 2.83s/it]
{'loss': '0.002443', 'grad_norm': '0.2546', 'learning_rate': '4.466e-05', 'neg_student_nll_loss': '0.002023', 'ce_loss': '0.002023', 'hpd_loss': '0.002443', 'sampled_diff_pct': '0.07494', 'adv1_neg_pct': '5.766', 'adv2_active_pct': '0.06632', 'mask3_pct': '0.06632', 'k1_gt_raw_mean': '0.0007106', 'k1_gt_raw_min': '-0.004439', 'k1_gt_raw_max': '0.02783', 'k1_gt_raw_neg_pct': '5.766', 'k1_sample_mean': '-0.1007', 'k1_sample_min': '-0.1007', 'k1_sample_max': '-0.1007', 'k1_sample_neg_pct': '7.188', 'epoch': '2.677'}
+
54%|█████▎ | 840/1570 [1:00:56<34:23, 2.83s/it]
54%|█████▎ | 841/1570 [1:00:59<34:16, 2.82s/it]
54%|█████▎ | 842/1570 [1:01:02<34:19, 2.83s/it]
54%|█████▎ | 843/1570 [1:01:05<34:15, 2.83s/it]
54%|█████▍ | 844/1570 [1:01:08<34:22, 2.84s/it]
54%|█████▍ | 845/1570 [1:01:11<34:19, 2.84s/it]
54%|█████▍ | 846/1570 [1:01:13<34:23, 2.85s/it]
54%|█████▍ | 847/1570 [1:01:16<34:18, 2.85s/it]
54%|█████▍ | 848/1570 [1:01:19<34:06, 2.83s/it]
54%|█████▍ | 849/1570 [1:01:22<33:59, 2.83s/it]
54%|█████▍ | 850/1570 [1:01:25<34:04, 2.84s/it]
54%|█████▍ | 851/1570 [1:01:28<34:01, 2.84s/it]
54%|█████▍ | 852/1570 [1:01:30<33:58, 2.84s/it]
54%|█████▍ | 853/1570 [1:01:33<33:54, 2.84s/it]
54%|█████▍ | 854/1570 [1:01:36<33:56, 2.84s/it]
54%|█████▍ | 855/1570 [1:01:39<33:51, 2.84s/it]
55%|█████▍ | 856/1570 [1:01:42<33:42, 2.83s/it]
55%|█████▍ | 857/1570 [1:01:45<33:50, 2.85s/it]
55%|█████▍ | 858/1570 [1:01:47<33:42, 2.84s/it]
55%|█████▍ | 859/1570 [1:01:50<33:31, 2.83s/it]
55%|█████▍ | 860/1570 [1:01:53<33:40, 2.85s/it]
{'loss': '0.001978', 'grad_norm': '0.2799', 'learning_rate': '4.268e-05', 'neg_student_nll_loss': '0.001998', 'ce_loss': '0.001998', 'hpd_loss': '0.001978', 'sampled_diff_pct': '0.1026', 'adv1_neg_pct': '6.066', 'adv2_active_pct': '0.1001', 'mask3_pct': '0.1001', 'k1_gt_raw_mean': '0.0005697', 'k1_gt_raw_min': '-0.003441', 'k1_gt_raw_max': '0.03277', 'k1_gt_raw_neg_pct': '6.066', 'k1_sample_mean': '-0.138', 'k1_sample_min': '-0.1927', 'k1_sample_max': '-0.08324', 'k1_sample_neg_pct': '9.375', 'epoch': '2.74'}
+
55%|█████▍ | 860/1570 [1:01:53<33:40, 2.85s/it]
55%|█████▍ | 861/1570 [1:01:56<33:27, 2.83s/it]
55%|█████▍ | 862/1570 [1:01:59<33:25, 2.83s/it]
55%|█████▍ | 863/1570 [1:02:02<33:12, 2.82s/it]
55%|█████▌ | 864/1570 [1:02:04<33:17, 2.83s/it]
55%|█████▌ | 865/1570 [1:02:07<33:18, 2.83s/it]
55%|█████▌ | 866/1570 [1:02:10<33:04, 2.82s/it]
55%|█████▌ | 867/1570 [1:02:13<32:57, 2.81s/it]
55%|█████▌ | 868/1570 [1:02:16<32:51, 2.81s/it]
55%|█████▌ | 869/1570 [1:02:18<32:57, 2.82s/it]
55%|█████▌ | 870/1570 [1:02:21<32:57, 2.82s/it]
55%|█████▌ | 871/1570 [1:02:24<32:56, 2.83s/it]
56%|█████▌ | 872/1570 [1:02:27<32:52, 2.83s/it]
56%|█████▌ | 873/1570 [1:02:30<32:50, 2.83s/it]
56%|█████▌ | 874/1570 [1:02:33<32:49, 2.83s/it]
56%|█████▌ | 875/1570 [1:02:35<32:47, 2.83s/it]
56%|█████▌ | 876/1570 [1:02:38<32:40, 2.83s/it]
56%|█████▌ | 877/1570 [1:02:41<32:45, 2.84s/it]
56%|█████▌ | 878/1570 [1:02:44<32:43, 2.84s/it]
56%|█████▌ | 879/1570 [1:02:47<32:49, 2.85s/it]
56%|█████▌ | 880/1570 [1:02:50<32:47, 2.85s/it]
{'loss': '0.001079', 'grad_norm': '0.09429', 'learning_rate': '4.071e-05', 'neg_student_nll_loss': '0.001433', 'ce_loss': '0.001433', 'hpd_loss': '0.001079', 'sampled_diff_pct': '0.07658', 'adv1_neg_pct': '5.892', 'adv2_active_pct': '0.07234', 'mask3_pct': '0.07234', 'k1_gt_raw_mean': '0.0007732', 'k1_gt_raw_min': '-0.003895', 'k1_gt_raw_max': '0.03859', 'k1_gt_raw_neg_pct': '5.892', 'k1_sample_mean': '-0.1251', 'k1_sample_min': '-0.1277', 'k1_sample_max': '-0.1225', 'k1_sample_neg_pct': '7.344', 'epoch': '2.804'}
+
56%|█████▌ | 880/1570 [1:02:50<32:47, 2.85s/it]
56%|█████▌ | 881/1570 [1:02:53<32:39, 2.84s/it]
56%|█████▌ | 882/1570 [1:02:55<32:34, 2.84s/it]
56%|█████▌ | 883/1570 [1:02:58<32:28, 2.84s/it]
56%|█████▋ | 884/1570 [1:03:01<32:22, 2.83s/it]
56%|█████▋ | 885/1570 [1:03:04<32:09, 2.82s/it]
56%|█████▋ | 886/1570 [1:03:07<32:16, 2.83s/it]
56%|█████▋ | 887/1570 [1:03:10<32:22, 2.84s/it]
57%|█████▋ | 888/1570 [1:03:12<32:24, 2.85s/it]
57%|█████▋ | 889/1570 [1:03:15<32:21, 2.85s/it]
57%|█████▋ | 890/1570 [1:03:18<32:08, 2.84s/it]
57%|█████▋ | 891/1570 [1:03:21<32:04, 2.83s/it]
57%|█████▋ | 892/1570 [1:03:24<32:06, 2.84s/it]
57%|█████▋ | 893/1570 [1:03:27<31:59, 2.84s/it]
57%|█████▋ | 894/1570 [1:03:29<32:05, 2.85s/it]
57%|█████▋ | 895/1570 [1:03:32<31:54, 2.84s/it]
57%|█████▋ | 896/1570 [1:03:35<31:50, 2.83s/it]
57%|█████▋ | 897/1570 [1:03:38<31:55, 2.85s/it]
57%|█████▋ | 898/1570 [1:03:41<32:01, 2.86s/it]
57%|█████▋ | 899/1570 [1:03:44<31:47, 2.84s/it]
57%|█████▋ | 900/1570 [1:03:47<31:45, 2.84s/it]
{'loss': '0.001599', 'grad_norm': '1.021', 'learning_rate': '3.875e-05', 'neg_student_nll_loss': '0.002096', 'ce_loss': '0.002096', 'hpd_loss': '0.001599', 'sampled_diff_pct': '0.1047', 'adv1_neg_pct': '6.012', 'adv2_active_pct': '0.09839', 'mask3_pct': '0.09839', 'k1_gt_raw_mean': '0.000616', 'k1_gt_raw_min': '-0.005019', 'k1_gt_raw_max': '0.03688', 'k1_gt_raw_neg_pct': '6.012', 'k1_sample_mean': '-0.2854', 'k1_sample_min': '-0.2913', 'k1_sample_max': '-0.2795', 'k1_sample_neg_pct': '8.75', 'epoch': '2.868'}
+
57%|█████▋ | 900/1570 [1:03:47<31:45, 2.84s/it]
57%|█████▋ | 901/1570 [1:03:49<31:40, 2.84s/it]
57%|█████▋ | 902/1570 [1:03:52<31:38, 2.84s/it]
58%|█████▊ | 903/1570 [1:03:55<31:34, 2.84s/it]
58%|█████▊ | 904/1570 [1:03:58<31:29, 2.84s/it]
58%|█████▊ | 905/1570 [1:04:01<31:31, 2.84s/it]
58%|█████▊ | 906/1570 [1:04:03<31:15, 2.82s/it]
58%|█████▊ | 907/1570 [1:04:06<31:16, 2.83s/it]
58%|█████▊ | 908/1570 [1:04:09<31:13, 2.83s/it]
58%|█████▊ | 909/1570 [1:04:12<31:13, 2.83s/it]
58%|█████▊ | 910/1570 [1:04:15<31:10, 2.83s/it]
58%|█████▊ | 911/1570 [1:04:18<31:11, 2.84s/it]
58%|█████▊ | 912/1570 [1:04:21<31:11, 2.84s/it]
58%|█████▊ | 913/1570 [1:04:23<31:03, 2.84s/it]
58%|█████▊ | 914/1570 [1:04:26<31:02, 2.84s/it]
58%|█████▊ | 915/1570 [1:04:29<31:04, 2.85s/it]
58%|█████▊ | 916/1570 [1:04:32<31:00, 2.84s/it]
58%|█████▊ | 917/1570 [1:04:35<30:47, 2.83s/it]
58%|█████▊ | 918/1570 [1:04:37<30:37, 2.82s/it]
59%|█████▊ | 919/1570 [1:04:40<30:38, 2.82s/it]
59%|█████▊ | 920/1570 [1:04:43<30:38, 2.83s/it]
{'loss': '0.005076', 'grad_norm': '0.00979', 'learning_rate': '3.682e-05', 'neg_student_nll_loss': '0.003196', 'ce_loss': '0.003196', 'hpd_loss': '0.005076', 'sampled_diff_pct': '0.1294', 'adv1_neg_pct': '5.845', 'adv2_active_pct': '0.1254', 'mask3_pct': '0.1254', 'k1_gt_raw_mean': '0.000579', 'k1_gt_raw_min': '-0.0043', 'k1_gt_raw_max': '0.03245', 'k1_gt_raw_neg_pct': '5.845', 'k1_sample_mean': '-0.1174', 'k1_sample_min': '-0.1418', 'k1_sample_max': '-0.09453', 'k1_sample_neg_pct': '8.438', 'epoch': '2.932'}
+
59%|█████▊ | 920/1570 [1:04:43<30:38, 2.83s/it]
59%|█████▊ | 921/1570 [1:04:46<30:37, 2.83s/it]
59%|█████▊ | 922/1570 [1:04:49<30:37, 2.84s/it]
59%|█████▉ | 923/1570 [1:04:52<30:39, 2.84s/it]
59%|█████▉ | 924/1570 [1:04:55<30:36, 2.84s/it]
59%|█████▉ | 925/1570 [1:04:57<30:34, 2.84s/it]
59%|█████▉ | 926/1570 [1:05:00<30:25, 2.83s/it]
59%|█████▉ | 927/1570 [1:05:03<30:21, 2.83s/it]
59%|█████▉ | 928/1570 [1:05:06<30:14, 2.83s/it]
59%|█████▉ | 929/1570 [1:05:09<30:20, 2.84s/it]
59%|█████▉ | 930/1570 [1:05:12<30:24, 2.85s/it]
59%|█████▉ | 931/1570 [1:05:14<30:16, 2.84s/it]
59%|█████▉ | 932/1570 [1:05:17<30:17, 2.85s/it]
59%|█████▉ | 933/1570 [1:05:20<30:20, 2.86s/it]
59%|█████▉ | 934/1570 [1:05:23<30:25, 2.87s/it]
60%|█████▉ | 935/1570 [1:05:26<30:21, 2.87s/it]
60%|█████▉ | 936/1570 [1:05:29<30:19, 2.87s/it]
60%|█████▉ | 937/1570 [1:05:32<30:10, 2.86s/it]
60%|█████▉ | 938/1570 [1:05:34<30:03, 2.85s/it]
60%|█████▉ | 939/1570 [1:05:37<29:59, 2.85s/it]
60%|█████▉ | 940/1570 [1:05:40<29:54, 2.85s/it]
{'loss': '0.001085', 'grad_norm': '0.1644', 'learning_rate': '3.49e-05', 'neg_student_nll_loss': '0.001649', 'ce_loss': '0.001649', 'hpd_loss': '0.001085', 'sampled_diff_pct': '0.08656', 'adv1_neg_pct': '6.187', 'adv2_active_pct': '0.0847', 'mask3_pct': '0.0847', 'k1_gt_raw_mean': '0.0005872', 'k1_gt_raw_min': '-0.006866', 'k1_gt_raw_max': '0.03225', 'k1_gt_raw_neg_pct': '6.187', 'k1_sample_mean': '-0.1391', 'k1_sample_min': '-0.1643', 'k1_sample_max': '-0.1139', 'k1_sample_neg_pct': '7.812', 'epoch': '2.996'}
+
60%|█████▉ | 940/1570 [1:05:40<29:54, 2.85s/it]
60%|█████▉ | 941/1570 [1:05:43<29:53, 2.85s/it]
60%|██████ | 942/1570 [1:05:44<24:30, 2.34s/it][INFO|trainer.py:2668] 2026-08-22 23:04:19,563 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 23:04:19,563 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 23:04:19,563 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/hpd/checkpoint-942
+[INFO|configuration_utils.py:780] 2026-08-22 23:14:12,532 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 23:14:12,537 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 23:14:12,571 >> chat template saved in results/qwen3/hpd/checkpoint-942/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 23:14:12,572 >> tokenizer config file saved in results/qwen3/hpd/checkpoint-942/tokenizer_config.json
+
60%|██████ | 943/1570 [1:15:41<31:28:07, 180.68s/it]
60%|██████ | 944/1570 [1:15:44<22:08:27, 127.33s/it]
60%|██████ | 945/1570 [1:15:47<15:37:25, 89.99s/it]
60%|██████ | 946/1570 [1:15:50<11:03:58, 63.84s/it]
60%|██████ | 947/1570 [1:15:52<7:52:49, 45.54s/it]
60%|██████ | 948/1570 [1:15:55<5:39:14, 32.72s/it]
60%|██████ | 949/1570 [1:15:58<4:05:51, 23.76s/it]
61%|██████ | 950/1570 [1:16:01<3:00:35, 17.48s/it]
61%|██████ | 951/1570 [1:16:04<2:14:54, 13.08s/it]
61%|██████ | 952/1570 [1:16:06<1:43:03, 10.01s/it]
61%|██████ | 953/1570 [1:16:09<1:20:46, 7.86s/it]
61%|██████ | 954/1570 [1:16:12<1:05:17, 6.36s/it]
61%|██████ | 955/1570 [1:16:15<54:26, 5.31s/it]
61%|██████ | 956/1570 [1:16:18<46:49, 4.58s/it]
61%|██████ | 957/1570 [1:16:21<41:25, 4.05s/it]
61%|██████ | 958/1570 [1:16:24<37:37, 3.69s/it]
61%|██████ | 959/1570 [1:16:26<34:49, 3.42s/it]
61%|██████ | 960/1570 [1:16:29<32:53, 3.24s/it]
{'loss': '0.002466', 'grad_norm': '0.08226', 'learning_rate': '3.301e-05', 'neg_student_nll_loss': '0.002485', 'ce_loss': '0.002485', 'hpd_loss': '0.002643', 'sampled_diff_pct': '0.1241', 'adv1_neg_pct': '5.785', 'adv2_active_pct': '0.1198', 'mask3_pct': '0.1198', 'k1_gt_raw_mean': '0.0009094', 'k1_gt_raw_min': '-0.00696', 'k1_gt_raw_max': '0.03072', 'k1_gt_raw_neg_pct': '5.785', 'k1_sample_mean': '-0.066', 'k1_sample_min': '-0.066', 'k1_sample_max': '-0.066', 'k1_sample_neg_pct': '6.944', 'epoch': '3.057'}
+
61%|██████ | 960/1570 [1:16:29<32:53, 3.24s/it]
61%|██████ | 961/1570 [1:16:32<31:29, 3.10s/it]
61%|██████▏ | 962/1570 [1:16:35<30:31, 3.01s/it]
61%|██████▏ | 963/1570 [1:16:38<29:54, 2.96s/it]
61%|██████▏ | 964/1570 [1:16:40<29:32, 2.93s/it]
61%|██████▏ | 965/1570 [1:16:43<29:16, 2.90s/it]
62%|██████▏ | 966/1570 [1:16:46<29:02, 2.88s/it]
62%|██████▏ | 967/1570 [1:16:49<28:37, 2.85s/it]
62%|██████▏ | 968/1570 [1:16:52<28:34, 2.85s/it]
62%|██████▏ | 969/1570 [1:16:55<28:28, 2.84s/it]
62%|██████▏ | 970/1570 [1:16:57<28:27, 2.85s/it]
62%|██████▏ | 971/1570 [1:17:00<28:13, 2.83s/it]
62%|██████▏ | 972/1570 [1:17:03<28:01, 2.81s/it]
62%|██████▏ | 973/1570 [1:17:06<27:56, 2.81s/it]
62%|██████▏ | 974/1570 [1:17:09<27:57, 2.81s/it]
62%|██████▏ | 975/1570 [1:17:11<27:53, 2.81s/it]
62%|██████▏ | 976/1570 [1:17:14<27:50, 2.81s/it]
62%|██████▏ | 977/1570 [1:17:17<27:56, 2.83s/it]
62%|██████▏ | 978/1570 [1:17:20<27:49, 2.82s/it]
62%|██████▏ | 979/1570 [1:17:23<27:45, 2.82s/it]
62%|██████▏ | 980/1570 [1:17:26<27:50, 2.83s/it]
{'loss': '0.002391', 'grad_norm': '0.002783', 'learning_rate': '3.114e-05', 'neg_student_nll_loss': '0.00291', 'ce_loss': '0.00291', 'hpd_loss': '0.002391', 'sampled_diff_pct': '0.1716', 'adv1_neg_pct': '6.276', 'adv2_active_pct': '0.1653', 'mask3_pct': '0.1653', 'k1_gt_raw_mean': '0.001082', 'k1_gt_raw_min': '-0.005311', 'k1_gt_raw_max': '0.03877', 'k1_gt_raw_neg_pct': '6.276', 'k1_sample_mean': '-0.1089', 'k1_sample_min': '-0.1162', 'k1_sample_max': '-0.1011', 'k1_sample_neg_pct': '7.812', 'epoch': '3.121'}
+
62%|██████▏ | 980/1570 [1:17:26<27:50, 2.83s/it]
62%|██████▏ | 981/1570 [1:17:28<27:44, 2.83s/it]
63%|██████▎ | 982/1570 [1:17:31<27:43, 2.83s/it]
63%|██████▎ | 983/1570 [1:17:34<27:44, 2.83s/it]
63%|██████▎ | 984/1570 [1:17:37<27:33, 2.82s/it]
63%|██████▎ | 985/1570 [1:17:40<27:30, 2.82s/it]
63%|██████▎ | 986/1570 [1:17:43<27:26, 2.82s/it]
63%|██████▎ | 987/1570 [1:17:45<27:19, 2.81s/it]
63%|██████▎ | 988/1570 [1:17:48<27:19, 2.82s/it]
63%|██████▎ | 989/1570 [1:17:51<27:20, 2.82s/it]
63%|██████▎ | 990/1570 [1:17:54<27:08, 2.81s/it]
63%|██████▎ | 991/1570 [1:17:57<27:13, 2.82s/it]
63%|██████▎ | 992/1570 [1:17:59<27:13, 2.83s/it]
63%|██████▎ | 993/1570 [1:18:02<27:08, 2.82s/it]
63%|██████▎ | 994/1570 [1:18:05<27:03, 2.82s/it]
63%|██████▎ | 995/1570 [1:18:08<27:02, 2.82s/it]
63%|██████▎ | 996/1570 [1:18:11<26:57, 2.82s/it]
64%|██████▎ | 997/1570 [1:18:13<26:48, 2.81s/it]
64%|██████▎ | 998/1570 [1:18:16<26:43, 2.80s/it]
64%|██████▎ | 999/1570 [1:18:19<26:49, 2.82s/it]
64%|██████▎ | 1000/1570 [1:18:22<26:49, 2.82s/it]
{'loss': '0.001036', 'grad_norm': '0.02559', 'learning_rate': '2.931e-05', 'neg_student_nll_loss': '0.000891', 'ce_loss': '0.000891', 'hpd_loss': '0.001036', 'sampled_diff_pct': '0.08395', 'adv1_neg_pct': '6.713', 'adv2_active_pct': '0.07989', 'mask3_pct': '0.07989', 'k1_gt_raw_mean': '0.0003879', 'k1_gt_raw_min': '-0.001881', 'k1_gt_raw_max': '0.02309', 'k1_gt_raw_neg_pct': '6.713', 'k1_sample_mean': '-0.07004', 'k1_sample_min': '-0.07004', 'k1_sample_max': '-0.07004', 'k1_sample_neg_pct': '6.562', 'epoch': '3.185'}
+
64%|██████▎ | 1000/1570 [1:18:22<26:49, 2.82s/it]
64%|██████▍ | 1001/1570 [1:18:25<26:41, 2.82s/it]
64%|██████▍ | 1002/1570 [1:18:28<26:35, 2.81s/it]
64%|██████▍ | 1003/1570 [1:18:30<26:44, 2.83s/it]
64%|██████▍ | 1004/1570 [1:18:33<26:37, 2.82s/it]
64%|██████▍ | 1005/1570 [1:18:36<26:35, 2.82s/it]
64%|██████▍ | 1006/1570 [1:18:39<26:35, 2.83s/it]
64%|██████▍ | 1007/1570 [1:18:42<26:35, 2.83s/it]
64%|██████▍ | 1008/1570 [1:18:45<26:31, 2.83s/it]
64%|██████▍ | 1009/1570 [1:18:47<26:22, 2.82s/it]
64%|██████▍ | 1010/1570 [1:18:50<26:13, 2.81s/it]
64%|██████▍ | 1011/1570 [1:18:53<26:18, 2.82s/it]
64%|██████▍ | 1012/1570 [1:18:56<26:19, 2.83s/it]
65%|██████▍ | 1013/1570 [1:18:59<26:13, 2.82s/it]
65%|██████▍ | 1014/1570 [1:19:02<26:13, 2.83s/it]
65%|██████▍ | 1015/1570 [1:19:04<26:07, 2.82s/it]
65%|██████▍ | 1016/1570 [1:19:07<26:08, 2.83s/it]
65%|██████▍ | 1017/1570 [1:19:10<26:05, 2.83s/it]
65%|██████▍ | 1018/1570 [1:19:13<25:57, 2.82s/it]
65%|██████▍ | 1019/1570 [1:19:16<25:49, 2.81s/it]
65%|██████▍ | 1020/1570 [1:19:18<25:51, 2.82s/it]
{'loss': '0.0005542', 'grad_norm': '0.04065', 'learning_rate': '2.751e-05', 'neg_student_nll_loss': '0.0008955', 'ce_loss': '0.0008955', 'hpd_loss': '0.0005542', 'sampled_diff_pct': '0.04862', 'adv1_neg_pct': '6.98', 'adv2_active_pct': '0.04862', 'mask3_pct': '0.04862', 'k1_gt_raw_mean': '0.0003737', 'k1_gt_raw_min': '-0.003176', 'k1_gt_raw_max': '0.02335', 'k1_gt_raw_neg_pct': '6.98', 'k1_sample_mean': '-0.02265', 'k1_sample_min': '-0.02265', 'k1_sample_max': '-0.02265', 'k1_sample_neg_pct': '5.625', 'epoch': '3.249'}
+
65%|██████▍ | 1020/1570 [1:19:18<25:51, 2.82s/it]
65%|██████▌ | 1021/1570 [1:19:21<25:54, 2.83s/it]
65%|██████▌ | 1022/1570 [1:19:24<25:50, 2.83s/it]
65%|██████▌ | 1023/1570 [1:19:27<25:49, 2.83s/it]
65%|██████▌ | 1024/1570 [1:19:30<25:48, 2.84s/it]
65%|██████▌ | 1025/1570 [1:19:33<25:43, 2.83s/it]
65%|██████▌ | 1026/1570 [1:19:35<25:40, 2.83s/it]
65%|██████▌ | 1027/1570 [1:19:39<27:40, 3.06s/it]
65%|██████▌ | 1028/1570 [1:19:43<29:04, 3.22s/it]
66%|██████▌ | 1029/1570 [1:19:46<29:35, 3.28s/it]
66%|██████▌ | 1030/1570 [1:19:50<30:12, 3.36s/it]
66%|██████▌ | 1031/1570 [1:19:53<30:36, 3.41s/it]
66%|██████▌ | 1032/1570 [1:19:57<30:57, 3.45s/it]
66%|██████▌ | 1033/1570 [1:20:00<30:54, 3.45s/it]
66%|██████▌ | 1034/1570 [1:20:04<30:44, 3.44s/it]
66%|██████▌ | 1035/1570 [1:20:07<31:05, 3.49s/it]
66%|██████▌ | 1036/1570 [1:20:11<30:46, 3.46s/it]
66%|██████▌ | 1037/1570 [1:20:14<30:44, 3.46s/it]
66%|██████▌ | 1038/1570 [1:20:18<31:03, 3.50s/it]
66%|██████▌ | 1039/1570 [1:20:21<31:06, 3.52s/it]
66%|██████▌ | 1040/1570 [1:20:25<31:05, 3.52s/it]
{'loss': '0.002801', 'grad_norm': '2.855', 'learning_rate': '2.574e-05', 'neg_student_nll_loss': '0.001963', 'ce_loss': '0.001963', 'hpd_loss': '0.002801', 'sampled_diff_pct': '0.1027', 'adv1_neg_pct': '6.829', 'adv2_active_pct': '0.08894', 'mask3_pct': '0.08894', 'k1_gt_raw_mean': '0.0004314', 'k1_gt_raw_min': '-0.006725', 'k1_gt_raw_max': '0.0212', 'k1_gt_raw_neg_pct': '6.829', 'k1_sample_mean': '-0.0774', 'k1_sample_min': '-0.0774', 'k1_sample_max': '-0.0774', 'k1_sample_neg_pct': '5', 'epoch': '3.313'}
+
66%|██████▌ | 1040/1570 [1:20:25<31:05, 3.52s/it]
66%|██████▋ | 1041/1570 [1:20:28<30:47, 3.49s/it]
66%|██████▋ | 1042/1570 [1:20:32<31:00, 3.52s/it]
66%|██████▋ | 1043/1570 [1:20:35<31:09, 3.55s/it]
66%|██████▋ | 1044/1570 [1:20:39<30:59, 3.53s/it]
67%|██████▋ | 1045/1570 [1:20:42<30:56, 3.54s/it]
67%|██████▋ | 1046/1570 [1:20:46<31:02, 3.55s/it]
67%|██████▋ | 1047/1570 [1:20:50<31:00, 3.56s/it]
67%|██████▋ | 1048/1570 [1:20:53<30:56, 3.56s/it]
67%|██████▋ | 1049/1570 [1:20:57<30:53, 3.56s/it]
67%|██████▋ | 1050/1570 [1:21:00<30:25, 3.51s/it]
67%|██████▋ | 1051/1570 [1:21:04<30:19, 3.51s/it]
67%|██████▋ | 1052/1570 [1:21:07<30:30, 3.53s/it]
67%|██████▋ | 1053/1570 [1:21:11<30:34, 3.55s/it]
67%|██████▋ | 1054/1570 [1:21:14<30:24, 3.54s/it]
67%|██████▋ | 1055/1570 [1:21:18<30:25, 3.54s/it]
67%|██████▋ | 1056/1570 [1:21:21<30:22, 3.55s/it]
67%|██████▋ | 1057/1570 [1:21:25<30:12, 3.53s/it]
67%|██████▋ | 1058/1570 [1:21:28<29:56, 3.51s/it]
67%|██████▋ | 1059/1570 [1:21:32<30:02, 3.53s/it]
68%|██████▊ | 1060/1570 [1:21:35<29:58, 3.53s/it]
{'loss': '0.006909', 'grad_norm': '0.004515', 'learning_rate': '2.402e-05', 'neg_student_nll_loss': '0.004101', 'ce_loss': '0.004101', 'hpd_loss': '0.006909', 'sampled_diff_pct': '0.1061', 'adv1_neg_pct': '7.224', 'adv2_active_pct': '0.1006', 'mask3_pct': '0.1006', 'k1_gt_raw_mean': '0.0002864', 'k1_gt_raw_min': '-0.003865', 'k1_gt_raw_max': '0.0179', 'k1_gt_raw_neg_pct': '7.224', 'k1_sample_mean': '-0.07833', 'k1_sample_min': '-0.07833', 'k1_sample_max': '-0.07833', 'k1_sample_neg_pct': '5.938', 'epoch': '3.377'}
+
68%|██████▊ | 1060/1570 [1:21:35<29:58, 3.53s/it]
68%|██████▊ | 1061/1570 [1:21:39<29:34, 3.49s/it]
68%|██████▊ | 1062/1570 [1:21:42<27:50, 3.29s/it]
68%|██████▊ | 1063/1570 [1:21:44<26:32, 3.14s/it]
68%|██████▊ | 1064/1570 [1:21:47<25:40, 3.04s/it]
68%|██████▊ | 1065/1570 [1:21:50<25:05, 2.98s/it]
68%|██████▊ | 1066/1570 [1:21:53<24:34, 2.93s/it]
68%|██████▊ | 1067/1570 [1:21:56<24:12, 2.89s/it]
68%|██████▊ | 1068/1570 [1:21:58<24:02, 2.87s/it]
68%|██████▊ | 1069/1570 [1:22:01<23:59, 2.87s/it]
68%|██████▊ | 1070/1570 [1:22:04<23:47, 2.86s/it]
68%|██████▊ | 1071/1570 [1:22:07<23:42, 2.85s/it]
68%|██████▊ | 1072/1570 [1:22:10<23:40, 2.85s/it]
68%|██████▊ | 1073/1570 [1:22:13<23:34, 2.85s/it]
68%|██████▊ | 1074/1570 [1:22:16<23:31, 2.85s/it]
68%|██████▊ | 1075/1570 [1:22:18<23:24, 2.84s/it]
69%|██████▊ | 1076/1570 [1:22:21<23:19, 2.83s/it]
69%|██████▊ | 1077/1570 [1:22:24<23:10, 2.82s/it]
69%|██████▊ | 1078/1570 [1:22:27<23:11, 2.83s/it]
69%|██████▊ | 1079/1570 [1:22:30<23:05, 2.82s/it]
69%|██████▉ | 1080/1570 [1:22:32<23:05, 2.83s/it]
{'loss': '0.0009618', 'grad_norm': '0.006864', 'learning_rate': '2.233e-05', 'neg_student_nll_loss': '0.001094', 'ce_loss': '0.001094', 'hpd_loss': '0.0009618', 'sampled_diff_pct': '0.05', 'adv1_neg_pct': '7.121', 'adv2_active_pct': '0.03579', 'mask3_pct': '0.03211', 'k1_gt_raw_mean': '0.0003525', 'k1_gt_raw_min': '-0.00392', 'k1_gt_raw_max': '0.01674', 'k1_gt_raw_neg_pct': '7.121', 'k1_sample_mean': '-0.02337', 'k1_sample_min': '-0.02337', 'k1_sample_max': '-0.02337', 'k1_sample_neg_pct': '4.062', 'epoch': '3.44'}
+
69%|██████▉ | 1080/1570 [1:22:32<23:05, 2.83s/it]
69%|██████▉ | 1081/1570 [1:22:35<23:07, 2.84s/it]
69%|██████▉ | 1082/1570 [1:22:38<23:04, 2.84s/it]
69%|██████▉ | 1083/1570 [1:22:41<22:57, 2.83s/it]
69%|██████▉ | 1084/1570 [1:22:44<22:59, 2.84s/it]
69%|██████▉ | 1085/1570 [1:22:47<22:53, 2.83s/it]
69%|██████▉ | 1086/1570 [1:22:49<22:48, 2.83s/it]
69%|██████▉ | 1087/1570 [1:22:52<22:49, 2.84s/it]
69%|██████▉ | 1088/1570 [1:22:55<22:44, 2.83s/it]
69%|██████▉ | 1089/1570 [1:22:58<22:46, 2.84s/it]
69%|██████▉ | 1090/1570 [1:23:01<22:39, 2.83s/it]
69%|██████▉ | 1091/1570 [1:23:04<22:30, 2.82s/it]
70%|██████▉ | 1092/1570 [1:23:06<22:28, 2.82s/it]
70%|██████▉ | 1093/1570 [1:23:09<22:26, 2.82s/it]
70%|██████▉ | 1094/1570 [1:23:12<22:28, 2.83s/it]
70%|██████▉ | 1095/1570 [1:23:15<22:19, 2.82s/it]
70%|██████▉ | 1096/1570 [1:23:18<22:18, 2.82s/it]
70%|██████▉ | 1097/1570 [1:23:21<22:19, 2.83s/it]
70%|██████▉ | 1098/1570 [1:23:23<22:20, 2.84s/it]
70%|███████ | 1099/1570 [1:23:26<22:24, 2.86s/it]
70%|███████ | 1100/1570 [1:23:29<22:19, 2.85s/it]
{'loss': '0.001107', 'grad_norm': '0.009582', 'learning_rate': '2.069e-05', 'neg_student_nll_loss': '0.001211', 'ce_loss': '0.001211', 'hpd_loss': '0.001107', 'sampled_diff_pct': '0.07695', 'adv1_neg_pct': '7.498', 'adv2_active_pct': '0.06837', 'mask3_pct': '0.06837', 'k1_gt_raw_mean': '0.0003861', 'k1_gt_raw_min': '-0.003896', 'k1_gt_raw_max': '0.01962', 'k1_gt_raw_neg_pct': '7.498', 'k1_sample_mean': '-0.03844', 'k1_sample_min': '-0.04218', 'k1_sample_max': '-0.0347', 'k1_sample_neg_pct': '5.781', 'epoch': '3.504'}
+
70%|███████ | 1100/1570 [1:23:29<22:19, 2.85s/it]
70%|███████ | 1101/1570 [1:23:32<22:19, 2.86s/it]
70%|███████ | 1102/1570 [1:23:35<22:14, 2.85s/it]
70%|███████ | 1103/1570 [1:23:38<22:08, 2.85s/it]
70%|███████ | 1104/1570 [1:23:41<22:03, 2.84s/it]
70%|███████ | 1105/1570 [1:23:43<21:58, 2.84s/it]
70%|███████ | 1106/1570 [1:23:46<21:55, 2.83s/it]
71%|███████ | 1107/1570 [1:23:49<21:54, 2.84s/it]
71%|███████ | 1108/1570 [1:23:52<21:50, 2.84s/it]
71%|███████ | 1109/1570 [1:23:55<21:46, 2.83s/it]
71%|███████ | 1110/1570 [1:23:58<21:43, 2.83s/it]
71%|███████ | 1111/1570 [1:24:00<21:35, 2.82s/it]
71%|███████ | 1112/1570 [1:24:03<21:37, 2.83s/it]
71%|███████ | 1113/1570 [1:24:06<21:42, 2.85s/it]
71%|███████ | 1114/1570 [1:24:09<21:36, 2.84s/it]
71%|███████ | 1115/1570 [1:24:12<21:35, 2.85s/it]
71%|███████ | 1116/1570 [1:24:15<21:27, 2.84s/it]
71%|███████ | 1117/1570 [1:24:17<21:21, 2.83s/it]
71%|███████ | 1118/1570 [1:24:20<21:17, 2.83s/it]
71%|███████▏ | 1119/1570 [1:24:23<21:23, 2.85s/it]
71%|███████▏ | 1120/1570 [1:24:26<21:16, 2.84s/it]
{'loss': '0.0004915', 'grad_norm': '0.07571', 'learning_rate': '1.91e-05', 'neg_student_nll_loss': '0.00116', 'ce_loss': '0.00116', 'hpd_loss': '0.0004915', 'sampled_diff_pct': '0.1085', 'adv1_neg_pct': '7.499', 'adv2_active_pct': '0.09933', 'mask3_pct': '0.09933', 'k1_gt_raw_mean': '0.0004522', 'k1_gt_raw_min': '-0.00512', 'k1_gt_raw_max': '0.02199', 'k1_gt_raw_neg_pct': '7.499', 'k1_sample_mean': '-0.06357', 'k1_sample_min': '-0.09476', 'k1_sample_max': '-0.03237', 'k1_sample_neg_pct': '6.562', 'epoch': '3.568'}
+
71%|███████▏ | 1120/1570 [1:24:26<21:16, 2.84s/it]
71%|███████▏ | 1121/1570 [1:24:29<21:19, 2.85s/it]
71%|███████▏ | 1122/1570 [1:24:32<21:17, 2.85s/it]
72%|███████▏ | 1123/1570 [1:24:34<21:11, 2.85s/it]
72%|███████▏ | 1124/1570 [1:24:37<20:58, 2.82s/it]
72%|███████▏ | 1125/1570 [1:24:40<20:52, 2.81s/it]
72%|███████▏ | 1126/1570 [1:24:43<20:53, 2.82s/it]
72%|███████▏ | 1127/1570 [1:24:46<20:49, 2.82s/it]
72%|███████▏ | 1128/1570 [1:24:49<20:50, 2.83s/it]
72%|███████▏ | 1129/1570 [1:24:51<20:46, 2.83s/it]
72%|███████▏ | 1130/1570 [1:24:54<20:45, 2.83s/it]
72%|███████▏ | 1131/1570 [1:24:57<20:49, 2.85s/it]
72%|███████▏ | 1132/1570 [1:25:00<20:41, 2.84s/it]
72%|███████▏ | 1133/1570 [1:25:03<20:37, 2.83s/it]
72%|███████▏ | 1134/1570 [1:25:06<20:34, 2.83s/it]
72%|███████▏ | 1135/1570 [1:25:08<20:36, 2.84s/it]
72%|███████▏ | 1136/1570 [1:25:11<20:31, 2.84s/it]
72%|███████▏ | 1137/1570 [1:25:14<20:23, 2.83s/it]
72%|███████▏ | 1138/1570 [1:25:17<20:25, 2.84s/it]
73%|███████▎ | 1139/1570 [1:25:20<20:20, 2.83s/it]
73%|███████▎ | 1140/1570 [1:25:23<20:25, 2.85s/it]
{'loss': '0.001302', 'grad_norm': '0.03061', 'learning_rate': '1.755e-05', 'neg_student_nll_loss': '0.001337', 'ce_loss': '0.001337', 'hpd_loss': '0.001302', 'sampled_diff_pct': '0.04095', 'adv1_neg_pct': '7.47', 'adv2_active_pct': '0.03868', 'mask3_pct': '0.03628', 'k1_gt_raw_mean': '0.0004279', 'k1_gt_raw_min': '-0.002206', 'k1_gt_raw_max': '0.01743', 'k1_gt_raw_neg_pct': '7.47', 'k1_sample_mean': '-0.02643', 'k1_sample_min': '-0.02643', 'k1_sample_max': '-0.02643', 'k1_sample_neg_pct': '4.375', 'epoch': '3.632'}
+
73%|███████▎ | 1140/1570 [1:25:23<20:25, 2.85s/it]
73%|███████▎ | 1141/1570 [1:25:26<20:27, 2.86s/it]
73%|███████▎ | 1142/1570 [1:25:28<20:20, 2.85s/it]
73%|███████▎ | 1143/1570 [1:25:31<20:15, 2.85s/it]
73%|███████▎ | 1144/1570 [1:25:34<20:12, 2.85s/it]
73%|███████▎ | 1145/1570 [1:25:37<20:08, 2.84s/it]
73%|███████▎ | 1146/1570 [1:25:40<20:07, 2.85s/it]
73%|███████▎ | 1147/1570 [1:25:43<20:03, 2.85s/it]
73%|███████▎ | 1148/1570 [1:25:45<20:01, 2.85s/it]
73%|███████▎ | 1149/1570 [1:25:48<19:56, 2.84s/it]
73%|███████▎ | 1150/1570 [1:25:51<19:52, 2.84s/it]
73%|███████▎ | 1151/1570 [1:25:54<19:47, 2.83s/it]
73%|███████▎ | 1152/1570 [1:25:57<19:40, 2.82s/it]
73%|███████▎ | 1153/1570 [1:26:00<19:35, 2.82s/it]
74%|███████▎ | 1154/1570 [1:26:02<19:32, 2.82s/it]
74%|███████▎ | 1155/1570 [1:26:05<19:32, 2.83s/it]
74%|███████▎ | 1156/1570 [1:26:08<19:25, 2.81s/it]
74%|███████▎ | 1157/1570 [1:26:11<19:22, 2.82s/it]
74%|███████▍ | 1158/1570 [1:26:14<19:23, 2.82s/it]
74%|███████▍ | 1159/1570 [1:26:16<19:23, 2.83s/it]
74%|███████▍ | 1160/1570 [1:26:19<19:18, 2.83s/it]
{'loss': '0.001421', 'grad_norm': '0.1907', 'learning_rate': '1.606e-05', 'neg_student_nll_loss': '0.002226', 'ce_loss': '0.002226', 'hpd_loss': '0.001421', 'sampled_diff_pct': '0.1271', 'adv1_neg_pct': '7.295', 'adv2_active_pct': '0.1167', 'mask3_pct': '0.1137', 'k1_gt_raw_mean': '0.0004437', 'k1_gt_raw_min': '-0.00281', 'k1_gt_raw_max': '0.0214', 'k1_gt_raw_neg_pct': '7.295', 'k1_sample_mean': '-0.02885', 'k1_sample_min': '-0.03075', 'k1_sample_max': '-0.02788', 'k1_sample_neg_pct': '4.948', 'epoch': '3.696'}
+
74%|███████▍ | 1160/1570 [1:26:19<19:18, 2.83s/it]
74%|███████▍ | 1161/1570 [1:26:23<20:46, 3.05s/it]
74%|███████▍ | 1162/1570 [1:26:26<21:37, 3.18s/it]
74%|███████▍ | 1163/1570 [1:26:30<22:11, 3.27s/it]
74%|███████▍ | 1164/1570 [1:26:33<22:38, 3.35s/it]
74%|███████▍ | 1165/1570 [1:26:37<23:02, 3.41s/it]
74%|███████▍ | 1166/1570 [1:26:40<23:16, 3.46s/it]
74%|███████▍ | 1167/1570 [1:26:44<23:24, 3.49s/it]
74%|███████▍ | 1168/1570 [1:26:48<23:30, 3.51s/it]
74%|███████▍ | 1169/1570 [1:26:51<23:27, 3.51s/it]
75%|███████▍ | 1170/1570 [1:26:55<23:23, 3.51s/it]
75%|███████▍ | 1171/1570 [1:26:58<23:22, 3.51s/it]
75%|███████▍ | 1172/1570 [1:27:02<23:08, 3.49s/it]
75%|███████▍ | 1173/1570 [1:27:05<23:02, 3.48s/it]
75%|███████▍ | 1174/1570 [1:27:09<23:13, 3.52s/it]
75%|███████▍ | 1175/1570 [1:27:12<22:54, 3.48s/it]
75%|███████▍ | 1176/1570 [1:27:15<22:44, 3.46s/it]
75%|███████▍ | 1177/1570 [1:27:19<22:45, 3.47s/it]
75%|███████▌ | 1178/1570 [1:27:23<22:53, 3.50s/it]
75%|███████▌ | 1179/1570 [1:27:26<22:54, 3.52s/it]
75%|███████▌ | 1180/1570 [1:27:30<22:51, 3.52s/it]
{'loss': '0.00157', 'grad_norm': '0.0118', 'learning_rate': '1.462e-05', 'neg_student_nll_loss': '0.002897', 'ce_loss': '0.002897', 'hpd_loss': '0.00157', 'sampled_diff_pct': '0.08027', 'adv1_neg_pct': '7.58', 'adv2_active_pct': '0.06753', 'mask3_pct': '0.06484', 'k1_gt_raw_mean': '0.0003268', 'k1_gt_raw_min': '-0.004132', 'k1_gt_raw_max': '0.01624', 'k1_gt_raw_neg_pct': '7.58', 'k1_sample_mean': '-0.08303', 'k1_sample_min': '-0.08577', 'k1_sample_max': '-0.08208', 'k1_sample_neg_pct': '3.681', 'epoch': '3.759'}
+
75%|███████▌ | 1180/1570 [1:27:30<22:51, 3.52s/it]
75%|███████▌ | 1181/1570 [1:27:33<22:53, 3.53s/it]
75%|███████▌ | 1182/1570 [1:27:37<22:50, 3.53s/it]
75%|███████▌ | 1183/1570 [1:27:40<22:46, 3.53s/it]
75%|███████▌ | 1184/1570 [1:27:44<22:44, 3.53s/it]
75%|███████▌ | 1185/1570 [1:27:47<22:45, 3.55s/it]
76%|███████▌ | 1186/1570 [1:27:51<22:40, 3.54s/it]
76%|███████▌ | 1187/1570 [1:27:54<22:41, 3.56s/it]
76%|███████▌ | 1188/1570 [1:27:58<22:38, 3.56s/it]
76%|███████▌ | 1189/1570 [1:28:02<22:33, 3.55s/it]
76%|███████▌ | 1190/1570 [1:28:05<22:29, 3.55s/it]
76%|███████▌ | 1191/1570 [1:28:09<22:28, 3.56s/it]
76%|███████▌ | 1192/1570 [1:28:12<22:16, 3.54s/it]
76%|███████▌ | 1193/1570 [1:28:16<22:21, 3.56s/it]
76%|███████▌ | 1194/1570 [1:28:19<22:21, 3.57s/it]
76%|███████▌ | 1195/1570 [1:28:23<22:11, 3.55s/it]
76%|███████▌ | 1196/1570 [1:28:26<21:39, 3.48s/it]
76%|███████▌ | 1197/1570 [1:28:30<21:46, 3.50s/it]
76%|███████▋ | 1198/1570 [1:28:33<21:45, 3.51s/it]
76%|███████▋ | 1199/1570 [1:28:37<21:33, 3.49s/it]
76%|███████▋ | 1200/1570 [1:28:40<21:28, 3.48s/it]
{'loss': '0.00137', 'grad_norm': '0.01449', 'learning_rate': '1.324e-05', 'neg_student_nll_loss': '0.002274', 'ce_loss': '0.002274', 'hpd_loss': '0.00137', 'sampled_diff_pct': '0.1825', 'adv1_neg_pct': '8.005', 'adv2_active_pct': '0.1786', 'mask3_pct': '0.1786', 'k1_gt_raw_mean': '0.0004731', 'k1_gt_raw_min': '-0.005409', 'k1_gt_raw_max': '0.02227', 'k1_gt_raw_neg_pct': '8.005', 'k1_sample_mean': '-0.1085', 'k1_sample_min': '-0.1382', 'k1_sample_max': '-0.0787', 'k1_sample_neg_pct': '6.562', 'epoch': '3.823'}
+
76%|███████▋ | 1200/1570 [1:28:40<21:28, 3.48s/it]
76%|███████▋ | 1201/1570 [1:28:44<21:37, 3.52s/it]
77%|███████▋ | 1202/1570 [1:28:47<21:37, 3.52s/it]
77%|███████▋ | 1203/1570 [1:28:51<21:31, 3.52s/it]
77%|███████▋ | 1204/1570 [1:28:54<21:34, 3.54s/it]
77%|███████▋ | 1205/1570 [1:28:58<21:37, 3.55s/it]
77%|███████▋ | 1206/1570 [1:29:01<21:27, 3.54s/it]
77%|███████▋ | 1207/1570 [1:29:05<21:26, 3.54s/it]
77%|███████▋ | 1208/1570 [1:29:09<21:19, 3.53s/it]
77%|███████▋ | 1209/1570 [1:29:12<21:14, 3.53s/it]
77%|███████▋ | 1210/1570 [1:29:16<21:11, 3.53s/it]
77%|███████▋ | 1211/1570 [1:29:19<21:13, 3.55s/it]
77%|███████▋ | 1212/1570 [1:29:23<21:10, 3.55s/it]
77%|███████▋ | 1213/1570 [1:29:26<21:11, 3.56s/it]
77%|███████▋ | 1214/1570 [1:29:30<20:50, 3.51s/it]
77%|███████▋ | 1215/1570 [1:29:33<20:40, 3.49s/it]
77%|███████▋ | 1216/1570 [1:29:37<20:42, 3.51s/it]
78%|███████▊ | 1217/1570 [1:29:40<20:45, 3.53s/it]
78%|███████▊ | 1218/1570 [1:29:44<20:46, 3.54s/it]
78%|███████▊ | 1219/1570 [1:29:47<20:46, 3.55s/it]
78%|███████▊ | 1220/1570 [1:29:51<20:45, 3.56s/it]
{'loss': '0.001496', 'grad_norm': '1.839', 'learning_rate': '1.192e-05', 'neg_student_nll_loss': '0.002031', 'ce_loss': '0.002031', 'hpd_loss': '0.001496', 'sampled_diff_pct': '0.1486', 'adv1_neg_pct': '7.971', 'adv2_active_pct': '0.1445', 'mask3_pct': '0.1445', 'k1_gt_raw_mean': '0.0005614', 'k1_gt_raw_min': '-0.003312', 'k1_gt_raw_max': '0.01991', 'k1_gt_raw_neg_pct': '7.971', 'k1_sample_mean': '-0.06567', 'k1_sample_min': '-0.06567', 'k1_sample_max': '-0.06567', 'k1_sample_neg_pct': '5.312', 'epoch': '3.887'}
+
78%|███████▊ | 1220/1570 [1:29:51<20:45, 3.56s/it]
78%|███████▊ | 1221/1570 [1:29:55<20:39, 3.55s/it]
78%|███████▊ | 1222/1570 [1:29:58<20:32, 3.54s/it]
78%|███████▊ | 1223/1570 [1:30:02<20:20, 3.52s/it]
78%|███████▊ | 1224/1570 [1:30:05<20:15, 3.51s/it]
78%|███████▊ | 1225/1570 [1:30:09<20:17, 3.53s/it]
78%|███████▊ | 1226/1570 [1:30:12<20:16, 3.54s/it]
78%|███████▊ | 1227/1570 [1:30:16<19:53, 3.48s/it]
78%|███████▊ | 1228/1570 [1:30:19<19:39, 3.45s/it]
78%|███████▊ | 1229/1570 [1:30:22<19:51, 3.50s/it]
78%|███████▊ | 1230/1570 [1:30:26<19:52, 3.51s/it]
78%|███████▊ | 1231/1570 [1:30:30<19:56, 3.53s/it]
78%|███████▊ | 1232/1570 [1:30:33<19:44, 3.50s/it]
79%|███████▊ | 1233/1570 [1:30:36<19:30, 3.47s/it]
79%|███████▊ | 1234/1570 [1:30:40<19:17, 3.44s/it]
79%|███████▊ | 1235/1570 [1:30:43<19:14, 3.44s/it]
79%|███████▊ | 1236/1570 [1:30:47<19:27, 3.50s/it]
79%|███████▉ | 1237/1570 [1:30:50<19:28, 3.51s/it]
79%|███████▉ | 1238/1570 [1:30:54<19:31, 3.53s/it]
79%|███████▉ | 1239/1570 [1:30:58<19:34, 3.55s/it]
79%|███████▉ | 1240/1570 [1:31:01<19:30, 3.55s/it]
{'loss': '0.0006963', 'grad_norm': '0.0158', 'learning_rate': '1.066e-05', 'neg_student_nll_loss': '0.0008984', 'ce_loss': '0.0008984', 'hpd_loss': '0.0006963', 'sampled_diff_pct': '0.06132', 'adv1_neg_pct': '7.753', 'adv2_active_pct': '0.05452', 'mask3_pct': '0.05452', 'k1_gt_raw_mean': '0.0002776', 'k1_gt_raw_min': '-0.002547', 'k1_gt_raw_max': '0.0175', 'k1_gt_raw_neg_pct': '7.753', 'k1_sample_mean': '-0.03892', 'k1_sample_min': '-0.03968', 'k1_sample_max': '-0.03816', 'k1_sample_neg_pct': '4.375', 'epoch': '3.951'}
+
79%|███████▉ | 1240/1570 [1:31:01<19:30, 3.55s/it]
79%|███████▉ | 1241/1570 [1:31:05<19:31, 3.56s/it]
79%|███████▉ | 1242/1570 [1:31:08<19:31, 3.57s/it]
79%|███████▉ | 1243/1570 [1:31:12<19:15, 3.53s/it]
79%|███████▉ | 1244/1570 [1:31:15<19:06, 3.52s/it]
79%|███████▉ | 1245/1570 [1:31:19<19:05, 3.52s/it]
79%|███████▉ | 1246/1570 [1:31:22<19:04, 3.53s/it]
79%|███████▉ | 1247/1570 [1:31:26<19:02, 3.54s/it]
79%|███████▉ | 1248/1570 [1:31:29<18:47, 3.50s/it]
80%|███████▉ | 1249/1570 [1:31:33<18:39, 3.49s/it]
80%|███████▉ | 1250/1570 [1:31:36<18:39, 3.50s/it]
80%|███████▉ | 1251/1570 [1:31:40<18:41, 3.52s/it]
80%|███████▉ | 1252/1570 [1:31:43<18:33, 3.50s/it]
80%|███████▉ | 1253/1570 [1:31:47<18:35, 3.52s/it]
80%|███████▉ | 1254/1570 [1:31:50<18:25, 3.50s/it]
80%|███████▉ | 1255/1570 [1:31:54<18:31, 3.53s/it]
80%|████████ | 1256/1570 [1:31:55<15:07, 2.89s/it][INFO|trainer.py:2668] 2026-08-22 23:30:30,714 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 23:30:30,714 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 23:30:30,714 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/hpd/checkpoint-1256
+[INFO|configuration_utils.py:780] 2026-08-22 23:48:55,502 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 23:48:55,505 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 23:48:55,554 >> chat template saved in results/qwen3/hpd/checkpoint-1256/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 23:48:55,554 >> tokenizer config file saved in results/qwen3/hpd/checkpoint-1256/tokenizer_config.json
+
80%|████████ | 1257/1570 [1:50:25<29:06:50, 334.86s/it]
80%|████████ | 1258/1570 [1:50:28<20:24:25, 235.47s/it]
80%|████████ | 1259/1570 [1:50:32<14:19:42, 165.86s/it]
80%|████████ | 1260/1570 [1:50:35<10:04:14, 116.95s/it]
{'loss': '0.001724', 'grad_norm': '0.05925', 'learning_rate': '9.464e-06', 'neg_student_nll_loss': '0.00513', 'ce_loss': '0.00513', 'hpd_loss': '0.003739', 'sampled_diff_pct': '0.1773', 'adv1_neg_pct': '8.281', 'adv2_active_pct': '0.1395', 'mask3_pct': '0.1395', 'k1_gt_raw_mean': '0.0003755', 'k1_gt_raw_min': '-0.009175', 'k1_gt_raw_max': '0.02487', 'k1_gt_raw_neg_pct': '8.281', 'k1_sample_mean': '-0.01378', 'k1_sample_min': '-0.02075', 'k1_sample_max': '-0.009558', 'k1_sample_neg_pct': '7.5', 'epoch': '4.013'}
+
80%|████████ | 1260/1570 [1:50:35<10:04:14, 116.95s/it]
80%|████████ | 1261/1570 [1:50:37<7:05:59, 82.72s/it]
80%|████████ | 1262/1570 [1:50:40<5:01:37, 58.76s/it]
80%|████████ | 1263/1570 [1:50:43<3:34:50, 41.99s/it]
81%|████████ | 1264/1570 [1:50:46<2:34:18, 30.26s/it]
81%|████████ | 1265/1570 [1:50:49<1:52:02, 22.04s/it]
81%|████████ | 1266/1570 [1:50:52<1:22:26, 16.27s/it]
81%|████████ | 1267/1570 [1:50:55<1:01:54, 12.26s/it]
81%|████████ | 1268/1570 [1:50:57<47:31, 9.44s/it]
81%|████████ | 1269/1570 [1:51:00<37:26, 7.46s/it]
81%|████████ | 1270/1570 [1:51:03<30:25, 6.08s/it]
81%|████████ | 1271/1570 [1:51:06<25:29, 5.11s/it]
81%|████████ | 1272/1570 [1:51:09<22:04, 4.44s/it]
81%|████████ | 1273/1570 [1:51:12<19:38, 3.97s/it]
81%|████████ | 1274/1570 [1:51:15<17:53, 3.63s/it]
81%|████████ | 1275/1570 [1:51:18<16:46, 3.41s/it]
81%|████████▏ | 1276/1570 [1:51:20<15:53, 3.24s/it]
81%|████████▏ | 1277/1570 [1:51:23<15:16, 3.13s/it]
81%|████████▏ | 1278/1570 [1:51:26<14:45, 3.03s/it]
81%|████████▏ | 1279/1570 [1:51:29<14:21, 2.96s/it]
82%|████████▏ | 1280/1570 [1:51:32<14:07, 2.92s/it]
{'loss': '0.0004134', 'grad_norm': '0.05798', 'learning_rate': '8.332e-06', 'neg_student_nll_loss': '0.0006055', 'ce_loss': '0.0006055', 'hpd_loss': '0.0004134', 'sampled_diff_pct': '0.04515', 'adv1_neg_pct': '7.801', 'adv2_active_pct': '0.04323', 'mask3_pct': '0.04323', 'k1_gt_raw_mean': '0.0002333', 'k1_gt_raw_min': '-0.003863', 'k1_gt_raw_max': '0.01553', 'k1_gt_raw_neg_pct': '7.801', 'k1_sample_mean': '-0.01468', 'k1_sample_min': '-0.01468', 'k1_sample_max': '-0.01468', 'k1_sample_neg_pct': '4.062', 'epoch': '4.077'}
+
82%|████████▏ | 1280/1570 [1:51:32<14:07, 2.92s/it]
82%|████████▏ | 1281/1570 [1:51:35<13:57, 2.90s/it]
82%|████████▏ | 1282/1570 [1:51:37<13:44, 2.86s/it]
82%|████████▏ | 1283/1570 [1:51:40<13:36, 2.85s/it]
82%|████████▏ | 1284/1570 [1:51:43<13:30, 2.84s/it]
82%|████████▏ | 1285/1570 [1:51:46<13:24, 2.82s/it]
82%|████████▏ | 1286/1570 [1:51:49<13:22, 2.83s/it]
82%|████████▏ | 1287/1570 [1:51:51<13:20, 2.83s/it]
82%|████████▏ | 1288/1570 [1:51:54<13:16, 2.82s/it]
82%|████████▏ | 1289/1570 [1:51:57<13:10, 2.81s/it]
82%|████████▏ | 1290/1570 [1:52:00<13:10, 2.82s/it]
82%|████████▏ | 1291/1570 [1:52:03<13:08, 2.83s/it]
82%|████████▏ | 1292/1570 [1:52:05<13:05, 2.83s/it]
82%|████████▏ | 1293/1570 [1:52:08<13:03, 2.83s/it]
82%|████████▏ | 1294/1570 [1:52:11<13:02, 2.83s/it]
82%|████████▏ | 1295/1570 [1:52:14<13:00, 2.84s/it]
83%|████████▎ | 1296/1570 [1:52:17<12:56, 2.84s/it]
83%|████████▎ | 1297/1570 [1:52:20<12:54, 2.84s/it]
83%|████████▎ | 1298/1570 [1:52:22<12:45, 2.81s/it]
83%|████████▎ | 1299/1570 [1:52:25<12:45, 2.83s/it]
83%|████████▎ | 1300/1570 [1:52:28<12:42, 2.83s/it]
{'loss': '0.0009472', 'grad_norm': '0.003798', 'learning_rate': '7.266e-06', 'neg_student_nll_loss': '0.0009541', 'ce_loss': '0.0009541', 'hpd_loss': '0.0009472', 'sampled_diff_pct': '0.09455', 'adv1_neg_pct': '8.048', 'adv2_active_pct': '0.08839', 'mask3_pct': '0.08839', 'k1_gt_raw_mean': '0.0002067', 'k1_gt_raw_min': '-0.006063', 'k1_gt_raw_max': '0.01332', 'k1_gt_raw_neg_pct': '8.048', 'k1_sample_mean': '-0.008535', 'k1_sample_min': '-0.008535', 'k1_sample_max': '-0.008535', 'k1_sample_neg_pct': '5.312', 'epoch': '4.14'}
+
83%|████████▎ | 1300/1570 [1:52:28<12:42, 2.83s/it]
83%|████████▎ | 1301/1570 [1:52:31<12:41, 2.83s/it]
83%|████████▎ | 1302/1570 [1:52:34<12:38, 2.83s/it]
83%|████████▎ | 1303/1570 [1:52:37<12:35, 2.83s/it]
83%|████████▎ | 1304/1570 [1:52:39<12:33, 2.83s/it]
83%|████████▎ | 1305/1570 [1:52:42<12:32, 2.84s/it]
83%|████████▎ | 1306/1570 [1:52:45<12:29, 2.84s/it]
83%|████████▎ | 1307/1570 [1:52:48<12:28, 2.85s/it]
83%|████████▎ | 1308/1570 [1:52:51<12:23, 2.84s/it]
83%|████████▎ | 1309/1570 [1:52:54<12:19, 2.83s/it]
83%|████████▎ | 1310/1570 [1:52:57<12:19, 2.84s/it]
84%|████████▎ | 1311/1570 [1:52:59<12:15, 2.84s/it]
84%|████████▎ | 1312/1570 [1:53:02<12:10, 2.83s/it]
84%|████████▎ | 1313/1570 [1:53:05<12:05, 2.82s/it]
84%|████████▎ | 1314/1570 [1:53:08<12:02, 2.82s/it]
84%|████████▍ | 1315/1570 [1:53:11<11:58, 2.82s/it]
84%|████████▍ | 1316/1570 [1:53:13<11:56, 2.82s/it]
84%|████████▍ | 1317/1570 [1:53:16<11:54, 2.82s/it]
84%|████████▍ | 1318/1570 [1:53:19<11:51, 2.83s/it]
84%|████████▍ | 1319/1570 [1:53:22<11:49, 2.83s/it]
84%|████████▍ | 1320/1570 [1:53:25<11:49, 2.84s/it]
{'loss': '0.0005514', 'grad_norm': '0.02296', 'learning_rate': '6.269e-06', 'neg_student_nll_loss': '0.0007429', 'ce_loss': '0.0007429', 'hpd_loss': '0.0005514', 'sampled_diff_pct': '0.04799', 'adv1_neg_pct': '8', 'adv2_active_pct': '0.03458', 'mask3_pct': '0.03458', 'k1_gt_raw_mean': '0.0002641', 'k1_gt_raw_min': '-0.002805', 'k1_gt_raw_max': '0.01538', 'k1_gt_raw_neg_pct': '8', 'k1_sample_mean': '-0.006288', 'k1_sample_min': '-0.006288', 'k1_sample_max': '-0.006288', 'k1_sample_neg_pct': '3.438', 'epoch': '4.204'}
+
84%|████████▍ | 1320/1570 [1:53:25<11:49, 2.84s/it]
84%|████████▍ | 1321/1570 [1:53:28<11:43, 2.83s/it]
84%|████████▍ | 1322/1570 [1:53:30<11:43, 2.84s/it]
84%|████████▍ | 1323/1570 [1:53:33<11:38, 2.83s/it]
84%|████████▍ | 1324/1570 [1:53:36<11:38, 2.84s/it]
84%|████████▍ | 1325/1570 [1:53:39<11:37, 2.85s/it]
84%|████████▍ | 1326/1570 [1:53:42<11:36, 2.85s/it]
85%|████████▍ | 1327/1570 [1:53:45<11:31, 2.85s/it]
85%|████████▍ | 1328/1570 [1:53:48<11:28, 2.84s/it]
85%|████████▍ | 1329/1570 [1:53:50<11:27, 2.85s/it]
85%|████████▍ | 1330/1570 [1:53:53<11:21, 2.84s/it]
85%|████████▍ | 1331/1570 [1:53:56<11:14, 2.82s/it]
85%|████████▍ | 1332/1570 [1:53:59<11:14, 2.84s/it]
85%|████████▍ | 1333/1570 [1:54:02<11:13, 2.84s/it]
85%|████████▍ | 1334/1570 [1:54:05<11:07, 2.83s/it]
85%|████████▌ | 1335/1570 [1:54:07<11:04, 2.83s/it]
85%|████████▌ | 1336/1570 [1:54:10<10:59, 2.82s/it]
85%|████████▌ | 1337/1570 [1:54:13<11:00, 2.83s/it]
85%|████████▌ | 1338/1570 [1:54:16<10:56, 2.83s/it]
85%|████████▌ | 1339/1570 [1:54:19<10:54, 2.83s/it]
85%|████████▌ | 1340/1570 [1:54:22<10:53, 2.84s/it]
{'loss': '0.001055', 'grad_norm': '0.05514', 'learning_rate': '5.342e-06', 'neg_student_nll_loss': '0.0009998', 'ce_loss': '0.0009998', 'hpd_loss': '0.001055', 'sampled_diff_pct': '0.03518', 'adv1_neg_pct': '8.061', 'adv2_active_pct': '0.02886', 'mask3_pct': '0.02886', 'k1_gt_raw_mean': '0.0003266', 'k1_gt_raw_min': '-0.00255', 'k1_gt_raw_max': '0.01425', 'k1_gt_raw_neg_pct': '8.061', 'k1_sample_mean': '-0.02303', 'k1_sample_min': '-0.02303', 'k1_sample_max': '-0.02303', 'k1_sample_neg_pct': '3.438', 'epoch': '4.268'}
+
85%|████████▌ | 1340/1570 [1:54:22<10:53, 2.84s/it]
85%|████████▌ | 1341/1570 [1:54:24<10:50, 2.84s/it]
85%|████████▌ | 1342/1570 [1:54:27<10:50, 2.85s/it]
86%|████████▌ | 1343/1570 [1:54:30<10:49, 2.86s/it]
86%|████████▌ | 1344/1570 [1:54:33<10:44, 2.85s/it]
86%|████████▌ | 1345/1570 [1:54:36<10:43, 2.86s/it]
86%|████████▌ | 1346/1570 [1:54:39<10:40, 2.86s/it]
86%|████████▌ | 1347/1570 [1:54:42<10:37, 2.86s/it]
86%|████████▌ | 1348/1570 [1:54:44<10:33, 2.85s/it]
86%|████████▌ | 1349/1570 [1:54:47<10:29, 2.85s/it]
86%|████████▌ | 1350/1570 [1:54:50<10:26, 2.85s/it]
86%|████████▌ | 1351/1570 [1:54:53<10:22, 2.84s/it]
86%|████████▌ | 1352/1570 [1:54:56<10:22, 2.85s/it]
86%|████████▌ | 1353/1570 [1:54:59<10:17, 2.85s/it]
86%|████████▌ | 1354/1570 [1:55:01<10:16, 2.85s/it]
86%|████████▋ | 1355/1570 [1:55:04<10:12, 2.85s/it]
86%|████████▋ | 1356/1570 [1:55:07<10:05, 2.83s/it]
86%|████████▋ | 1357/1570 [1:55:10<10:02, 2.83s/it]
86%|████████▋ | 1358/1570 [1:55:13<09:59, 2.83s/it]
87%|████████▋ | 1359/1570 [1:55:16<09:57, 2.83s/it]
87%|████████▋ | 1360/1570 [1:55:18<09:57, 2.85s/it]
{'loss': '0.0008158', 'grad_norm': '0.02074', 'learning_rate': '4.486e-06', 'neg_student_nll_loss': '0.000963', 'ce_loss': '0.000963', 'hpd_loss': '0.0008158', 'sampled_diff_pct': '0.03094', 'adv1_neg_pct': '8.274', 'adv2_active_pct': '0.0227', 'mask3_pct': '0.0227', 'k1_gt_raw_mean': '0.000417', 'k1_gt_raw_min': '-0.003592', 'k1_gt_raw_max': '0.01817', 'k1_gt_raw_neg_pct': '8.274', 'k1_sample_mean': '-0.006042', 'k1_sample_min': '-0.006042', 'k1_sample_max': '-0.006042', 'k1_sample_neg_pct': '2.812', 'epoch': '4.332'}
+
87%|████████▋ | 1360/1570 [1:55:18<09:57, 2.85s/it]
87%|████████▋ | 1361/1570 [1:55:21<09:54, 2.85s/it]
87%|████████▋ | 1362/1570 [1:55:24<09:48, 2.83s/it]
87%|████████▋ | 1363/1570 [1:55:27<09:43, 2.82s/it]
87%|████████▋ | 1364/1570 [1:55:30<09:43, 2.83s/it]
87%|████████▋ | 1365/1570 [1:55:33<09:40, 2.83s/it]
87%|████████▋ | 1366/1570 [1:55:35<09:38, 2.84s/it]
87%|████████▋ | 1367/1570 [1:55:38<09:37, 2.84s/it]
87%|████████▋ | 1368/1570 [1:55:42<09:56, 2.95s/it]
87%|████████▋ | 1369/1570 [1:55:44<09:44, 2.91s/it]
87%|████████▋ | 1370/1570 [1:55:47<09:38, 2.89s/it]
87%|████████▋ | 1371/1570 [1:55:50<09:33, 2.88s/it]
87%|████████▋ | 1372/1570 [1:55:53<09:30, 2.88s/it]
87%|████████▋ | 1373/1570 [1:55:56<09:24, 2.87s/it]
88%|████████▊ | 1374/1570 [1:55:59<09:19, 2.86s/it]
88%|████████▊ | 1375/1570 [1:56:01<09:18, 2.86s/it]
88%|████████▊ | 1376/1570 [1:56:04<09:13, 2.85s/it]
88%|████████▊ | 1377/1570 [1:56:07<09:05, 2.83s/it]
88%|████████▊ | 1378/1570 [1:56:10<09:30, 2.97s/it]
88%|████████▊ | 1379/1570 [1:56:13<09:22, 2.95s/it]
88%|████████▊ | 1380/1570 [1:56:16<09:15, 2.92s/it]
{'loss': '0.0007023', 'grad_norm': '0.03812', 'learning_rate': '3.704e-06', 'neg_student_nll_loss': '0.001035', 'ce_loss': '0.001035', 'hpd_loss': '0.0007023', 'sampled_diff_pct': '0.05637', 'adv1_neg_pct': '7.839', 'adv2_active_pct': '0.04923', 'mask3_pct': '0.04923', 'k1_gt_raw_mean': '0.0002174', 'k1_gt_raw_min': '-0.00536', 'k1_gt_raw_max': '0.01398', 'k1_gt_raw_neg_pct': '7.839', 'k1_sample_mean': '-0.04113', 'k1_sample_min': '-0.04113', 'k1_sample_max': '-0.04113', 'k1_sample_neg_pct': '4.688', 'epoch': '4.396'}
+
88%|████████▊ | 1380/1570 [1:56:16<09:15, 2.92s/it]
88%|████████▊ | 1381/1570 [1:56:19<09:09, 2.91s/it]
88%|████████▊ | 1382/1570 [1:56:22<09:02, 2.89s/it]
88%|████████▊ | 1383/1570 [1:56:25<08:58, 2.88s/it]
88%|████████▊ | 1384/1570 [1:56:28<08:51, 2.86s/it]
88%|████████▊ | 1385/1570 [1:56:30<08:49, 2.86s/it]
88%|████████▊ | 1386/1570 [1:56:33<08:43, 2.84s/it]
88%|████████▊ | 1387/1570 [1:56:36<08:40, 2.84s/it]
88%|████████▊ | 1388/1570 [1:56:40<09:16, 3.06s/it]
88%|████████▊ | 1389/1570 [1:56:43<09:42, 3.22s/it]
89%|████████▊ | 1390/1570 [1:56:47<09:54, 3.30s/it]
89%|████████▊ | 1391/1570 [1:56:50<09:57, 3.34s/it]
89%|████████▊ | 1392/1570 [1:56:54<09:59, 3.37s/it]
89%|████████▊ | 1393/1570 [1:56:57<10:05, 3.42s/it]
89%|████████▉ | 1394/1570 [1:57:01<10:07, 3.45s/it]
89%|████████▉ | 1395/1570 [1:57:04<10:10, 3.49s/it]
89%|████████▉ | 1396/1570 [1:57:08<10:11, 3.51s/it]
89%|████████▉ | 1397/1570 [1:57:11<10:05, 3.50s/it]
89%|████████▉ | 1398/1570 [1:57:15<10:06, 3.53s/it]
89%|████████▉ | 1399/1570 [1:57:18<10:00, 3.51s/it]
89%|████████▉ | 1400/1570 [1:57:22<09:59, 3.53s/it]
{'loss': '0.001596', 'grad_norm': '0.04451', 'learning_rate': '2.996e-06', 'neg_student_nll_loss': '0.002534', 'ce_loss': '0.002534', 'hpd_loss': '0.001596', 'sampled_diff_pct': '0.0759', 'adv1_neg_pct': '8.549', 'adv2_active_pct': '0.06189', 'mask3_pct': '0.05998', 'k1_gt_raw_mean': '0.0002977', 'k1_gt_raw_min': '-0.003343', 'k1_gt_raw_max': '0.01712', 'k1_gt_raw_neg_pct': '8.549', 'k1_sample_mean': '-0.02025', 'k1_sample_min': '-0.02279', 'k1_sample_max': '-0.01877', 'k1_sample_neg_pct': '4.7', 'epoch': '4.46'}
+
89%|████████▉ | 1400/1570 [1:57:22<09:59, 3.53s/it]
89%|████████▉ | 1401/1570 [1:57:25<09:58, 3.54s/it]
89%|████████▉ | 1402/1570 [1:57:29<09:55, 3.55s/it]
89%|████████▉ | 1403/1570 [1:57:33<09:53, 3.56s/it]
89%|████████▉ | 1404/1570 [1:57:36<09:51, 3.56s/it]
89%|████████▉ | 1405/1570 [1:57:40<09:49, 3.57s/it]
90%|████████▉ | 1406/1570 [1:57:43<09:46, 3.58s/it]
90%|████████▉ | 1407/1570 [1:57:47<09:40, 3.56s/it]
90%|████████▉ | 1408/1570 [1:57:50<09:03, 3.35s/it]
90%|████████▉ | 1409/1570 [1:57:53<08:36, 3.21s/it]
90%|████████▉ | 1410/1570 [1:57:55<08:17, 3.11s/it]
90%|████████▉ | 1411/1570 [1:57:58<08:00, 3.02s/it]
90%|████████▉ | 1412/1570 [1:58:01<07:50, 2.98s/it]
90%|█████████ | 1413/1570 [1:58:04<07:41, 2.94s/it]
90%|█████████ | 1414/1570 [1:58:07<07:35, 2.92s/it]
90%|█████████ | 1415/1570 [1:58:10<07:31, 2.91s/it]
90%|█████████ | 1416/1570 [1:58:13<07:24, 2.89s/it]
90%|█████████ | 1417/1570 [1:58:15<07:18, 2.87s/it]
90%|█████████ | 1418/1570 [1:58:18<07:14, 2.86s/it]
90%|█████████ | 1419/1570 [1:58:21<07:10, 2.85s/it]
90%|█████████ | 1420/1570 [1:58:24<07:09, 2.86s/it]
{'loss': '0.001859', 'grad_norm': '0.03609', 'learning_rate': '2.363e-06', 'neg_student_nll_loss': '0.001653', 'ce_loss': '0.001653', 'hpd_loss': '0.001859', 'sampled_diff_pct': '0.04169', 'adv1_neg_pct': '8.14', 'adv2_active_pct': '0.02855', 'mask3_pct': '0.02855', 'k1_gt_raw_mean': '0.0004284', 'k1_gt_raw_min': '-0.004388', 'k1_gt_raw_max': '0.01588', 'k1_gt_raw_neg_pct': '8.14', 'k1_sample_mean': '-0.01924', 'k1_sample_min': '-0.01924', 'k1_sample_max': '-0.01924', 'k1_sample_neg_pct': '3.125', 'epoch': '4.523'}
+
90%|█████████ | 1420/1570 [1:58:24<07:09, 2.86s/it]
91%|█████████ | 1421/1570 [1:58:27<07:05, 2.86s/it]
91%|█████████ | 1422/1570 [1:58:30<07:01, 2.85s/it]
91%|█████████ | 1423/1570 [1:58:32<06:57, 2.84s/it]
91%|█████████ | 1424/1570 [1:58:35<06:56, 2.85s/it]
91%|█████████ | 1425/1570 [1:58:38<06:54, 2.86s/it]
91%|█████████ | 1426/1570 [1:58:41<06:51, 2.86s/it]
91%|█████████ | 1427/1570 [1:58:44<06:46, 2.84s/it]
91%|█████████ | 1428/1570 [1:58:47<06:43, 2.84s/it]
91%|█████████ | 1429/1570 [1:58:50<06:39, 2.84s/it]
91%|█████████ | 1430/1570 [1:58:52<06:37, 2.84s/it]
91%|█████████ | 1431/1570 [1:58:55<06:34, 2.84s/it]
91%|█████████ | 1432/1570 [1:58:58<06:32, 2.85s/it]
91%|█████████▏| 1433/1570 [1:59:01<06:30, 2.85s/it]
91%|█████████▏| 1434/1570 [1:59:04<06:30, 2.87s/it]
91%|█████████▏| 1435/1570 [1:59:07<06:27, 2.87s/it]
91%|█████████▏| 1436/1570 [1:59:10<06:22, 2.86s/it]
92%|█████████▏| 1437/1570 [1:59:12<06:20, 2.86s/it]
92%|█████████▏| 1438/1570 [1:59:15<06:18, 2.87s/it]
92%|█████████▏| 1439/1570 [1:59:18<06:14, 2.86s/it]
92%|█████████▏| 1440/1570 [1:59:21<06:11, 2.85s/it]
{'loss': '0.000685', 'grad_norm': '0.01626', 'learning_rate': '1.806e-06', 'neg_student_nll_loss': '0.000753', 'ce_loss': '0.000753', 'hpd_loss': '0.000685', 'sampled_diff_pct': '0.06721', 'adv1_neg_pct': '8.328', 'adv2_active_pct': '0.05099', 'mask3_pct': '0.05099', 'k1_gt_raw_mean': '0.0002936', 'k1_gt_raw_min': '-0.003724', 'k1_gt_raw_max': '0.01486', 'k1_gt_raw_neg_pct': '8.328', 'k1_sample_mean': '-0.02844', 'k1_sample_min': '-0.02844', 'k1_sample_max': '-0.02844', 'k1_sample_neg_pct': '4.688', 'epoch': '4.587'}
+
92%|█████████▏| 1440/1570 [1:59:21<06:11, 2.85s/it]
92%|█████████▏| 1441/1570 [1:59:24<06:09, 2.86s/it]
92%|█████████▏| 1442/1570 [1:59:27<06:03, 2.84s/it]
92%|█████████▏| 1443/1570 [1:59:30<06:00, 2.84s/it]
92%|█████████▏| 1444/1570 [1:59:32<05:58, 2.85s/it]
92%|█████████▏| 1445/1570 [1:59:35<05:55, 2.84s/it]
92%|█████████▏| 1446/1570 [1:59:38<05:52, 2.85s/it]
92%|█████████▏| 1447/1570 [1:59:41<05:50, 2.85s/it]
92%|█████████▏| 1448/1570 [1:59:44<05:47, 2.85s/it]
92%|█████████▏| 1449/1570 [1:59:47<05:43, 2.84s/it]
92%|█████████▏| 1450/1570 [1:59:49<05:40, 2.84s/it]
92%|█████████▏| 1451/1570 [1:59:52<05:40, 2.86s/it]
92%|█████████▏| 1452/1570 [1:59:55<05:35, 2.85s/it]
93%|█████████▎| 1453/1570 [1:59:58<05:34, 2.86s/it]
93%|█████████▎| 1454/1570 [2:00:01<05:31, 2.86s/it]
93%|█████████▎| 1455/1570 [2:00:04<05:27, 2.85s/it]
93%|█████████▎| 1456/1570 [2:00:07<05:24, 2.85s/it]
93%|█████████▎| 1457/1570 [2:00:09<05:20, 2.84s/it]
93%|█████████▎| 1458/1570 [2:00:12<05:18, 2.85s/it]
93%|█████████▎| 1459/1570 [2:00:15<05:15, 2.84s/it]
93%|█████████▎| 1460/1570 [2:00:18<05:12, 2.84s/it]
{'loss': '0.0007729', 'grad_norm': '0.7523', 'learning_rate': '1.327e-06', 'neg_student_nll_loss': '0.0007837', 'ce_loss': '0.0007837', 'hpd_loss': '0.0007729', 'sampled_diff_pct': '0.04776', 'adv1_neg_pct': '7.96', 'adv2_active_pct': '0.03355', 'mask3_pct': '0.03355', 'k1_gt_raw_mean': '0.0002109', 'k1_gt_raw_min': '-0.003532', 'k1_gt_raw_max': '0.01454', 'k1_gt_raw_neg_pct': '7.96', 'k1_sample_mean': '-0.03664', 'k1_sample_min': '-0.06582', 'k1_sample_max': '-0.007456', 'k1_sample_neg_pct': '3.281', 'epoch': '4.651'}
+
93%|█████████▎| 1460/1570 [2:00:18<05:12, 2.84s/it]
93%|█████████▎| 1461/1570 [2:00:21<05:08, 2.83s/it]
93%|█████████▎| 1462/1570 [2:00:24<05:05, 2.83s/it]
93%|█████████▎| 1463/1570 [2:00:26<05:03, 2.84s/it]
93%|█████████▎| 1464/1570 [2:00:29<05:01, 2.84s/it]
93%|█████████▎| 1465/1570 [2:00:32<04:58, 2.84s/it]
93%|█████████▎| 1466/1570 [2:00:35<04:56, 2.85s/it]
93%|█████████▎| 1467/1570 [2:00:38<04:52, 2.84s/it]
94%|█████████▎| 1468/1570 [2:00:41<04:50, 2.85s/it]
94%|█████████▎| 1469/1570 [2:00:43<04:46, 2.84s/it]
94%|█████████▎| 1470/1570 [2:00:46<04:44, 2.85s/it]
94%|█████████▎| 1471/1570 [2:00:49<04:41, 2.84s/it]
94%|█████████▍| 1472/1570 [2:00:52<04:37, 2.83s/it]
94%|█████████▍| 1473/1570 [2:00:55<04:34, 2.83s/it]
94%|█████████▍| 1474/1570 [2:00:58<04:31, 2.83s/it]
94%|█████████▍| 1475/1570 [2:01:00<04:28, 2.83s/it]
94%|█████████▍| 1476/1570 [2:01:03<04:25, 2.83s/it]
94%|█████████▍| 1477/1570 [2:01:06<04:22, 2.83s/it]
94%|█████████▍| 1478/1570 [2:01:09<04:20, 2.83s/it]
94%|█████████▍| 1479/1570 [2:01:12<04:16, 2.82s/it]
94%|█████████▍| 1480/1570 [2:01:15<04:13, 2.82s/it]
{'loss': '0.0004799', 'grad_norm': '0.006932', 'learning_rate': '9.258e-07', 'neg_student_nll_loss': '0.0005918', 'ce_loss': '0.0005918', 'hpd_loss': '0.0004799', 'sampled_diff_pct': '0.0297', 'adv1_neg_pct': '8.404', 'adv2_active_pct': '0.02325', 'mask3_pct': '0.02325', 'k1_gt_raw_mean': '0.0001835', 'k1_gt_raw_min': '-0.004523', 'k1_gt_raw_max': '0.01146', 'k1_gt_raw_neg_pct': '8.404', 'k1_sample_mean': '-0.008638', 'k1_sample_min': '-0.01221', 'k1_sample_max': '-0.005946', 'k1_sample_neg_pct': '2.708', 'epoch': '4.715'}
+
94%|█████████▍| 1480/1570 [2:01:15<04:13, 2.82s/it]
94%|█████████▍| 1481/1570 [2:01:17<04:11, 2.83s/it]
94%|█████████▍| 1482/1570 [2:01:20<04:07, 2.81s/it]
94%|█████████▍| 1483/1570 [2:01:23<04:05, 2.82s/it]
95%|█████████▍| 1484/1570 [2:01:26<04:03, 2.83s/it]
95%|█████████▍| 1485/1570 [2:01:29<03:59, 2.82s/it]
95%|█████████▍| 1486/1570 [2:01:31<03:56, 2.82s/it]
95%|█████████▍| 1487/1570 [2:01:34<03:53, 2.81s/it]
95%|█████████▍| 1488/1570 [2:01:37<03:52, 2.83s/it]
95%|█████████▍| 1489/1570 [2:01:40<03:49, 2.83s/it]
95%|█████████▍| 1490/1570 [2:01:43<03:47, 2.84s/it]
95%|█████████▍| 1491/1570 [2:01:46<03:43, 2.83s/it]
95%|█████████▌| 1492/1570 [2:01:48<03:40, 2.83s/it]
95%|█████████▌| 1493/1570 [2:01:51<03:37, 2.83s/it]
95%|█████████▌| 1494/1570 [2:01:54<03:35, 2.83s/it]
95%|█████████▌| 1495/1570 [2:01:57<03:31, 2.82s/it]
95%|█████████▌| 1496/1570 [2:02:00<03:29, 2.83s/it]
95%|█████████▌| 1497/1570 [2:02:03<03:27, 2.84s/it]
95%|█████████▌| 1498/1570 [2:02:05<03:24, 2.84s/it]
95%|█████████▌| 1499/1570 [2:02:08<03:21, 2.84s/it]
96%|█████████▌| 1500/1570 [2:02:11<03:17, 2.82s/it]
{'loss': '0.001218', 'grad_norm': '0.08339', 'learning_rate': '6.033e-07', 'neg_student_nll_loss': '0.001714', 'ce_loss': '0.001714', 'hpd_loss': '0.001218', 'sampled_diff_pct': '0.05943', 'adv1_neg_pct': '7.901', 'adv2_active_pct': '0.05479', 'mask3_pct': '0.05479', 'k1_gt_raw_mean': '0.000339', 'k1_gt_raw_min': '-0.001879', 'k1_gt_raw_max': '0.01581', 'k1_gt_raw_neg_pct': '7.901', 'k1_sample_mean': '-0.02366', 'k1_sample_min': '-0.02366', 'k1_sample_max': '-0.02366', 'k1_sample_neg_pct': '5.938', 'epoch': '4.779'}
+
96%|█████████▌| 1500/1570 [2:02:11<03:17, 2.82s/it]
96%|█████████▌| 1501/1570 [2:02:14<03:15, 2.83s/it]
96%|█████████▌| 1502/1570 [2:02:17<03:11, 2.82s/it]
96%|█████████▌| 1503/1570 [2:02:20<03:08, 2.82s/it]
96%|█████████▌| 1504/1570 [2:02:22<03:07, 2.83s/it]
96%|█████████▌| 1505/1570 [2:02:25<03:04, 2.84s/it]
96%|█████████▌| 1506/1570 [2:02:28<03:02, 2.85s/it]
96%|█████████▌| 1507/1570 [2:02:31<02:59, 2.84s/it]
96%|█████████▌| 1508/1570 [2:02:34<02:56, 2.84s/it]
96%|█████████▌| 1509/1570 [2:02:37<02:53, 2.84s/it]
96%|█████████▌| 1510/1570 [2:02:39<02:50, 2.84s/it]
96%|█████████▌| 1511/1570 [2:02:42<02:48, 2.85s/it]
96%|█████████▋| 1512/1570 [2:02:45<02:45, 2.85s/it]
96%|█████████▋| 1513/1570 [2:02:48<02:41, 2.84s/it]
96%|█████████▋| 1514/1570 [2:02:51<02:38, 2.84s/it]
96%|█████████▋| 1515/1570 [2:02:54<02:36, 2.84s/it]
97%|█████████▋| 1516/1570 [2:02:57<02:32, 2.83s/it]
97%|█████████▋| 1517/1570 [2:02:59<02:31, 2.85s/it]
97%|█████████▋| 1518/1570 [2:03:02<02:28, 2.85s/it]
97%|█████████▋| 1519/1570 [2:03:05<02:25, 2.86s/it]
97%|█████████▋| 1520/1570 [2:03:08<02:21, 2.84s/it]
{'loss': '0.007525', 'grad_norm': '0.01087', 'learning_rate': '3.599e-07', 'neg_student_nll_loss': '0.004173', 'ce_loss': '0.004173', 'hpd_loss': '0.007525', 'sampled_diff_pct': '0.1114', 'adv1_neg_pct': '8.26', 'adv2_active_pct': '0.09987', 'mask3_pct': '0.09987', 'k1_gt_raw_mean': '0.0002368', 'k1_gt_raw_min': '-0.003339', 'k1_gt_raw_max': '0.01568', 'k1_gt_raw_neg_pct': '8.26', 'k1_sample_mean': '-0.05138', 'k1_sample_min': '-0.05283', 'k1_sample_max': '-0.04992', 'k1_sample_neg_pct': '5.625', 'epoch': '4.842'}
+
97%|█████████▋| 1520/1570 [2:03:08<02:21, 2.84s/it]
97%|█████████▋| 1521/1570 [2:03:11<02:19, 2.85s/it]
97%|█████████▋| 1522/1570 [2:03:14<02:17, 2.86s/it]
97%|█████████▋| 1523/1570 [2:03:17<02:14, 2.87s/it]
97%|█████████▋| 1524/1570 [2:03:19<02:11, 2.87s/it]
97%|█████████▋| 1525/1570 [2:03:22<02:08, 2.86s/it]
97%|█████████▋| 1526/1570 [2:03:25<02:05, 2.86s/it]
97%|█████████▋| 1527/1570 [2:03:28<02:03, 2.86s/it]
97%|█████████▋| 1528/1570 [2:03:31<02:00, 2.86s/it]
97%|█████████▋| 1529/1570 [2:03:34<01:57, 2.85s/it]
97%|█████████▋| 1530/1570 [2:03:37<01:54, 2.86s/it]
98%|█████████▊| 1531/1570 [2:03:39<01:51, 2.85s/it]
98%|█████████▊| 1532/1570 [2:03:42<01:48, 2.84s/it]
98%|█████████▊| 1533/1570 [2:03:45<01:45, 2.85s/it]
98%|█████████▊| 1534/1570 [2:03:48<01:42, 2.83s/it]
98%|█████████▊| 1535/1570 [2:03:51<01:39, 2.85s/it]
98%|█████████▊| 1536/1570 [2:03:54<01:37, 2.86s/it]
98%|█████████▊| 1537/1570 [2:03:57<01:34, 2.87s/it]
98%|█████████▊| 1538/1570 [2:03:59<01:31, 2.86s/it]
98%|█████████▊| 1539/1570 [2:04:02<01:28, 2.85s/it]
98%|█████████▊| 1540/1570 [2:04:05<01:25, 2.85s/it]
{'loss': '0.0006394', 'grad_norm': '0.09346', 'learning_rate': '1.961e-07', 'neg_student_nll_loss': '0.0007239', 'ce_loss': '0.0007239', 'hpd_loss': '0.0006394', 'sampled_diff_pct': '0.04649', 'adv1_neg_pct': '8.342', 'adv2_active_pct': '0.04412', 'mask3_pct': '0.04412', 'k1_gt_raw_mean': '0.0001652', 'k1_gt_raw_min': '-0.004152', 'k1_gt_raw_max': '0.01238', 'k1_gt_raw_neg_pct': '8.342', 'k1_sample_mean': '-0.009642', 'k1_sample_min': '-0.0129', 'k1_sample_max': '-0.007577', 'k1_sample_neg_pct': '4.688', 'epoch': '4.906'}
+
98%|█████████▊| 1540/1570 [2:04:05<01:25, 2.85s/it]
98%|█████████▊| 1541/1570 [2:04:08<01:22, 2.86s/it]
98%|█████████▊| 1542/1570 [2:04:11<01:19, 2.84s/it]
98%|█████████▊| 1543/1570 [2:04:14<01:17, 2.86s/it]
98%|█████████▊| 1544/1570 [2:04:16<01:14, 2.85s/it]
98%|█████████▊| 1545/1570 [2:04:19<01:11, 2.86s/it]
98%|█████████▊| 1546/1570 [2:04:22<01:08, 2.85s/it]
99%|█████████▊| 1547/1570 [2:04:25<01:05, 2.86s/it]
99%|█████████▊| 1548/1570 [2:04:28<01:02, 2.85s/it]
99%|█████████▊| 1549/1570 [2:04:31<01:00, 2.87s/it]
99%|█████████▊| 1550/1570 [2:04:34<00:57, 2.87s/it]
99%|█████████▉| 1551/1570 [2:04:37<00:54, 2.86s/it]
99%|█████████▉| 1552/1570 [2:04:39<00:51, 2.85s/it]
99%|█████████▉| 1553/1570 [2:04:42<00:48, 2.85s/it]
99%|█████████▉| 1554/1570 [2:04:45<00:45, 2.85s/it]
99%|█████████▉| 1555/1570 [2:04:48<00:42, 2.85s/it]
99%|█████████▉| 1556/1570 [2:04:51<00:39, 2.85s/it]
99%|█████████▉| 1557/1570 [2:04:54<00:37, 2.85s/it]
99%|█████████▉| 1558/1570 [2:04:56<00:34, 2.84s/it]
99%|█████████▉| 1559/1570 [2:04:59<00:31, 2.83s/it]
99%|█████████▉| 1560/1570 [2:05:02<00:28, 2.83s/it]
{'loss': '0.001344', 'grad_norm': '0.0289', 'learning_rate': '1.121e-07', 'neg_student_nll_loss': '0.001185', 'ce_loss': '0.001185', 'hpd_loss': '0.001344', 'sampled_diff_pct': '0.07132', 'adv1_neg_pct': '8.459', 'adv2_active_pct': '0.06292', 'mask3_pct': '0.0578', 'k1_gt_raw_mean': '0.0003474', 'k1_gt_raw_min': '-0.00367', 'k1_gt_raw_max': '0.01375', 'k1_gt_raw_neg_pct': '8.459', 'k1_sample_mean': '-0.0103', 'k1_sample_min': '-0.01135', 'k1_sample_max': '-0.009253', 'k1_sample_neg_pct': '2.344', 'epoch': '4.97'}
+
99%|█████████▉| 1560/1570 [2:05:02<00:28, 2.83s/it]
99%|█████████▉| 1561/1570 [2:05:05<00:25, 2.85s/it]
99%|█████████▉| 1562/1570 [2:05:08<00:22, 2.85s/it]
100%|█████████▉| 1563/1570 [2:05:11<00:19, 2.84s/it]
100%|█████████▉| 1564/1570 [2:05:13<00:17, 2.84s/it]
100%|█████████▉| 1565/1570 [2:05:16<00:14, 2.85s/it]
100%|█████████▉| 1566/1570 [2:05:19<00:11, 2.86s/it]
100%|█████████▉| 1567/1570 [2:05:22<00:08, 2.86s/it]
100%|█████████▉| 1568/1570 [2:05:25<00:05, 2.86s/it]
100%|█████████▉| 1569/1570 [2:05:28<00:02, 2.87s/it]
100%|██████████| 1570/1570 [2:05:29<00:00, 2.35s/it][INFO|trainer.py:2668] 2026-08-23 00:04:04,361 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 00:04:04,361 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 00:04:04,361 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/hpd/checkpoint-1570
+[INFO|configuration_utils.py:780] 2026-08-23 00:13:58,705 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 00:13:58,709 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 00:13:58,745 >> chat template saved in results/qwen3/hpd/checkpoint-1570/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 00:13:58,746 >> tokenizer config file saved in results/qwen3/hpd/checkpoint-1570/tokenizer_config.json
+[INFO|trainer.py:1823] 2026-08-23 00:13:59,563 >>
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
{'train_runtime': '8125', 'train_samples_per_second': '6.171', 'train_steps_per_second': '0.193', 'train_loss': '0.01828', 'epoch': '5'}
+
100%|██████████| 1570/1570 [2:15:24<00:00, 2.35s/it]
100%|██████████| 1570/1570 [2:15:24<00:00, 5.17s/it]
+[INFO|trainer.py:3815] 2026-08-23 00:14:00,028 >> Saving model checkpoint to results/qwen3/hpd
+[INFO|configuration_utils.py:780] 2026-08-23 00:14:00,519 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-23 00:14:00,522 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-23 00:14:00,559 >> chat template saved in results/qwen3/hpd/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-23 00:14:00,559 >> tokenizer config file saved in results/qwen3/hpd/tokenizer_config.json
+[transformers] The following generation flags are not valid and may be ignored: ['temperature', 'top_p']. Set `TRANSFORMERS_VERBOSITY=info` for more details.
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+***** train metrics *****
+ epoch = 5.0
+ total_flos = 54682091GF
+ train_loss = 0.0183
+ train_runtime = 2:15:24.69
+ train_samples_per_second = 6.171
+ train_steps_per_second = 0.193
+[INFO|trainer.py:2668] 2026-08-23 00:14:00,692 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-23 00:14:00,692 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-23 00:14:00,692 >> Batch size = 16
+[WARNING|logging.py:340] 2026-08-23 00:14:00,810 >> The following generation flags are not valid and may be ignored: ['temperature', 'top_p'].
+[INFO|logging.py:355] 2026-08-23 00:14:00,810 >> - `temperature`: `do_sample` is not set to `True`. However, `temperature` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.
+- `top_p`: `do_sample` is not set to `True`. However, `top_p` is set to `0.7` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.
+If you're using a pretrained model, note that some of these attributes may be set through the model's `generation_config.json` file.
+[WARNING|utils.py:1642] 2026-08-23 00:14:00,812 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
0%| | 0/107 [00:00> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
2%|▏ | 2/107 [00:05<04:45, 2.72s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:14:11,863 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
3%|▎ | 3/107 [00:10<06:37, 3.82s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:14:17,227 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
4%|▎ | 4/107 [00:16<07:38, 4.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:14:22,729 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
5%|▍ | 5/107 [00:21<08:07, 4.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:14:28,143 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
6%|▌ | 6/107 [00:26<08:02, 4.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:14:32,916 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 7/107 [00:31<08:16, 4.97s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:14:38,284 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 8/107 [00:37<08:23, 5.08s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:14:43,621 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
8%|▊ | 9/107 [00:43<08:42, 5.34s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:14:49,513 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
9%|▉ | 10/107 [00:48<08:45, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:14:55,106 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
10%|█ | 11/107 [00:53<08:14, 5.15s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:14:59,664 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
11%|█ | 12/107 [00:58<08:19, 5.26s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:15:05,174 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
12%|█▏ | 13/107 [01:04<08:18, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:15:10,580 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
13%|█▎ | 14/107 [01:09<08:10, 5.28s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:15:15,791 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
14%|█▍ | 15/107 [01:14<08:11, 5.34s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:15:21,274 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
15%|█▍ | 16/107 [01:20<08:08, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:15:26,726 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
16%|█▌ | 17/107 [01:26<08:12, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:15:32,446 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
17%|█▋ | 18/107 [01:31<08:08, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:15:37,974 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
18%|█▊ | 19/107 [01:36<07:57, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:15:43,228 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
19%|█▊ | 20/107 [01:42<07:50, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:15:48,588 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
20%|█▉ | 21/107 [01:46<07:27, 5.21s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:15:53,344 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██ | 22/107 [01:52<07:31, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:15:58,907 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██▏ | 23/107 [01:57<07:28, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:16:04,287 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
22%|██▏ | 24/107 [02:03<07:23, 5.34s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:16:09,641 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
23%|██▎ | 25/107 [02:09<07:42, 5.64s/it][WARNING|utils.py:1642] 2026-08-23 00:16:15,966 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
24%|██▍ | 26/107 [02:15<07:37, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:16:21,630 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
25%|██▌ | 27/107 [02:20<07:26, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:16:27,085 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
26%|██▌ | 28/107 [02:26<07:20, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:16:32,641 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
27%|██▋ | 29/107 [02:32<07:23, 5.69s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:16:38,576 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
28%|██▊ | 30/107 [02:37<07:10, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:16:43,935 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
29%|██▉ | 31/107 [02:41<06:39, 5.25s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:16:48,405 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
30%|██▉ | 32/107 [02:47<06:36, 5.28s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:16:53,754 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
31%|███ | 33/107 [02:52<06:34, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:16:59,196 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
32%|███▏ | 34/107 [02:59<06:49, 5.60s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:17:05,440 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
33%|███▎ | 35/107 [03:04<06:40, 5.56s/it][WARNING|utils.py:1642] 2026-08-23 00:17:10,885 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
34%|███▎ | 36/107 [03:09<06:27, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:17:16,130 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
35%|███▍ | 37/107 [03:14<06:05, 5.22s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:17:20,779 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▌ | 38/107 [03:19<06:08, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:17:26,385 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▋ | 39/107 [03:24<05:54, 5.22s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:17:31,327 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
37%|███▋ | 40/107 [03:30<05:50, 5.23s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:17:36,595 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
38%|███▊ | 41/107 [03:35<05:50, 5.32s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:17:42,112 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
39%|███▉ | 42/107 [03:41<06:00, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:17:48,190 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
40%|████ | 43/107 [03:47<06:01, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:17:54,088 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
41%|████ | 44/107 [03:53<05:50, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:17:59,435 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
42%|████▏ | 45/107 [03:58<05:34, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:18:04,455 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
43%|████▎ | 46/107 [04:03<05:27, 5.36s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:18:09,731 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
44%|████▍ | 47/107 [04:09<05:30, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:18:15,566 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
45%|████▍ | 48/107 [04:14<05:27, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:18:21,241 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
46%|████▌ | 49/107 [04:20<05:18, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:18:26,575 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
47%|████▋ | 50/107 [04:25<05:10, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:18:31,923 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
48%|████▊ | 51/107 [04:31<05:08, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:18:37,549 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
49%|████▊ | 52/107 [04:36<04:57, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:18:42,719 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|████▉ | 53/107 [04:42<04:59, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:18:48,627 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|█████ | 54/107 [04:47<04:48, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:18:53,785 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
51%|█████▏ | 55/107 [04:52<04:45, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:18:59,388 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
52%|█████▏ | 56/107 [04:58<04:43, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:19:05,133 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
53%|█████▎ | 57/107 [05:04<04:39, 5.60s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:19:10,810 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
54%|█████▍ | 58/107 [05:09<04:30, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:19:16,163 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
55%|█████▌ | 59/107 [05:15<04:23, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:19:21,540 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
56%|█████▌ | 60/107 [05:20<04:15, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:19:26,900 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
57%|█████▋ | 61/107 [05:25<04:01, 5.24s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:19:31,668 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
58%|█████▊ | 62/107 [05:30<03:56, 5.25s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:19:36,930 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
59%|█████▉ | 63/107 [05:35<03:53, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:19:42,393 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
60%|█████▉ | 64/107 [05:41<03:46, 5.27s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:19:47,581 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
61%|██████ | 65/107 [05:46<03:40, 5.26s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:19:52,792 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
62%|██████▏ | 66/107 [05:51<03:36, 5.28s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:19:58,135 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
63%|██████▎ | 67/107 [05:56<03:31, 5.28s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:20:03,410 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▎ | 68/107 [06:01<03:19, 5.11s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:20:08,123 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▍ | 69/107 [06:06<03:13, 5.08s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:20:13,132 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
65%|██████▌ | 70/107 [06:12<03:16, 5.32s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:20:19,001 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
66%|██████▋ | 71/107 [06:18<03:15, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:20:24,674 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
67%|██████▋ | 72/107 [06:23<03:07, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:20:29,904 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
68%|██████▊ | 73/107 [06:28<03:03, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:20:35,354 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
69%|██████▉ | 74/107 [06:34<02:57, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:20:40,689 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
70%|███████ | 75/107 [06:39<02:50, 5.34s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:20:45,940 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
71%|███████ | 76/107 [06:44<02:46, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:20:51,390 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
72%|███████▏ | 77/107 [06:50<02:41, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:20:56,808 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
73%|███████▎ | 78/107 [06:55<02:37, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:21:02,328 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
74%|███████▍ | 79/107 [07:01<02:31, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:21:07,716 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
75%|███████▍ | 80/107 [07:07<02:28, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:21:13,446 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
76%|███████▌ | 81/107 [07:12<02:21, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:21:18,785 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
77%|███████▋ | 82/107 [07:17<02:14, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:21:24,015 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
78%|███████▊ | 83/107 [07:22<02:07, 5.32s/it][WARNING|utils.py:1642] 2026-08-23 00:21:29,179 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▊ | 84/107 [07:27<02:00, 5.26s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:21:34,293 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▉ | 85/107 [07:33<01:59, 5.44s/it][WARNING|utils.py:1642] 2026-08-23 00:21:40,156 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
80%|████████ | 86/107 [07:39<01:54, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:21:45,652 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
81%|████████▏ | 87/107 [07:43<01:43, 5.17s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:21:50,164 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
82%|████████▏ | 88/107 [07:49<01:39, 5.22s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:21:55,500 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
83%|████████▎ | 89/107 [07:55<01:39, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:22:01,808 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
84%|████████▍ | 90/107 [08:00<01:33, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:22:07,152 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
85%|████████▌ | 91/107 [08:05<01:25, 5.35s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:22:12,198 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
86%|████████▌ | 92/107 [08:10<01:19, 5.27s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:22:17,265 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
87%|████████▋ | 93/107 [08:16<01:13, 5.27s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:22:22,526 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
88%|████████▊ | 94/107 [08:21<01:09, 5.34s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:22:28,026 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
89%|████████▉ | 95/107 [08:26<01:03, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:22:33,351 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
90%|████████▉ | 96/107 [08:32<00:58, 5.28s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:22:38,524 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
91%|█████████ | 97/107 [08:37<00:53, 5.34s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:22:44,002 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
92%|█████████▏| 98/107 [08:43<00:48, 5.42s/it][WARNING|utils.py:1642] 2026-08-23 00:22:49,610 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 99/107 [08:48<00:43, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:22:55,074 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 100/107 [08:53<00:37, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:23:00,277 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
94%|█████████▍| 101/107 [08:59<00:32, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:23:06,028 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
95%|█████████▌| 102/107 [09:05<00:27, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:23:11,550 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
96%|█████████▋| 103/107 [09:10<00:21, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:23:16,770 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
97%|█████████▋| 104/107 [09:15<00:16, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:23:22,235 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
98%|█████████▊| 105/107 [09:21<00:11, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:23:27,932 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
99%|█████████▉| 106/107 [09:27<00:05, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-23 00:23:33,520 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
100%|██████████| 107/107 [09:32<00:00, 5.55s/it]
100%|██████████| 107/107 [09:33<00:00, 5.36s/it]
+***** eval metrics *****
+ epoch = 5.0
+ eval_generator_count = 1712.0
+ eval_generator_exact_match = 96.6121
+ eval_generator_rouge1 = 99.8409
+ eval_generator_rouge2 = 99.7633
+ eval_generator_rougeL = 99.6798
+ eval_loss = 0.0033
+ eval_runtime = 0:09:39.53
+ eval_samples_per_second = 5.891
+ eval_selector_accuracy = 98.7734
+ eval_selector_count = 1712.0
+ eval_steps_per_second = 0.185
+[INFO|modelcard.py:263] 2026-08-23 00:23:40,228 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
diff --git a/qwen3/run_all_logs/20260822_031516/rkl.log b/qwen3/run_all_logs/20260822_031516/rkl.log
new file mode 100644
index 0000000000000000000000000000000000000000..0423fa4a2a5cd3f10abbba9ecc0f95fafb471b8c
--- /dev/null
+++ b/qwen3/run_all_logs/20260822_031516/rkl.log
@@ -0,0 +1,2100 @@
+W0822 12:38:52.904000 1376586 torch/distributed/run.py:803]
+W0822 12:38:52.904000 1376586 torch/distributed/run.py:803] *****************************************
+W0822 12:38:52.904000 1376586 torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+W0822 12:38:52.904000 1376586 torch/distributed/run.py:803] *****************************************
+[INFO|2026-08-22 12:39:00] llamafactory.hparams.parser:651 >> Process rank: 1, world size: 2, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16
+[INFO|2026-08-22 12:39:00] llamafactory.hparams.parser:144 >> Set `ddp_find_unused_parameters` to False in DDP training since LoRA is enabled.
+[INFO|2026-08-22 12:39:00] llamafactory.hparams.parser:651 >> Process rank: 0, world size: 2, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16
+Starting rkl with LlamaFactory and DeepSpeed=True.
+[INFO|configuration_utils.py:780] 2026-08-22 12:39:01,961 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 12:39:01,965 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 12:39:05,080 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 12:39:05,081 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 12:39:05,320 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 12:39:05,322 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 12:39:06] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_train.jsonl...
+[INFO|2026-08-22 12:39:07] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_eval.jsonl...
+training example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 1947, 19029, 10592, 311, 35592, 304, 279, 17328, 1947, 1707, 429, 9482, 1573, 279, 1042, 220, 16, 21, 15, 15, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 9286, 38050, 756, 414, 330, 13193, 788, 341, 286, 330, 408, 14645, 788, 330, 68689, 756, 286, 330, 2468, 14645, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 37525, 756, 414, 330, 13193, 788, 5613, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 18098, 287, 756, 414, 330, 13193, 788, 341, 286, 330, 11141, 3575, 34043, 788, 330, 16468, 756, 286, 330, 37375, 14645, 788, 330, 68689, 698, 414, 456, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 9286, 38050, 756, 414, 330, 1313, 788, 330, 49087, 2354, 698, 262, 1153, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 37525, 756, 414, 330, 1313, 788, 330, 4648, 37525, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of art movements linked to paintings in the genre art style that ended before the year 1600?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "ArtMovement",
+ "properties": {
+ "end_year": "INTEGER",
+ "start_year": "INTEGER"
+ }
+ },
+ {
+ "label": "Genre",
+ "properties": {}
+ },
+ {
+ "label": "Painting",
+ "properties": {
+ "country_of_origin": "STRING",
+ "creation_year": "INTEGER"
+ }
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "ArtMovement",
+ "type": "associatedWith"
+ },
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "Genre",
+ "type": "hasGenre"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+eval example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 9893, 1483, 304, 2987, 279, 220, 17, 15, 17, 16, 55218, 18144, 73379, 476, 279, 23007, 24847, 7755, 9257, 36331, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 13193, 788, 341, 286, 330, 1028, 788, 330, 7097, 756, 286, 330, 31309, 788, 330, 22852, 756, 286, 330, 4082, 3575, 2259, 26286, 788, 330, 68689, 756, 286, 330, 4082, 3575, 1243, 89827, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 28214, 756, 414, 330, 13193, 788, 5613, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 5657, 788, 330, 28214, 756, 414, 330, 1313, 788, 330, 30210, 1047, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of weapons used in either the 2021 Baghdad suicide bombings or the Botroseya Church bombing?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "TerroristAttack",
+ "properties": {
+ "date": "DATE",
+ "locations": "LIST",
+ "number_of_deaths": "INTEGER",
+ "number_of_injuries": "INTEGER"
+ }
+ },
+ {
+ "label": "Weapon",
+ "properties": {}
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "TerroristAttack",
+ "target": "Weapon",
+ "type": "employs"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+[INFO|configuration_utils.py:780] 2026-08-22 12:39:11,360 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 12:39:11,361 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 12:39:11] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training.
+[INFO|modeling_utils.py:769] 2026-08-22 12:39:12,076 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/model.safetensors
+[INFO|modeling_utils.py:839] 2026-08-22 12:39:12,076 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 12:39:12,077 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": false
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 12:39:12,798 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95
+}
+
+[INFO|2026-08-22 12:39:12] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled.
+[INFO|2026-08-22 12:39:12] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-22 12:39:12] llamafactory.model.adapter:144 >> Upcasting trainable params to float32.
+[INFO|2026-08-22 12:39:12] llamafactory.model.adapter:144 >> Fine-tuning method: LoRA
+[INFO|2026-08-22 12:39:12] llamafactory.model.model_utils.misc:144 >> Found linear modules: down_proj,up_proj,q_proj,v_proj,k_proj,o_proj,gate_proj
+[INFO|2026-08-22 12:39:13] llamafactory.model.loader:144 >> trainable params: 20,185,088 || all params: 616,235,008 || trainable%: 3.2756
+[INFO|configuration_utils.py:780] 2026-08-22 12:39:14,162 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 12:39:14,165 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 12:39:17,091 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 12:39:17,330 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 12:39:17,333 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 12:39:19,470 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 12:39:19,471 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 12:39:19] llamafactory.model.model_utils.kv_cache:144 >> KV cache is enabled for faster generation.
+[INFO|modeling_utils.py:769] 2026-08-22 12:39:19,713 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model.safetensors.index.json
+[INFO|modeling_utils.py:839] 2026-08-22 12:39:19,713 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 12:39:19,714 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": true
+}
+
+
Loading weights: 0%| | 0/398 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 12:39:21,779 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.7,
+ "top_k": 20,
+ "top_p": 0.8
+}
+
+[INFO|2026-08-22 12:39:21] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-22 12:39:22] llamafactory.model.adapter:144 >> Merged 1 adapter(s).
+[INFO|2026-08-22 12:39:22] llamafactory.model.adapter:144 >> Loaded adapter(s): results/qwen3/teacher_lora
+[INFO|2026-08-22 12:39:22] llamafactory.model.loader:144 >> all params: 4,022,468,096
+[INFO|2026-08-22 12:39:22] llamafactory.train.trainer_utils:144 >> Created reference model from Qwen/Qwen3-4B-Instruct-2507
+
Loading weights: 0%| | 0/398 [00:00> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}.
+Gradient accumulation steps mismatch: GradientAccumulationPlugin has 1, DeepSpeed config has 8. Using DeepSpeed's value.
+[INFO|trainer.py:1469] 2026-08-22 12:39:27,846 >> ***** Running training *****
+[INFO|trainer.py:1470] 2026-08-22 12:39:27,846 >> Num examples = 10,027
+[INFO|trainer.py:1471] 2026-08-22 12:39:27,846 >> Num Epochs = 5
+[INFO|trainer.py:1472] 2026-08-22 12:39:27,846 >> Num update steps per epoch = 314
+[INFO|trainer.py:1473] 2026-08-22 12:39:27,846 >> Instantaneous batch size per device = 2
+[INFO|trainer.py:1476] 2026-08-22 12:39:27,846 >> Total train batch size (w. parallel, distributed & accumulation) = 32
+[INFO|trainer.py:1477] 2026-08-22 12:39:27,846 >> Gradient Accumulation steps = 8
+[INFO|trainer.py:1478] 2026-08-22 12:39:27,846 >> Total optimization steps = 1,570
+[INFO|trainer.py:1479] 2026-08-22 12:39:27,850 >> Number of trainable parameters = 20,185,088
+
0%| | 0/1570 [00:00>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 12:54:43,954 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 12:54:43,954 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/rkl/checkpoint-314
+[INFO|configuration_utils.py:780] 2026-08-22 13:04:56,666 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 13:04:56,668 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 13:04:56,703 >> chat template saved in results/qwen3/rkl/checkpoint-314/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 13:04:56,704 >> tokenizer config file saved in results/qwen3/rkl/checkpoint-314/tokenizer_config.json
+
20%|██ | 315/1570 [25:32<65:03:08, 186.60s/it]
20%|██ | 316/1570 [25:35<45:47:47, 131.47s/it]
20%|██ | 317/1570 [25:38<32:19:37, 92.88s/it]
20%|██ | 318/1570 [25:41<22:54:42, 65.88s/it]
20%|██ | 319/1570 [25:44<16:19:23, 46.97s/it]
20%|██ | 320/1570 [25:46<11:42:32, 33.72s/it]
{'loss': '0.04533', 'grad_norm': '8.917', 'learning_rate': '9.016e-05', 'epoch': '1.019'}
+
20%|██ | 320/1570 [25:46<11:42:32, 33.72s/it]
20%|██ | 321/1570 [25:49<8:29:12, 24.46s/it]
21%|██ | 322/1570 [25:52<6:13:28, 17.96s/it]
21%|██ | 323/1570 [25:55<4:38:58, 13.42s/it]
21%|██ | 324/1570 [25:58<3:32:56, 10.25s/it]
21%|██ | 325/1570 [26:01<2:46:43, 8.03s/it]
21%|██ | 326/1570 [26:03<2:13:58, 6.46s/it]
21%|██ | 327/1570 [26:06<1:51:23, 5.38s/it]
21%|██ | 328/1570 [26:09<1:35:38, 4.62s/it]
21%|██ | 329/1570 [26:12<1:24:39, 4.09s/it]
21%|██ | 330/1570 [26:15<1:17:12, 3.74s/it]
21%|██ | 331/1570 [26:18<1:11:41, 3.47s/it]
21%|██ | 332/1570 [26:20<1:07:40, 3.28s/it]
21%|██ | 333/1570 [26:23<1:05:06, 3.16s/it]
21%|██▏ | 334/1570 [26:26<1:03:23, 3.08s/it]
21%|██▏ | 335/1570 [26:29<1:01:55, 3.01s/it]
21%|██▏ | 336/1570 [26:32<1:00:58, 2.97s/it]
21%|██▏ | 337/1570 [26:35<1:00:01, 2.92s/it]
22%|██▏ | 338/1570 [26:38<59:24, 2.89s/it]
22%|██▏ | 339/1570 [26:40<59:06, 2.88s/it]
22%|██▏ | 340/1570 [26:43<59:20, 2.89s/it]
{'loss': '0.03384', 'grad_norm': '0.2432', 'learning_rate': '8.894e-05', 'epoch': '1.083'}
+
22%|██▏ | 340/1570 [26:43<59:20, 2.89s/it]
22%|██▏ | 341/1570 [26:46<59:19, 2.90s/it]
22%|██▏ | 342/1570 [26:49<59:23, 2.90s/it]
22%|██▏ | 343/1570 [26:52<58:51, 2.88s/it]
22%|██▏ | 344/1570 [26:55<58:29, 2.86s/it]
22%|██▏ | 345/1570 [26:58<58:14, 2.85s/it]
22%|██▏ | 346/1570 [27:01<58:03, 2.85s/it]
22%|██▏ | 347/1570 [27:03<58:11, 2.85s/it]
22%|██▏ | 348/1570 [27:06<58:07, 2.85s/it]
22%|██▏ | 349/1570 [27:09<57:57, 2.85s/it]
22%|██▏ | 350/1570 [27:12<57:51, 2.85s/it]
22%|██▏ | 351/1570 [27:15<57:41, 2.84s/it]
22%|██▏ | 352/1570 [27:18<57:38, 2.84s/it]
22%|██▏ | 353/1570 [27:20<57:48, 2.85s/it]
23%|██▎ | 354/1570 [27:23<57:57, 2.86s/it]
23%|██▎ | 355/1570 [27:26<57:49, 2.86s/it]
23%|██▎ | 356/1570 [27:29<57:50, 2.86s/it]
23%|██▎ | 357/1570 [27:32<57:46, 2.86s/it]
23%|██▎ | 358/1570 [27:35<57:39, 2.85s/it]
23%|██▎ | 359/1570 [27:38<57:38, 2.86s/it]
23%|██▎ | 360/1570 [27:40<57:15, 2.84s/it]
{'loss': '0.02805', 'grad_norm': '0.1467', 'learning_rate': '8.766e-05', 'epoch': '1.147'}
+
23%|██▎ | 360/1570 [27:40<57:15, 2.84s/it]
23%|██▎ | 361/1570 [27:43<57:26, 2.85s/it]
23%|██▎ | 362/1570 [27:46<57:34, 2.86s/it]
23%|██▎ | 363/1570 [27:49<57:13, 2.84s/it]
23%|██▎ | 364/1570 [27:52<57:22, 2.85s/it]
23%|██▎ | 365/1570 [27:55<57:09, 2.85s/it]
23%|██▎ | 366/1570 [27:58<57:05, 2.85s/it]
23%|██▎ | 367/1570 [28:00<56:54, 2.84s/it]
23%|██▎ | 368/1570 [28:03<56:48, 2.84s/it]
24%|██▎ | 369/1570 [28:06<56:52, 2.84s/it]
24%|██▎ | 370/1570 [28:09<56:53, 2.84s/it]
24%|██▎ | 371/1570 [28:12<56:50, 2.84s/it]
24%|██▎ | 372/1570 [28:15<56:43, 2.84s/it]
24%|██▍ | 373/1570 [28:17<56:43, 2.84s/it]
24%|██▍ | 374/1570 [28:20<56:31, 2.84s/it]
24%|██▍ | 375/1570 [28:23<56:55, 2.86s/it]
24%|██▍ | 376/1570 [28:26<56:58, 2.86s/it]
24%|██▍ | 377/1570 [28:29<56:52, 2.86s/it]
24%|██▍ | 378/1570 [28:32<56:50, 2.86s/it]
24%|██▍ | 379/1570 [28:35<56:54, 2.87s/it]
24%|██▍ | 380/1570 [28:37<56:32, 2.85s/it]
{'loss': '0.01414', 'grad_norm': '0.4752', 'learning_rate': '8.631e-05', 'epoch': '1.211'}
+
24%|██▍ | 380/1570 [28:37<56:32, 2.85s/it]
24%|██▍ | 381/1570 [28:40<56:38, 2.86s/it]
24%|██▍ | 382/1570 [28:43<56:27, 2.85s/it]
24%|██▍ | 383/1570 [28:46<56:30, 2.86s/it]
24%|██▍ | 384/1570 [28:49<56:33, 2.86s/it]
25%|██▍ | 385/1570 [28:52<56:25, 2.86s/it]
25%|██▍ | 386/1570 [28:55<56:11, 2.85s/it]
25%|██▍ | 387/1570 [28:57<56:29, 2.87s/it]
25%|██▍ | 388/1570 [29:00<56:16, 2.86s/it]
25%|██▍ | 389/1570 [29:03<56:03, 2.85s/it]
25%|██▍ | 390/1570 [29:06<56:08, 2.85s/it]
25%|██▍ | 391/1570 [29:09<56:10, 2.86s/it]
25%|██▍ | 392/1570 [29:12<56:08, 2.86s/it]
25%|██▌ | 393/1570 [29:15<55:53, 2.85s/it]
25%|██▌ | 394/1570 [29:17<56:17, 2.87s/it]
25%|██▌ | 395/1570 [29:20<56:16, 2.87s/it]
25%|██▌ | 396/1570 [29:23<56:12, 2.87s/it]
25%|██▌ | 397/1570 [29:26<56:14, 2.88s/it]
25%|██▌ | 398/1570 [29:29<56:12, 2.88s/it]
25%|██▌ | 399/1570 [29:32<55:59, 2.87s/it]
25%|██▌ | 400/1570 [29:35<56:02, 2.87s/it]
{'loss': '0.01117', 'grad_norm': '0.4543', 'learning_rate': '8.491e-05', 'epoch': '1.274'}
+
25%|██▌ | 400/1570 [29:35<56:02, 2.87s/it]
26%|██▌ | 401/1570 [29:38<56:13, 2.89s/it]
26%|██▌ | 402/1570 [29:40<55:52, 2.87s/it]
26%|██▌ | 403/1570 [29:43<55:47, 2.87s/it]
26%|██▌ | 404/1570 [29:46<55:37, 2.86s/it]
26%|██▌ | 405/1570 [29:49<55:34, 2.86s/it]
26%|██▌ | 406/1570 [29:52<55:51, 2.88s/it]
26%|██▌ | 407/1570 [29:55<55:50, 2.88s/it]
26%|██▌ | 408/1570 [29:58<55:25, 2.86s/it]
26%|██▌ | 409/1570 [30:01<55:32, 2.87s/it]
26%|██▌ | 410/1570 [30:03<55:27, 2.87s/it]
26%|██▌ | 411/1570 [30:06<55:12, 2.86s/it]
26%|██▌ | 412/1570 [30:09<55:04, 2.85s/it]
26%|██▋ | 413/1570 [30:12<55:05, 2.86s/it]
26%|██▋ | 414/1570 [30:15<55:17, 2.87s/it]
26%|██▋ | 415/1570 [30:18<54:55, 2.85s/it]
26%|██▋ | 416/1570 [30:21<55:07, 2.87s/it]
27%|██▋ | 417/1570 [30:23<54:51, 2.85s/it]
27%|██▋ | 418/1570 [30:26<54:49, 2.86s/it]
27%|██▋ | 419/1570 [30:29<54:30, 2.84s/it]
27%|██▋ | 420/1570 [30:32<54:32, 2.85s/it]
{'loss': '0.02132', 'grad_norm': '0.3609', 'learning_rate': '8.345e-05', 'epoch': '1.338'}
+
27%|██▋ | 420/1570 [30:32<54:32, 2.85s/it]
27%|██▋ | 421/1570 [30:35<54:48, 2.86s/it]
27%|██▋ | 422/1570 [30:38<54:24, 2.84s/it]
27%|██▋ | 423/1570 [30:40<54:32, 2.85s/it]
27%|██▋ | 424/1570 [30:43<54:23, 2.85s/it]
27%|██▋ | 425/1570 [30:46<54:22, 2.85s/it]
27%|██▋ | 426/1570 [30:49<54:18, 2.85s/it]
27%|██▋ | 427/1570 [30:52<54:13, 2.85s/it]
27%|██▋ | 428/1570 [30:55<54:07, 2.84s/it]
27%|██▋ | 429/1570 [30:58<54:13, 2.85s/it]
27%|██▋ | 430/1570 [31:00<54:19, 2.86s/it]
27%|██▋ | 431/1570 [31:03<54:22, 2.86s/it]
28%|██▊ | 432/1570 [31:06<54:17, 2.86s/it]
28%|██▊ | 433/1570 [31:09<54:06, 2.86s/it]
28%|██▊ | 434/1570 [31:12<53:53, 2.85s/it]
28%|██▊ | 435/1570 [31:15<53:56, 2.85s/it]
28%|██▊ | 436/1570 [31:18<54:05, 2.86s/it]
28%|██▊ | 437/1570 [31:20<54:09, 2.87s/it]
28%|██▊ | 438/1570 [31:23<54:04, 2.87s/it]
28%|██▊ | 439/1570 [31:26<53:52, 2.86s/it]
28%|██▊ | 440/1570 [31:29<54:03, 2.87s/it]
{'loss': '0.01834', 'grad_norm': '0.2029', 'learning_rate': '8.194e-05', 'epoch': '1.402'}
+
28%|██▊ | 440/1570 [31:29<54:03, 2.87s/it]
28%|██▊ | 441/1570 [31:32<54:04, 2.87s/it]
28%|██▊ | 442/1570 [31:35<54:03, 2.88s/it]
28%|██▊ | 443/1570 [31:38<53:59, 2.87s/it]
28%|██▊ | 444/1570 [31:41<53:59, 2.88s/it]
28%|██▊ | 445/1570 [31:43<53:50, 2.87s/it]
28%|██▊ | 446/1570 [31:46<54:03, 2.89s/it]
28%|██▊ | 447/1570 [31:49<54:04, 2.89s/it]
29%|██▊ | 448/1570 [31:52<53:52, 2.88s/it]
29%|██▊ | 449/1570 [31:55<54:14, 2.90s/it]
29%|██▊ | 450/1570 [31:58<54:17, 2.91s/it]
29%|██▊ | 451/1570 [32:01<54:03, 2.90s/it]
29%|██▉ | 452/1570 [32:04<54:03, 2.90s/it]
29%|██▉ | 453/1570 [32:07<53:51, 2.89s/it]
29%|██▉ | 454/1570 [32:10<53:40, 2.89s/it]
29%|██▉ | 455/1570 [32:12<53:38, 2.89s/it]
29%|██▉ | 456/1570 [32:15<53:43, 2.89s/it]
29%|██▉ | 457/1570 [32:18<53:28, 2.88s/it]
29%|██▉ | 458/1570 [32:21<53:27, 2.88s/it]
29%|██▉ | 459/1570 [32:24<53:15, 2.88s/it]
29%|██▉ | 460/1570 [32:27<52:52, 2.86s/it]
{'loss': '0.01772', 'grad_norm': '0.3136', 'learning_rate': '8.037e-05', 'epoch': '1.466'}
+
29%|██▉ | 460/1570 [32:27<52:52, 2.86s/it]
29%|██▉ | 461/1570 [32:30<52:42, 2.85s/it]
29%|██▉ | 462/1570 [32:32<52:41, 2.85s/it]
29%|██▉ | 463/1570 [32:35<52:48, 2.86s/it]
30%|██▉ | 464/1570 [32:38<52:45, 2.86s/it]
30%|██▉ | 465/1570 [32:41<52:32, 2.85s/it]
30%|██▉ | 466/1570 [32:44<52:13, 2.84s/it]
30%|██▉ | 467/1570 [32:47<52:23, 2.85s/it]
30%|██▉ | 468/1570 [32:50<52:23, 2.85s/it]
30%|██▉ | 469/1570 [32:52<52:20, 2.85s/it]
30%|██▉ | 470/1570 [32:55<52:07, 2.84s/it]
30%|███ | 471/1570 [32:58<52:06, 2.84s/it]
30%|███ | 472/1570 [33:01<52:07, 2.85s/it]
30%|███ | 473/1570 [33:04<52:09, 2.85s/it]
30%|███ | 474/1570 [33:07<51:49, 2.84s/it]
30%|███ | 475/1570 [33:09<51:54, 2.84s/it]
30%|███ | 476/1570 [33:12<51:58, 2.85s/it]
30%|███ | 477/1570 [33:15<51:59, 2.85s/it]
30%|███ | 478/1570 [33:18<51:55, 2.85s/it]
31%|███ | 479/1570 [33:21<51:58, 2.86s/it]
31%|███ | 480/1570 [33:24<52:00, 2.86s/it]
{'loss': '0.007667', 'grad_norm': '0.8575', 'learning_rate': '7.876e-05', 'epoch': '1.53'}
+
31%|███ | 480/1570 [33:24<52:00, 2.86s/it]
31%|███ | 481/1570 [33:27<52:03, 2.87s/it]
31%|███ | 482/1570 [33:30<51:55, 2.86s/it]
31%|███ | 483/1570 [33:32<51:44, 2.86s/it]
31%|███ | 484/1570 [33:35<51:33, 2.85s/it]
31%|███ | 485/1570 [33:38<51:37, 2.85s/it]
31%|███ | 486/1570 [33:41<51:37, 2.86s/it]
31%|███ | 487/1570 [33:44<51:39, 2.86s/it]
31%|███ | 488/1570 [33:47<51:31, 2.86s/it]
31%|███ | 489/1570 [33:49<51:18, 2.85s/it]
31%|███ | 490/1570 [33:52<51:22, 2.85s/it]
31%|███▏ | 491/1570 [33:55<51:21, 2.86s/it]
31%|███▏ | 492/1570 [33:58<51:28, 2.87s/it]
31%|███▏ | 493/1570 [34:01<51:27, 2.87s/it]
31%|███▏ | 494/1570 [34:04<51:12, 2.86s/it]
32%|███▏ | 495/1570 [34:07<51:10, 2.86s/it]
32%|███▏ | 496/1570 [34:09<50:49, 2.84s/it]
32%|███▏ | 497/1570 [34:12<51:00, 2.85s/it]
32%|███▏ | 498/1570 [34:15<51:12, 2.87s/it]
32%|███▏ | 499/1570 [34:18<51:16, 2.87s/it]
32%|███▏ | 500/1570 [34:21<50:57, 2.86s/it]
{'loss': '0.01248', 'grad_norm': '0.7694', 'learning_rate': '7.71e-05', 'epoch': '1.594'}
+
32%|███▏ | 500/1570 [34:21<50:57, 2.86s/it]
32%|███▏ | 501/1570 [34:24<50:48, 2.85s/it]
32%|███▏ | 502/1570 [34:27<50:54, 2.86s/it]
32%|███▏ | 503/1570 [34:30<51:02, 2.87s/it]
32%|███▏ | 504/1570 [34:32<50:59, 2.87s/it]
32%|███▏ | 505/1570 [34:35<51:04, 2.88s/it]
32%|███▏ | 506/1570 [34:38<50:49, 2.87s/it]
32%|███▏ | 507/1570 [34:41<50:51, 2.87s/it]
32%|███▏ | 508/1570 [34:44<50:48, 2.87s/it]
32%|███▏ | 509/1570 [34:47<50:33, 2.86s/it]
32%|███▏ | 510/1570 [34:50<50:35, 2.86s/it]
33%|███▎ | 511/1570 [34:52<50:36, 2.87s/it]
33%|███▎ | 512/1570 [34:55<50:38, 2.87s/it]
33%|███▎ | 513/1570 [34:58<50:46, 2.88s/it]
33%|███▎ | 514/1570 [35:01<50:58, 2.90s/it]
33%|███▎ | 515/1570 [35:04<50:47, 2.89s/it]
33%|███▎ | 516/1570 [35:07<50:51, 2.89s/it]
33%|███▎ | 517/1570 [35:10<50:17, 2.87s/it]
33%|███▎ | 518/1570 [35:13<50:16, 2.87s/it]
33%|███▎ | 519/1570 [35:16<50:21, 2.87s/it]
33%|███▎ | 520/1570 [35:18<50:15, 2.87s/it]
{'loss': '0.01497', 'grad_norm': '0.1912', 'learning_rate': '7.54e-05', 'epoch': '1.657'}
+
33%|███▎ | 520/1570 [35:18<50:15, 2.87s/it]
33%|███▎ | 521/1570 [35:21<50:19, 2.88s/it]
33%|███▎ | 522/1570 [35:24<50:21, 2.88s/it]
33%|███▎ | 523/1570 [35:27<50:11, 2.88s/it]
33%|███▎ | 524/1570 [35:30<50:19, 2.89s/it]
33%|███▎ | 525/1570 [35:33<50:12, 2.88s/it]
34%|███▎ | 526/1570 [35:36<50:01, 2.88s/it]
34%|███▎ | 527/1570 [35:39<49:50, 2.87s/it]
34%|███▎ | 528/1570 [35:41<49:37, 2.86s/it]
34%|███▎ | 529/1570 [35:44<49:24, 2.85s/it]
34%|███▍ | 530/1570 [35:47<49:16, 2.84s/it]
34%|███▍ | 531/1570 [35:50<49:22, 2.85s/it]
34%|███▍ | 532/1570 [35:53<49:17, 2.85s/it]
34%|███▍ | 533/1570 [35:56<49:17, 2.85s/it]
34%|███▍ | 534/1570 [35:59<49:28, 2.87s/it]
34%|███▍ | 535/1570 [36:01<49:17, 2.86s/it]
34%|███▍ | 536/1570 [36:04<49:14, 2.86s/it]
34%|███▍ | 537/1570 [36:07<49:00, 2.85s/it]
34%|███▍ | 538/1570 [36:10<49:08, 2.86s/it]
34%|███▍ | 539/1570 [36:13<48:56, 2.85s/it]
34%|███▍ | 540/1570 [36:16<48:54, 2.85s/it]
{'loss': '0.01157', 'grad_norm': '0.524', 'learning_rate': '7.366e-05', 'epoch': '1.721'}
+
34%|███▍ | 540/1570 [36:16<48:54, 2.85s/it]
34%|███▍ | 541/1570 [36:18<48:59, 2.86s/it]
35%|███▍ | 542/1570 [36:21<48:55, 2.86s/it]
35%|███▍ | 543/1570 [36:24<48:50, 2.85s/it]
35%|███▍ | 544/1570 [36:27<48:47, 2.85s/it]
35%|███▍ | 545/1570 [36:30<48:47, 2.86s/it]
35%|███▍ | 546/1570 [36:33<48:40, 2.85s/it]
35%|███▍ | 547/1570 [36:36<48:39, 2.85s/it]
35%|███▍ | 548/1570 [36:38<48:32, 2.85s/it]
35%|███▍ | 549/1570 [36:41<48:11, 2.83s/it]
35%|███▌ | 550/1570 [36:44<48:20, 2.84s/it]
35%|███▌ | 551/1570 [36:47<48:36, 2.86s/it]
35%|███▌ | 552/1570 [36:50<48:31, 2.86s/it]
35%|███▌ | 553/1570 [36:53<48:16, 2.85s/it]
35%|███▌ | 554/1570 [36:56<48:21, 2.86s/it]
35%|███▌ | 555/1570 [36:58<48:21, 2.86s/it]
35%|███▌ | 556/1570 [37:01<48:23, 2.86s/it]
35%|███▌ | 557/1570 [37:04<48:19, 2.86s/it]
36%|███▌ | 558/1570 [37:07<48:12, 2.86s/it]
36%|███▌ | 559/1570 [37:10<48:13, 2.86s/it]
36%|███▌ | 560/1570 [37:13<48:05, 2.86s/it]
{'loss': '0.007554', 'grad_norm': '0.2922', 'learning_rate': '7.188e-05', 'epoch': '1.785'}
+
36%|███▌ | 560/1570 [37:13<48:05, 2.86s/it]
36%|███▌ | 561/1570 [37:16<48:14, 2.87s/it]
36%|███▌ | 562/1570 [37:18<47:56, 2.85s/it]
36%|███▌ | 563/1570 [37:21<47:52, 2.85s/it]
36%|███▌ | 564/1570 [37:24<47:42, 2.85s/it]
36%|███▌ | 565/1570 [37:27<47:58, 2.86s/it]
36%|███▌ | 566/1570 [37:30<47:44, 2.85s/it]
36%|███▌ | 567/1570 [37:33<47:43, 2.86s/it]
36%|███▌ | 568/1570 [37:36<47:45, 2.86s/it]
36%|███▌ | 569/1570 [37:38<47:52, 2.87s/it]
36%|███▋ | 570/1570 [37:41<47:41, 2.86s/it]
36%|███▋ | 571/1570 [37:44<47:41, 2.86s/it]
36%|███▋ | 572/1570 [37:47<47:41, 2.87s/it]
36%|███▋ | 573/1570 [37:50<47:58, 2.89s/it]
37%|███▋ | 574/1570 [37:53<47:43, 2.88s/it]
37%|███▋ | 575/1570 [37:56<47:31, 2.87s/it]
37%|███▋ | 576/1570 [37:58<47:13, 2.85s/it]
37%|███▋ | 577/1570 [38:01<47:28, 2.87s/it]
37%|███▋ | 578/1570 [38:04<47:09, 2.85s/it]
37%|███▋ | 579/1570 [38:07<47:05, 2.85s/it]
37%|███▋ | 580/1570 [38:10<47:16, 2.86s/it]
{'loss': '0.02118', 'grad_norm': '0.6604', 'learning_rate': '7.006e-05', 'epoch': '1.849'}
+
37%|███▋ | 580/1570 [38:10<47:16, 2.86s/it]
37%|███▋ | 581/1570 [38:13<47:03, 2.85s/it]
37%|███▋ | 582/1570 [38:16<47:05, 2.86s/it]
37%|███▋ | 583/1570 [38:19<47:03, 2.86s/it]
37%|███▋ | 584/1570 [38:21<47:02, 2.86s/it]
37%|███▋ | 585/1570 [38:24<46:56, 2.86s/it]
37%|███▋ | 586/1570 [38:27<46:51, 2.86s/it]
37%|███▋ | 587/1570 [38:30<46:44, 2.85s/it]
37%|███▋ | 588/1570 [38:33<46:24, 2.84s/it]
38%|███▊ | 589/1570 [38:36<46:48, 2.86s/it]
38%|███▊ | 590/1570 [38:38<46:35, 2.85s/it]
38%|███▊ | 591/1570 [38:41<46:31, 2.85s/it]
38%|███▊ | 592/1570 [38:44<46:45, 2.87s/it]
38%|███▊ | 593/1570 [38:47<46:36, 2.86s/it]
38%|███▊ | 594/1570 [38:50<46:22, 2.85s/it]
38%|███▊ | 595/1570 [38:53<46:16, 2.85s/it]
38%|███▊ | 596/1570 [38:56<46:24, 2.86s/it]
38%|███▊ | 597/1570 [38:58<46:21, 2.86s/it]
38%|███▊ | 598/1570 [39:01<46:12, 2.85s/it]
38%|███▊ | 599/1570 [39:04<46:11, 2.85s/it]
38%|███▊ | 600/1570 [39:07<46:01, 2.85s/it]
{'loss': '0.005898', 'grad_norm': '0.2436', 'learning_rate': '6.821e-05', 'epoch': '1.913'}
+
38%|███▊ | 600/1570 [39:07<46:01, 2.85s/it]
38%|███▊ | 601/1570 [39:10<46:05, 2.85s/it]
38%|███▊ | 602/1570 [39:13<46:06, 2.86s/it]
38%|███▊ | 603/1570 [39:16<46:16, 2.87s/it]
38%|███▊ | 604/1570 [39:19<46:19, 2.88s/it]
39%|███▊ | 605/1570 [39:21<46:15, 2.88s/it]
39%|███▊ | 606/1570 [39:24<46:11, 2.87s/it]
39%|███▊ | 607/1570 [39:27<45:59, 2.87s/it]
39%|███▊ | 608/1570 [39:30<45:50, 2.86s/it]
39%|███▉ | 609/1570 [39:33<45:40, 2.85s/it]
39%|███▉ | 610/1570 [39:36<45:41, 2.86s/it]
39%|███▉ | 611/1570 [39:39<45:35, 2.85s/it]
39%|███▉ | 612/1570 [39:41<45:33, 2.85s/it]
39%|███▉ | 613/1570 [39:44<45:26, 2.85s/it]
39%|███▉ | 614/1570 [39:47<45:25, 2.85s/it]
39%|███▉ | 615/1570 [39:50<45:29, 2.86s/it]
39%|███▉ | 616/1570 [39:53<45:35, 2.87s/it]
39%|███▉ | 617/1570 [39:56<45:37, 2.87s/it]
39%|███▉ | 618/1570 [39:59<45:24, 2.86s/it]
39%|███▉ | 619/1570 [40:01<45:34, 2.88s/it]
39%|███▉ | 620/1570 [40:04<45:27, 2.87s/it]
{'loss': '0.006591', 'grad_norm': '0.621', 'learning_rate': '6.634e-05', 'epoch': '1.976'}
+
39%|███▉ | 620/1570 [40:04<45:27, 2.87s/it]
40%|███▉ | 621/1570 [40:07<45:28, 2.88s/it]
40%|███▉ | 622/1570 [40:10<45:10, 2.86s/it]
40%|███▉ | 623/1570 [40:13<45:16, 2.87s/it]
40%|███▉ | 624/1570 [40:16<45:30, 2.89s/it]
40%|███▉ | 625/1570 [40:19<45:24, 2.88s/it]
40%|███▉ | 626/1570 [40:22<45:16, 2.88s/it]
40%|███▉ | 627/1570 [40:24<45:10, 2.87s/it]
40%|████ | 628/1570 [40:26<36:58, 2.35s/it][INFO|trainer.py:2668] 2026-08-22 13:19:53,967 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 13:19:53,967 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 13:19:53,967 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/rkl/checkpoint-628
+[INFO|configuration_utils.py:780] 2026-08-22 13:30:10,743 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 13:30:10,747 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 13:30:10,783 >> chat template saved in results/qwen3/rkl/checkpoint-628/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 13:30:10,783 >> tokenizer config file saved in results/qwen3/rkl/checkpoint-628/tokenizer_config.json
+
40%|████ | 629/1570 [50:46<49:06:00, 187.84s/it]
40%|████ | 630/1570 [50:49<34:33:25, 132.35s/it]
40%|████ | 631/1570 [50:52<24:23:08, 93.49s/it]
40%|████ | 632/1570 [50:55<17:16:23, 66.29s/it]
40%|████ | 633/1570 [50:58<12:18:07, 47.27s/it]
40%|████ | 634/1570 [51:01<8:49:38, 33.95s/it]
40%|████ | 635/1570 [51:03<6:23:56, 24.64s/it]
41%|████ | 636/1570 [51:06<4:41:49, 18.10s/it]
41%|████ | 637/1570 [51:09<3:30:16, 13.52s/it]
41%|████ | 638/1570 [51:12<2:40:27, 10.33s/it]
41%|████ | 639/1570 [51:15<2:05:21, 8.08s/it]
41%|████ | 640/1570 [51:18<1:41:12, 6.53s/it]
{'loss': '0.00606', 'grad_norm': '0.1027', 'learning_rate': '6.444e-05', 'epoch': '2.038'}
+
41%|████ | 640/1570 [51:18<1:41:12, 6.53s/it]
41%|████ | 641/1570 [51:21<1:24:08, 5.43s/it]
41%|████ | 642/1570 [51:23<1:12:09, 4.67s/it]
41%|████ | 643/1570 [51:26<1:03:39, 4.12s/it]
41%|████ | 644/1570 [51:29<57:45, 3.74s/it]
41%|████ | 645/1570 [51:32<53:43, 3.48s/it]
41%|████ | 646/1570 [51:35<50:48, 3.30s/it]
41%|████ | 647/1570 [51:38<48:49, 3.17s/it]
41%|████▏ | 648/1570 [51:41<47:18, 3.08s/it]
41%|████▏ | 649/1570 [51:44<46:23, 3.02s/it]
41%|████▏ | 650/1570 [51:46<45:35, 2.97s/it]
41%|████▏ | 651/1570 [51:49<45:01, 2.94s/it]
42%|████▏ | 652/1570 [51:52<44:36, 2.92s/it]
42%|████▏ | 653/1570 [51:55<44:29, 2.91s/it]
42%|████▏ | 654/1570 [51:58<43:57, 2.88s/it]
42%|████▏ | 655/1570 [52:01<43:43, 2.87s/it]
42%|████▏ | 656/1570 [52:04<43:40, 2.87s/it]
42%|████▏ | 657/1570 [52:06<43:28, 2.86s/it]
42%|████▏ | 658/1570 [52:09<43:26, 2.86s/it]
42%|████▏ | 659/1570 [52:12<43:24, 2.86s/it]
42%|████▏ | 660/1570 [52:15<43:20, 2.86s/it]
{'loss': '0.01207', 'grad_norm': '0.5004', 'learning_rate': '6.251e-05', 'epoch': '2.102'}
+
42%|████▏ | 660/1570 [52:15<43:20, 2.86s/it]
42%|████▏ | 661/1570 [52:18<43:24, 2.87s/it]
42%|████▏ | 662/1570 [52:21<43:10, 2.85s/it]
42%|████▏ | 663/1570 [52:24<43:05, 2.85s/it]
42%|████▏ | 664/1570 [52:26<43:09, 2.86s/it]
42%|████▏ | 665/1570 [52:29<43:14, 2.87s/it]
42%|████▏ | 666/1570 [52:32<43:28, 2.89s/it]
42%|████▏ | 667/1570 [52:35<43:15, 2.87s/it]
43%|████▎ | 668/1570 [52:38<43:03, 2.86s/it]
43%|████▎ | 669/1570 [52:41<42:59, 2.86s/it]
43%|████▎ | 670/1570 [52:44<43:10, 2.88s/it]
43%|████▎ | 671/1570 [52:47<43:15, 2.89s/it]
43%|████▎ | 672/1570 [52:50<43:30, 2.91s/it]
43%|████▎ | 673/1570 [52:52<43:31, 2.91s/it]
43%|████▎ | 674/1570 [52:56<44:28, 2.98s/it]
43%|████▎ | 675/1570 [52:59<44:24, 2.98s/it]
43%|████▎ | 676/1570 [53:02<44:43, 3.00s/it]
43%|████▎ | 677/1570 [53:05<44:15, 2.97s/it]
43%|████▎ | 678/1570 [53:08<44:59, 3.03s/it]
43%|████▎ | 679/1570 [53:11<44:35, 3.00s/it]
43%|████▎ | 680/1570 [53:14<44:04, 2.97s/it]
{'loss': '0.003572', 'grad_norm': '0.07481', 'learning_rate': '6.057e-05', 'epoch': '2.166'}
+
43%|████▎ | 680/1570 [53:14<44:04, 2.97s/it]
43%|████▎ | 681/1570 [53:16<43:44, 2.95s/it]
43%|████▎ | 682/1570 [53:19<43:33, 2.94s/it]
44%|████▎ | 683/1570 [53:22<44:02, 2.98s/it]
44%|████▎ | 684/1570 [53:26<45:33, 3.09s/it]
44%|████▎ | 685/1570 [53:29<45:32, 3.09s/it]
44%|████▎ | 686/1570 [53:32<44:59, 3.05s/it]
44%|████▍ | 687/1570 [53:35<44:13, 3.00s/it]
44%|████▍ | 688/1570 [53:38<44:41, 3.04s/it]
44%|████▍ | 689/1570 [53:41<44:04, 3.00s/it]
44%|████▍ | 690/1570 [53:44<44:04, 3.01s/it]
44%|████▍ | 691/1570 [53:47<43:28, 2.97s/it]
44%|████▍ | 692/1570 [53:50<43:55, 3.00s/it]
44%|████▍ | 693/1570 [53:53<43:30, 2.98s/it]
44%|████▍ | 694/1570 [53:56<44:12, 3.03s/it]
44%|████▍ | 695/1570 [53:59<43:43, 3.00s/it]
44%|████▍ | 696/1570 [54:02<43:56, 3.02s/it]
44%|████▍ | 697/1570 [54:05<43:18, 2.98s/it]
44%|████▍ | 698/1570 [54:08<43:36, 3.00s/it]
45%|████▍ | 699/1570 [54:11<43:09, 2.97s/it]
45%|████▍ | 700/1570 [54:14<43:40, 3.01s/it]
{'loss': '0.004928', 'grad_norm': '0.01576', 'learning_rate': '5.86e-05', 'epoch': '2.23'}
+
45%|████▍ | 700/1570 [54:14<43:40, 3.01s/it]
45%|████▍ | 701/1570 [54:17<43:09, 2.98s/it]
45%|████▍ | 702/1570 [54:20<43:50, 3.03s/it]
45%|████▍ | 703/1570 [54:23<44:26, 3.08s/it]
45%|████▍ | 704/1570 [54:26<45:48, 3.17s/it]
45%|████▍ | 705/1570 [54:30<46:48, 3.25s/it]
45%|████▍ | 706/1570 [54:33<46:17, 3.21s/it]
45%|████▌ | 707/1570 [54:36<44:45, 3.11s/it]
45%|████▌ | 708/1570 [54:39<43:57, 3.06s/it]
45%|████▌ | 709/1570 [54:42<43:11, 3.01s/it]
45%|████▌ | 710/1570 [54:45<42:32, 2.97s/it]
45%|████▌ | 711/1570 [54:47<42:13, 2.95s/it]
45%|████▌ | 712/1570 [54:50<41:59, 2.94s/it]
45%|████▌ | 713/1570 [54:53<41:50, 2.93s/it]
45%|████▌ | 714/1570 [54:56<41:44, 2.93s/it]
46%|████▌ | 715/1570 [54:59<41:41, 2.93s/it]
46%|████▌ | 716/1570 [55:02<41:36, 2.92s/it]
46%|████▌ | 717/1570 [55:05<41:25, 2.91s/it]
46%|████▌ | 718/1570 [55:08<41:28, 2.92s/it]
46%|████▌ | 719/1570 [55:11<41:33, 2.93s/it]
46%|████▌ | 720/1570 [55:14<41:29, 2.93s/it]
{'loss': '0.004035', 'grad_norm': '0.04448', 'learning_rate': '5.663e-05', 'epoch': '2.294'}
+
46%|████▌ | 720/1570 [55:14<41:29, 2.93s/it]
46%|████▌ | 721/1570 [55:17<41:16, 2.92s/it]
46%|████▌ | 722/1570 [55:19<41:04, 2.91s/it]
46%|████▌ | 723/1570 [55:22<41:09, 2.92s/it]
46%|████▌ | 724/1570 [55:25<41:12, 2.92s/it]
46%|████▌ | 725/1570 [55:28<41:21, 2.94s/it]
46%|████▌ | 726/1570 [55:31<41:15, 2.93s/it]
46%|████▋ | 727/1570 [55:34<41:14, 2.94s/it]
46%|████▋ | 728/1570 [55:37<41:19, 2.95s/it]
46%|████▋ | 729/1570 [55:40<41:10, 2.94s/it]
46%|████▋ | 730/1570 [55:43<40:47, 2.91s/it]
47%|████▋ | 731/1570 [55:46<40:47, 2.92s/it]
47%|████▋ | 732/1570 [55:49<40:47, 2.92s/it]
47%|████▋ | 733/1570 [55:52<40:37, 2.91s/it]
47%|████▋ | 734/1570 [55:55<40:34, 2.91s/it]
47%|████▋ | 735/1570 [55:57<40:16, 2.89s/it]
47%|████▋ | 736/1570 [56:00<40:12, 2.89s/it]
47%|████▋ | 737/1570 [56:03<39:57, 2.88s/it]
47%|████▋ | 738/1570 [56:06<40:01, 2.89s/it]
47%|████▋ | 739/1570 [56:09<39:49, 2.88s/it]
47%|████▋ | 740/1570 [56:12<39:46, 2.87s/it]
{'loss': '0.004731', 'grad_norm': '0.09333', 'learning_rate': '5.464e-05', 'epoch': '2.357'}
+
47%|████▋ | 740/1570 [56:12<39:46, 2.87s/it]
47%|████▋ | 741/1570 [56:15<39:47, 2.88s/it]
47%|████▋ | 742/1570 [56:18<39:52, 2.89s/it]
47%|████▋ | 743/1570 [56:21<40:09, 2.91s/it]
47%|████▋ | 744/1570 [56:24<40:14, 2.92s/it]
47%|████▋ | 745/1570 [56:26<40:01, 2.91s/it]
48%|████▊ | 746/1570 [56:29<39:48, 2.90s/it]
48%|████▊ | 747/1570 [56:32<39:50, 2.90s/it]
48%|████▊ | 748/1570 [56:35<39:42, 2.90s/it]
48%|████▊ | 749/1570 [56:38<39:52, 2.91s/it]
48%|████▊ | 750/1570 [56:41<39:33, 2.89s/it]
48%|████▊ | 751/1570 [56:44<39:49, 2.92s/it]
48%|████▊ | 752/1570 [56:47<39:38, 2.91s/it]
48%|████▊ | 753/1570 [56:50<39:41, 2.91s/it]
48%|████▊ | 754/1570 [56:53<39:34, 2.91s/it]
48%|████▊ | 755/1570 [56:55<39:35, 2.91s/it]
48%|████▊ | 756/1570 [56:58<39:30, 2.91s/it]
48%|████▊ | 757/1570 [57:01<39:24, 2.91s/it]
48%|████▊ | 758/1570 [57:04<39:50, 2.94s/it]
48%|████▊ | 759/1570 [57:07<39:43, 2.94s/it]
48%|████▊ | 760/1570 [57:10<39:34, 2.93s/it]
{'loss': '0.002293', 'grad_norm': '0.003711', 'learning_rate': '5.265e-05', 'epoch': '2.421'}
+
48%|████▊ | 760/1570 [57:10<39:34, 2.93s/it]
48%|████▊ | 761/1570 [57:13<39:32, 2.93s/it]
49%|████▊ | 762/1570 [57:16<39:34, 2.94s/it]
49%|████▊ | 763/1570 [57:19<39:21, 2.93s/it]
49%|████▊ | 764/1570 [57:22<39:50, 2.97s/it]
49%|████▊ | 765/1570 [57:25<39:28, 2.94s/it]
49%|████▉ | 766/1570 [57:28<39:10, 2.92s/it]
49%|████▉ | 767/1570 [57:31<38:57, 2.91s/it]
49%|████▉ | 768/1570 [57:34<39:36, 2.96s/it]
49%|████▉ | 769/1570 [57:37<39:14, 2.94s/it]
49%|████▉ | 770/1570 [57:40<39:55, 2.99s/it]
49%|████▉ | 771/1570 [57:43<39:28, 2.96s/it]
49%|████▉ | 772/1570 [57:46<39:54, 3.00s/it]
49%|████▉ | 773/1570 [57:49<39:23, 2.97s/it]
49%|████▉ | 774/1570 [57:52<39:53, 3.01s/it]
49%|████▉ | 775/1570 [57:55<39:33, 2.99s/it]
49%|████▉ | 776/1570 [57:58<39:59, 3.02s/it]
49%|████▉ | 777/1570 [58:01<39:31, 2.99s/it]
50%|████▉ | 778/1570 [58:04<39:55, 3.02s/it]
50%|████▉ | 779/1570 [58:07<39:23, 2.99s/it]
50%|████▉ | 780/1570 [58:10<39:48, 3.02s/it]
{'loss': '0.005583', 'grad_norm': '0.01113', 'learning_rate': '5.065e-05', 'epoch': '2.485'}
+
50%|████▉ | 780/1570 [58:10<39:48, 3.02s/it]
50%|████▉ | 781/1570 [58:13<39:26, 3.00s/it]
50%|████▉ | 782/1570 [58:16<39:59, 3.04s/it]
50%|████▉ | 783/1570 [58:19<40:51, 3.12s/it]
50%|████▉ | 784/1570 [58:23<41:53, 3.20s/it]
50%|█████ | 785/1570 [58:26<42:28, 3.25s/it]
50%|█████ | 786/1570 [58:29<41:58, 3.21s/it]
50%|█████ | 787/1570 [58:32<42:25, 3.25s/it]
50%|█████ | 788/1570 [58:35<41:33, 3.19s/it]
50%|█████ | 789/1570 [58:38<40:24, 3.10s/it]
50%|█████ | 790/1570 [58:41<39:37, 3.05s/it]
50%|█████ | 791/1570 [58:44<39:05, 3.01s/it]
50%|█████ | 792/1570 [58:47<38:30, 2.97s/it]
51%|█████ | 793/1570 [58:50<38:08, 2.94s/it]
51%|█████ | 794/1570 [58:53<37:59, 2.94s/it]
51%|█████ | 795/1570 [58:56<37:41, 2.92s/it]
51%|█████ | 796/1570 [58:59<37:43, 2.92s/it]
51%|█████ | 797/1570 [59:02<37:34, 2.92s/it]
51%|█████ | 798/1570 [59:04<37:23, 2.91s/it]
51%|█████ | 799/1570 [59:07<37:22, 2.91s/it]
51%|█████ | 800/1570 [59:10<37:35, 2.93s/it]
{'loss': '0.006191', 'grad_norm': '0.1051', 'learning_rate': '4.865e-05', 'epoch': '2.549'}
+
51%|█████ | 800/1570 [59:10<37:35, 2.93s/it]
51%|█████ | 801/1570 [59:13<37:27, 2.92s/it]
51%|█████ | 802/1570 [59:16<37:16, 2.91s/it]
51%|█████ | 803/1570 [59:19<37:13, 2.91s/it]
51%|█████ | 804/1570 [59:22<37:01, 2.90s/it]
51%|█████▏ | 805/1570 [59:25<36:58, 2.90s/it]
51%|█████▏ | 806/1570 [59:28<36:51, 2.89s/it]
51%|█████▏ | 807/1570 [59:31<36:50, 2.90s/it]
51%|█████▏ | 808/1570 [59:33<36:40, 2.89s/it]
52%|█████▏ | 809/1570 [59:36<36:39, 2.89s/it]
52%|█████▏ | 810/1570 [59:39<36:37, 2.89s/it]
52%|█████▏ | 811/1570 [59:42<36:29, 2.88s/it]
52%|█████▏ | 812/1570 [59:45<36:32, 2.89s/it]
52%|█████▏ | 813/1570 [59:48<36:26, 2.89s/it]
52%|█████▏ | 814/1570 [59:51<36:29, 2.90s/it]
52%|█████▏ | 815/1570 [59:54<36:21, 2.89s/it]
52%|█████▏ | 816/1570 [59:57<36:59, 2.94s/it]
52%|█████▏ | 817/1570 [1:00:00<36:55, 2.94s/it]
52%|█████▏ | 818/1570 [1:00:03<36:58, 2.95s/it]
52%|█████▏ | 819/1570 [1:00:06<37:12, 2.97s/it]
52%|█████▏ | 820/1570 [1:00:09<36:56, 2.95s/it]
{'loss': '0.01037', 'grad_norm': '0.2308', 'learning_rate': '4.665e-05', 'epoch': '2.613'}
+
52%|█████▏ | 820/1570 [1:00:09<36:56, 2.95s/it]
52%|█████▏ | 821/1570 [1:00:12<36:51, 2.95s/it]
52%|█████▏ | 822/1570 [1:00:15<36:50, 2.96s/it]
52%|█████▏ | 823/1570 [1:00:18<36:51, 2.96s/it]
52%|█████▏ | 824/1570 [1:00:20<36:39, 2.95s/it]
53%|█████▎ | 825/1570 [1:00:23<36:54, 2.97s/it]
53%|█████▎ | 826/1570 [1:00:27<38:53, 3.14s/it]
53%|█████▎ | 827/1570 [1:00:30<40:05, 3.24s/it]
53%|█████▎ | 828/1570 [1:00:34<41:17, 3.34s/it]
53%|█████▎ | 829/1570 [1:00:38<41:57, 3.40s/it]
53%|█████▎ | 830/1570 [1:00:41<42:02, 3.41s/it]
53%|█████▎ | 831/1570 [1:00:44<42:10, 3.42s/it]
53%|█████▎ | 832/1570 [1:00:48<43:17, 3.52s/it]
53%|█████▎ | 833/1570 [1:00:52<43:50, 3.57s/it]
53%|█████▎ | 834/1570 [1:00:55<43:50, 3.57s/it]
53%|█████▎ | 835/1570 [1:00:58<41:39, 3.40s/it]
53%|█████▎ | 836/1570 [1:01:02<41:58, 3.43s/it]
53%|█████▎ | 837/1570 [1:01:06<42:49, 3.51s/it]
53%|█████▎ | 838/1570 [1:01:09<42:40, 3.50s/it]
53%|█████▎ | 839/1570 [1:01:13<42:45, 3.51s/it]
54%|█████▎ | 840/1570 [1:01:16<43:11, 3.55s/it]
{'loss': '0.003532', 'grad_norm': '0.5436', 'learning_rate': '4.466e-05', 'epoch': '2.677'}
+
54%|█████▎ | 840/1570 [1:01:16<43:11, 3.55s/it]
54%|█████▎ | 841/1570 [1:01:20<43:04, 3.55s/it]
54%|█████▎ | 842/1570 [1:01:23<40:43, 3.36s/it]
54%|█████▎ | 843/1570 [1:01:26<38:54, 3.21s/it]
54%|█████▍ | 844/1570 [1:01:29<37:45, 3.12s/it]
54%|█████▍ | 845/1570 [1:01:31<36:48, 3.05s/it]
54%|█████▍ | 846/1570 [1:01:34<36:19, 3.01s/it]
54%|█████▍ | 847/1570 [1:01:37<35:47, 2.97s/it]
54%|█████▍ | 848/1570 [1:01:40<35:20, 2.94s/it]
54%|█████▍ | 849/1570 [1:01:43<35:00, 2.91s/it]
54%|█████▍ | 850/1570 [1:01:46<34:57, 2.91s/it]
54%|█████▍ | 851/1570 [1:01:49<34:43, 2.90s/it]
54%|█████▍ | 852/1570 [1:01:52<34:35, 2.89s/it]
54%|█████▍ | 853/1570 [1:01:54<34:29, 2.89s/it]
54%|█████▍ | 854/1570 [1:01:57<34:33, 2.90s/it]
54%|█████▍ | 855/1570 [1:02:00<34:28, 2.89s/it]
55%|█████▍ | 856/1570 [1:02:03<34:23, 2.89s/it]
55%|█████▍ | 857/1570 [1:02:06<34:28, 2.90s/it]
55%|█████▍ | 858/1570 [1:02:09<34:23, 2.90s/it]
55%|█████▍ | 859/1570 [1:02:12<34:08, 2.88s/it]
55%|█████▍ | 860/1570 [1:02:15<34:15, 2.89s/it]
{'loss': '0.004157', 'grad_norm': '0.3665', 'learning_rate': '4.268e-05', 'epoch': '2.74'}
+
55%|█████▍ | 860/1570 [1:02:15<34:15, 2.89s/it]
55%|█████▍ | 861/1570 [1:02:18<34:02, 2.88s/it]
55%|█████▍ | 862/1570 [1:02:20<33:57, 2.88s/it]
55%|█████▍ | 863/1570 [1:02:23<33:53, 2.88s/it]
55%|█████▌ | 864/1570 [1:02:26<33:55, 2.88s/it]
55%|█████▌ | 865/1570 [1:02:29<33:58, 2.89s/it]
55%|█████▌ | 866/1570 [1:02:32<33:42, 2.87s/it]
55%|█████▌ | 867/1570 [1:02:35<33:36, 2.87s/it]
55%|█████▌ | 868/1570 [1:02:38<33:27, 2.86s/it]
55%|█████▌ | 869/1570 [1:02:41<33:32, 2.87s/it]
55%|█████▌ | 870/1570 [1:02:43<33:31, 2.87s/it]
55%|█████▌ | 871/1570 [1:02:46<33:32, 2.88s/it]
56%|█████▌ | 872/1570 [1:02:49<33:27, 2.88s/it]
56%|█████▌ | 873/1570 [1:02:52<33:23, 2.87s/it]
56%|█████▌ | 874/1570 [1:02:55<33:22, 2.88s/it]
56%|█████▌ | 875/1570 [1:02:58<33:23, 2.88s/it]
56%|█████▌ | 876/1570 [1:03:01<33:16, 2.88s/it]
56%|█████▌ | 877/1570 [1:03:04<33:25, 2.89s/it]
56%|█████▌ | 878/1570 [1:03:07<33:26, 2.90s/it]
56%|█████▌ | 879/1570 [1:03:09<33:32, 2.91s/it]
56%|█████▌ | 880/1570 [1:03:13<34:19, 2.98s/it]
{'loss': '0.003846', 'grad_norm': '0.1195', 'learning_rate': '4.071e-05', 'epoch': '2.804'}
+
56%|█████▌ | 880/1570 [1:03:13<34:19, 2.98s/it]
56%|█████▌ | 881/1570 [1:03:16<35:21, 3.08s/it]
56%|█████▌ | 882/1570 [1:03:19<35:58, 3.14s/it]
56%|█████▌ | 883/1570 [1:03:22<36:19, 3.17s/it]
56%|█████▋ | 884/1570 [1:03:26<36:41, 3.21s/it]
56%|█████▋ | 885/1570 [1:03:29<36:51, 3.23s/it]
56%|█████▋ | 886/1570 [1:03:32<37:04, 3.25s/it]
56%|█████▋ | 887/1570 [1:03:36<37:47, 3.32s/it]
57%|█████▋ | 888/1570 [1:03:39<38:18, 3.37s/it]
57%|█████▋ | 889/1570 [1:03:43<39:05, 3.44s/it]
57%|█████▋ | 890/1570 [1:03:47<39:30, 3.49s/it]
57%|█████▋ | 891/1570 [1:03:50<39:48, 3.52s/it]
57%|█████▋ | 892/1570 [1:03:54<39:59, 3.54s/it]
57%|█████▋ | 893/1570 [1:03:57<40:07, 3.56s/it]
57%|█████▋ | 894/1570 [1:04:01<39:49, 3.54s/it]
57%|█████▋ | 895/1570 [1:04:04<40:00, 3.56s/it]
57%|█████▋ | 896/1570 [1:04:08<40:06, 3.57s/it]
57%|█████▋ | 897/1570 [1:04:12<40:05, 3.57s/it]
57%|█████▋ | 898/1570 [1:04:15<40:14, 3.59s/it]
57%|█████▋ | 899/1570 [1:04:19<40:04, 3.58s/it]
57%|█████▋ | 900/1570 [1:04:22<39:31, 3.54s/it]
{'loss': '0.005603', 'grad_norm': '0.1879', 'learning_rate': '3.875e-05', 'epoch': '2.868'}
+
57%|█████▋ | 900/1570 [1:04:22<39:31, 3.54s/it]
57%|█████▋ | 901/1570 [1:04:26<39:44, 3.56s/it]
57%|█████▋ | 902/1570 [1:04:29<39:49, 3.58s/it]
58%|█████▊ | 903/1570 [1:04:33<39:41, 3.57s/it]
58%|█████▊ | 904/1570 [1:04:37<39:40, 3.57s/it]
58%|█████▊ | 905/1570 [1:04:40<39:45, 3.59s/it]
58%|█████▊ | 906/1570 [1:04:44<39:41, 3.59s/it]
58%|█████▊ | 907/1570 [1:04:47<39:26, 3.57s/it]
58%|█████▊ | 908/1570 [1:04:51<39:34, 3.59s/it]
58%|█████▊ | 909/1570 [1:04:55<39:38, 3.60s/it]
58%|█████▊ | 910/1570 [1:04:58<39:30, 3.59s/it]
58%|█████▊ | 911/1570 [1:05:02<39:09, 3.57s/it]
58%|█████▊ | 912/1570 [1:05:05<39:10, 3.57s/it]
58%|█████▊ | 913/1570 [1:05:09<39:01, 3.56s/it]
58%|█████▊ | 914/1570 [1:05:12<39:08, 3.58s/it]
58%|█████▊ | 915/1570 [1:05:16<39:50, 3.65s/it]
58%|█████▊ | 916/1570 [1:05:20<41:27, 3.80s/it]
58%|█████▊ | 917/1570 [1:05:24<42:05, 3.87s/it]
58%|█████▊ | 918/1570 [1:05:29<42:53, 3.95s/it]
59%|█████▊ | 919/1570 [1:05:33<43:29, 4.01s/it]
59%|█████▊ | 920/1570 [1:05:37<43:30, 4.02s/it]
{'loss': '0.006679', 'grad_norm': '0.3081', 'learning_rate': '3.682e-05', 'epoch': '2.932'}
+
59%|█████▊ | 920/1570 [1:05:37<43:30, 4.02s/it]
59%|█████▊ | 921/1570 [1:05:41<43:40, 4.04s/it]
59%|█████▊ | 922/1570 [1:05:45<44:00, 4.07s/it]
59%|█████▉ | 923/1570 [1:05:49<44:06, 4.09s/it]
59%|█████▉ | 924/1570 [1:05:53<43:41, 4.06s/it]
59%|█████▉ | 925/1570 [1:05:57<43:43, 4.07s/it]
59%|█████▉ | 926/1570 [1:06:01<43:16, 4.03s/it]
59%|█████▉ | 927/1570 [1:06:05<43:39, 4.07s/it]
59%|█████▉ | 928/1570 [1:06:09<43:44, 4.09s/it]
59%|█████▉ | 929/1570 [1:06:14<43:58, 4.12s/it]
59%|█████▉ | 930/1570 [1:06:17<42:36, 3.99s/it]
59%|█████▉ | 931/1570 [1:06:21<41:17, 3.88s/it]
59%|█████▉ | 932/1570 [1:06:24<39:56, 3.76s/it]
59%|█████▉ | 933/1570 [1:06:28<39:11, 3.69s/it]
59%|█████▉ | 934/1570 [1:06:31<38:32, 3.64s/it]
60%|█████▉ | 935/1570 [1:06:35<37:57, 3.59s/it]
60%|█████▉ | 936/1570 [1:06:38<37:06, 3.51s/it]
60%|█████▉ | 937/1570 [1:06:42<36:24, 3.45s/it]
60%|█████▉ | 938/1570 [1:06:45<35:55, 3.41s/it]
60%|█████▉ | 939/1570 [1:06:48<35:07, 3.34s/it]
60%|█████▉ | 940/1570 [1:06:51<33:44, 3.21s/it]
{'loss': '0.003512', 'grad_norm': '0.09405', 'learning_rate': '3.49e-05', 'epoch': '2.996'}
+
60%|█████▉ | 940/1570 [1:06:51<33:44, 3.21s/it]
60%|█████▉ | 941/1570 [1:06:54<32:41, 3.12s/it]
60%|██████ | 942/1570 [1:06:55<26:28, 2.53s/it][INFO|trainer.py:2668] 2026-08-22 13:46:23,338 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 13:46:23,338 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 13:46:23,338 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/rkl/checkpoint-942
+[INFO|configuration_utils.py:780] 2026-08-22 13:56:29,080 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 13:56:29,083 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 13:56:29,118 >> chat template saved in results/qwen3/rkl/checkpoint-942/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 13:56:29,118 >> tokenizer config file saved in results/qwen3/rkl/checkpoint-942/tokenizer_config.json
+
60%|██████ | 943/1570 [1:17:05<32:09:47, 184.67s/it]
60%|██████ | 944/1570 [1:17:08<22:37:46, 130.14s/it]
60%|██████ | 945/1570 [1:17:10<15:58:07, 91.98s/it]
60%|██████ | 946/1570 [1:17:13<11:18:36, 65.25s/it]
60%|██████ | 947/1570 [1:17:16<8:03:17, 46.54s/it]
60%|██████ | 948/1570 [1:17:19<5:46:44, 33.45s/it]
60%|██████ | 949/1570 [1:17:22<4:11:20, 24.28s/it]
61%|██████ | 950/1570 [1:17:25<3:04:34, 17.86s/it]
61%|██████ | 951/1570 [1:17:28<2:17:56, 13.37s/it]
61%|██████ | 952/1570 [1:17:31<1:45:22, 10.23s/it]
61%|██████ | 953/1570 [1:17:34<1:22:37, 8.04s/it]
61%|██████ | 954/1570 [1:17:37<1:06:45, 6.50s/it]
61%|██████ | 955/1570 [1:17:40<55:42, 5.43s/it]
61%|██████ | 956/1570 [1:17:42<47:54, 4.68s/it]
61%|██████ | 957/1570 [1:17:45<42:25, 4.15s/it]
61%|██████ | 958/1570 [1:17:48<38:33, 3.78s/it]
61%|██████ | 959/1570 [1:17:51<35:47, 3.51s/it]
61%|██████ | 960/1570 [1:17:54<33:53, 3.33s/it]
{'loss': '0.004304', 'grad_norm': '0.1636', 'learning_rate': '3.301e-05', 'epoch': '3.057'}
+
61%|██████ | 960/1570 [1:17:54<33:53, 3.33s/it]
61%|██████ | 961/1570 [1:17:57<32:25, 3.19s/it]
61%|██████▏ | 962/1570 [1:18:00<31:25, 3.10s/it]
61%|██████▏ | 963/1570 [1:18:03<30:46, 3.04s/it]
61%|██████▏ | 964/1570 [1:18:06<30:20, 3.00s/it]
61%|██████▏ | 965/1570 [1:18:09<30:00, 2.98s/it]
62%|██████▏ | 966/1570 [1:18:11<29:44, 2.95s/it]
62%|██████▏ | 967/1570 [1:18:14<29:21, 2.92s/it]
62%|██████▏ | 968/1570 [1:18:17<29:19, 2.92s/it]
62%|██████▏ | 969/1570 [1:18:20<29:14, 2.92s/it]
62%|██████▏ | 970/1570 [1:18:23<29:14, 2.92s/it]
62%|██████▏ | 971/1570 [1:18:26<29:01, 2.91s/it]
62%|██████▏ | 972/1570 [1:18:29<28:43, 2.88s/it]
62%|██████▏ | 973/1570 [1:18:32<28:39, 2.88s/it]
62%|██████▏ | 974/1570 [1:18:35<28:37, 2.88s/it]
62%|██████▏ | 975/1570 [1:18:37<28:31, 2.88s/it]
62%|██████▏ | 976/1570 [1:18:40<28:25, 2.87s/it]
62%|██████▏ | 977/1570 [1:18:43<28:30, 2.88s/it]
62%|██████▏ | 978/1570 [1:18:46<28:28, 2.89s/it]
62%|██████▏ | 979/1570 [1:18:49<28:22, 2.88s/it]
62%|██████▏ | 980/1570 [1:18:52<28:27, 2.89s/it]
{'loss': '0.00188', 'grad_norm': '0.04238', 'learning_rate': '3.114e-05', 'epoch': '3.121'}
+
62%|██████▏ | 980/1570 [1:18:52<28:27, 2.89s/it]
62%|██████▏ | 981/1570 [1:18:55<28:24, 2.89s/it]
63%|██████▎ | 982/1570 [1:18:58<28:20, 2.89s/it]
63%|██████▎ | 983/1570 [1:19:01<28:21, 2.90s/it]
63%|██████▎ | 984/1570 [1:19:03<28:13, 2.89s/it]
63%|██████▎ | 985/1570 [1:19:06<28:09, 2.89s/it]
63%|██████▎ | 986/1570 [1:19:09<28:06, 2.89s/it]
63%|██████▎ | 987/1570 [1:19:12<27:59, 2.88s/it]
63%|██████▎ | 988/1570 [1:19:15<28:01, 2.89s/it]
63%|██████▎ | 989/1570 [1:19:18<28:00, 2.89s/it]
63%|██████▎ | 990/1570 [1:19:21<27:47, 2.88s/it]
63%|██████▎ | 991/1570 [1:19:24<27:49, 2.88s/it]
63%|██████▎ | 992/1570 [1:19:26<27:49, 2.89s/it]
63%|██████▎ | 993/1570 [1:19:29<27:40, 2.88s/it]
63%|██████▎ | 994/1570 [1:19:32<27:34, 2.87s/it]
63%|██████▎ | 995/1570 [1:19:35<27:37, 2.88s/it]
63%|██████▎ | 996/1570 [1:19:38<27:34, 2.88s/it]
64%|██████▎ | 997/1570 [1:19:41<27:23, 2.87s/it]
64%|██████▎ | 998/1570 [1:19:44<27:21, 2.87s/it]
64%|██████▎ | 999/1570 [1:19:47<27:27, 2.89s/it]
64%|██████▎ | 1000/1570 [1:19:50<27:30, 2.90s/it]
{'loss': '0.002382', 'grad_norm': '0.007349', 'learning_rate': '2.931e-05', 'epoch': '3.185'}
+
64%|██████▎ | 1000/1570 [1:19:50<27:30, 2.90s/it]
64%|██████▍ | 1001/1570 [1:19:52<27:27, 2.90s/it]
64%|██████▍ | 1002/1570 [1:19:55<27:16, 2.88s/it]
64%|██████▍ | 1003/1570 [1:19:58<27:23, 2.90s/it]
64%|██████▍ | 1004/1570 [1:20:01<27:19, 2.90s/it]
64%|██████▍ | 1005/1570 [1:20:04<27:16, 2.90s/it]
64%|██████▍ | 1006/1570 [1:20:07<27:10, 2.89s/it]
64%|██████▍ | 1007/1570 [1:20:10<27:12, 2.90s/it]
64%|██████▍ | 1008/1570 [1:20:13<27:02, 2.89s/it]
64%|██████▍ | 1009/1570 [1:20:16<26:56, 2.88s/it]
64%|██████▍ | 1010/1570 [1:20:18<26:50, 2.88s/it]
64%|██████▍ | 1011/1570 [1:20:21<26:59, 2.90s/it]
64%|██████▍ | 1012/1570 [1:20:24<26:55, 2.90s/it]
65%|██████▍ | 1013/1570 [1:20:27<26:56, 2.90s/it]
65%|██████▍ | 1014/1570 [1:20:30<26:52, 2.90s/it]
65%|██████▍ | 1015/1570 [1:20:33<26:50, 2.90s/it]
65%|██████▍ | 1016/1570 [1:20:36<26:53, 2.91s/it]
65%|██████▍ | 1017/1570 [1:20:39<26:51, 2.91s/it]
65%|██████▍ | 1018/1570 [1:20:42<26:38, 2.90s/it]
65%|██████▍ | 1019/1570 [1:20:45<26:30, 2.89s/it]
65%|██████▍ | 1020/1570 [1:20:47<26:24, 2.88s/it]
{'loss': '0.006687', 'grad_norm': '0.08004', 'learning_rate': '2.751e-05', 'epoch': '3.249'}
+
65%|██████▍ | 1020/1570 [1:20:47<26:24, 2.88s/it]
65%|██████▌ | 1021/1570 [1:20:50<26:22, 2.88s/it]
65%|██████▌ | 1022/1570 [1:20:53<26:17, 2.88s/it]
65%|██████▌ | 1023/1570 [1:20:56<26:10, 2.87s/it]
65%|██████▌ | 1024/1570 [1:20:59<26:06, 2.87s/it]
65%|██████▌ | 1025/1570 [1:21:02<26:02, 2.87s/it]
65%|██████▌ | 1026/1570 [1:21:05<26:05, 2.88s/it]
65%|██████▌ | 1027/1570 [1:21:08<26:09, 2.89s/it]
65%|██████▌ | 1028/1570 [1:21:10<26:02, 2.88s/it]
66%|██████▌ | 1029/1570 [1:21:13<26:03, 2.89s/it]
66%|██████▌ | 1030/1570 [1:21:16<25:57, 2.88s/it]
66%|██████▌ | 1031/1570 [1:21:19<25:55, 2.89s/it]
66%|██████▌ | 1032/1570 [1:21:22<25:59, 2.90s/it]
66%|██████▌ | 1033/1570 [1:21:25<25:58, 2.90s/it]
66%|██████▌ | 1034/1570 [1:21:28<26:00, 2.91s/it]
66%|██████▌ | 1035/1570 [1:21:31<25:47, 2.89s/it]
66%|██████▌ | 1036/1570 [1:21:34<25:37, 2.88s/it]
66%|██████▌ | 1037/1570 [1:21:36<25:43, 2.90s/it]
66%|██████▌ | 1038/1570 [1:21:39<25:46, 2.91s/it]
66%|██████▌ | 1039/1570 [1:21:42<25:43, 2.91s/it]
66%|██████▌ | 1040/1570 [1:21:45<25:29, 2.89s/it]
{'loss': '0.003947', 'grad_norm': '0.3457', 'learning_rate': '2.574e-05', 'epoch': '3.313'}
+
66%|██████▌ | 1040/1570 [1:21:45<25:29, 2.89s/it]
66%|██████▋ | 1041/1570 [1:21:48<25:13, 2.86s/it]
66%|██████▋ | 1042/1570 [1:21:51<25:13, 2.87s/it]
66%|██████▋ | 1043/1570 [1:21:54<25:12, 2.87s/it]
66%|██████▋ | 1044/1570 [1:21:57<25:14, 2.88s/it]
67%|██████▋ | 1045/1570 [1:22:00<25:09, 2.88s/it]
67%|██████▋ | 1046/1570 [1:22:02<25:05, 2.87s/it]
67%|██████▋ | 1047/1570 [1:22:05<24:58, 2.87s/it]
67%|██████▋ | 1048/1570 [1:22:08<24:56, 2.87s/it]
67%|██████▋ | 1049/1570 [1:22:11<25:04, 2.89s/it]
67%|██████▋ | 1050/1570 [1:22:14<25:00, 2.88s/it]
67%|██████▋ | 1051/1570 [1:22:17<24:50, 2.87s/it]
67%|██████▋ | 1052/1570 [1:22:20<24:50, 2.88s/it]
67%|██████▋ | 1053/1570 [1:22:23<24:48, 2.88s/it]
67%|██████▋ | 1054/1570 [1:22:25<24:41, 2.87s/it]
67%|██████▋ | 1055/1570 [1:22:28<24:32, 2.86s/it]
67%|██████▋ | 1056/1570 [1:22:31<24:40, 2.88s/it]
67%|██████▋ | 1057/1570 [1:22:34<24:29, 2.86s/it]
67%|██████▋ | 1058/1570 [1:22:37<24:38, 2.89s/it]
67%|██████▋ | 1059/1570 [1:22:40<24:39, 2.89s/it]
68%|██████▊ | 1060/1570 [1:22:43<24:40, 2.90s/it]
{'loss': '0.004489', 'grad_norm': '0.006578', 'learning_rate': '2.402e-05', 'epoch': '3.377'}
+
68%|██████▊ | 1060/1570 [1:22:43<24:40, 2.90s/it]
68%|██████▊ | 1061/1570 [1:22:46<24:46, 2.92s/it]
68%|██████▊ | 1062/1570 [1:22:49<24:40, 2.91s/it]
68%|██████▊ | 1063/1570 [1:22:51<24:25, 2.89s/it]
68%|██████▊ | 1064/1570 [1:22:54<24:25, 2.90s/it]
68%|██████▊ | 1065/1570 [1:22:57<24:27, 2.91s/it]
68%|██████▊ | 1066/1570 [1:23:00<24:25, 2.91s/it]
68%|██████▊ | 1067/1570 [1:23:03<24:17, 2.90s/it]
68%|██████▊ | 1068/1570 [1:23:06<24:13, 2.90s/it]
68%|██████▊ | 1069/1570 [1:23:09<24:21, 2.92s/it]
68%|██████▊ | 1070/1570 [1:23:12<24:13, 2.91s/it]
68%|██████▊ | 1071/1570 [1:23:15<24:19, 2.92s/it]
68%|██████▊ | 1072/1570 [1:23:18<24:21, 2.94s/it]
68%|██████▊ | 1073/1570 [1:23:21<24:17, 2.93s/it]
68%|██████▊ | 1074/1570 [1:23:24<24:14, 2.93s/it]
68%|██████▊ | 1075/1570 [1:23:26<24:03, 2.92s/it]
69%|██████▊ | 1076/1570 [1:23:29<23:56, 2.91s/it]
69%|██████▊ | 1077/1570 [1:23:32<23:44, 2.89s/it]
69%|██████▊ | 1078/1570 [1:23:35<23:49, 2.91s/it]
69%|██████▊ | 1079/1570 [1:23:38<23:36, 2.88s/it]
69%|██████▉ | 1080/1570 [1:23:41<23:30, 2.88s/it]
{'loss': '0.006796', 'grad_norm': '0.08877', 'learning_rate': '2.233e-05', 'epoch': '3.44'}
+
69%|██████▉ | 1080/1570 [1:23:41<23:30, 2.88s/it]
69%|██████▉ | 1081/1570 [1:23:44<23:30, 2.88s/it]
69%|██████▉ | 1082/1570 [1:23:47<23:25, 2.88s/it]
69%|██████▉ | 1083/1570 [1:23:50<23:25, 2.89s/it]
69%|██████▉ | 1084/1570 [1:23:52<23:27, 2.90s/it]
69%|██████▉ | 1085/1570 [1:23:55<23:21, 2.89s/it]
69%|██████▉ | 1086/1570 [1:23:58<23:16, 2.88s/it]
69%|██████▉ | 1087/1570 [1:24:01<23:11, 2.88s/it]
69%|██████▉ | 1088/1570 [1:24:04<23:09, 2.88s/it]
69%|██████▉ | 1089/1570 [1:24:07<23:05, 2.88s/it]
69%|██████▉ | 1090/1570 [1:24:10<23:00, 2.88s/it]
69%|██████▉ | 1091/1570 [1:24:13<22:56, 2.87s/it]
70%|██████▉ | 1092/1570 [1:24:15<22:59, 2.89s/it]
70%|██████▉ | 1093/1570 [1:24:18<22:52, 2.88s/it]
70%|██████▉ | 1094/1570 [1:24:21<22:47, 2.87s/it]
70%|██████▉ | 1095/1570 [1:24:24<22:41, 2.87s/it]
70%|██████▉ | 1096/1570 [1:24:27<22:43, 2.88s/it]
70%|██████▉ | 1097/1570 [1:24:30<22:44, 2.88s/it]
70%|██████▉ | 1098/1570 [1:24:33<22:47, 2.90s/it]
70%|███████ | 1099/1570 [1:24:36<22:50, 2.91s/it]
70%|███████ | 1100/1570 [1:24:39<22:43, 2.90s/it]
{'loss': '0.001858', 'grad_norm': '0.03748', 'learning_rate': '2.069e-05', 'epoch': '3.504'}
+
70%|███████ | 1100/1570 [1:24:39<22:43, 2.90s/it]
70%|███████ | 1101/1570 [1:24:42<22:45, 2.91s/it]
70%|███████ | 1102/1570 [1:24:44<22:42, 2.91s/it]
70%|███████ | 1103/1570 [1:24:47<22:37, 2.91s/it]
70%|███████ | 1104/1570 [1:24:50<22:25, 2.89s/it]
70%|███████ | 1105/1570 [1:24:53<22:21, 2.88s/it]
70%|███████ | 1106/1570 [1:24:56<22:21, 2.89s/it]
71%|███████ | 1107/1570 [1:24:59<22:21, 2.90s/it]
71%|███████ | 1108/1570 [1:25:02<22:10, 2.88s/it]
71%|███████ | 1109/1570 [1:25:05<22:06, 2.88s/it]
71%|███████ | 1110/1570 [1:25:07<22:02, 2.87s/it]
71%|███████ | 1111/1570 [1:25:10<21:57, 2.87s/it]
71%|███████ | 1112/1570 [1:25:13<21:54, 2.87s/it]
71%|███████ | 1113/1570 [1:25:16<21:56, 2.88s/it]
71%|███████ | 1114/1570 [1:25:19<21:52, 2.88s/it]
71%|███████ | 1115/1570 [1:25:22<21:55, 2.89s/it]
71%|███████ | 1116/1570 [1:25:25<21:50, 2.89s/it]
71%|███████ | 1117/1570 [1:25:28<21:50, 2.89s/it]
71%|███████ | 1118/1570 [1:25:31<21:41, 2.88s/it]
71%|███████▏ | 1119/1570 [1:25:33<21:42, 2.89s/it]
71%|███████▏ | 1120/1570 [1:25:36<21:31, 2.87s/it]
{'loss': '0.0009692', 'grad_norm': '0.04825', 'learning_rate': '1.91e-05', 'epoch': '3.568'}
+
71%|███████▏ | 1120/1570 [1:25:36<21:31, 2.87s/it]
71%|███████▏ | 1121/1570 [1:25:39<21:30, 2.87s/it]
71%|███████▏ | 1122/1570 [1:25:42<21:34, 2.89s/it]
72%|███████▏ | 1123/1570 [1:25:45<21:26, 2.88s/it]
72%|███████▏ | 1124/1570 [1:25:48<21:17, 2.86s/it]
72%|███████▏ | 1125/1570 [1:25:51<21:14, 2.86s/it]
72%|███████▏ | 1126/1570 [1:25:54<21:17, 2.88s/it]
72%|███████▏ | 1127/1570 [1:25:56<21:07, 2.86s/it]
72%|███████▏ | 1128/1570 [1:25:59<21:10, 2.87s/it]
72%|███████▏ | 1129/1570 [1:26:02<21:10, 2.88s/it]
72%|███████▏ | 1130/1570 [1:26:05<21:11, 2.89s/it]
72%|███████▏ | 1131/1570 [1:26:08<21:12, 2.90s/it]
72%|███████▏ | 1132/1570 [1:26:11<21:09, 2.90s/it]
72%|███████▏ | 1133/1570 [1:26:14<21:05, 2.89s/it]
72%|███████▏ | 1134/1570 [1:26:17<20:56, 2.88s/it]
72%|███████▏ | 1135/1570 [1:26:20<20:58, 2.89s/it]
72%|███████▏ | 1136/1570 [1:26:22<20:54, 2.89s/it]
72%|███████▏ | 1137/1570 [1:26:25<20:44, 2.87s/it]
72%|███████▏ | 1138/1570 [1:26:28<20:42, 2.88s/it]
73%|███████▎ | 1139/1570 [1:26:31<20:34, 2.86s/it]
73%|███████▎ | 1140/1570 [1:26:34<20:38, 2.88s/it]
{'loss': '0.001389', 'grad_norm': '0.3363', 'learning_rate': '1.755e-05', 'epoch': '3.632'}
+
73%|███████▎ | 1140/1570 [1:26:34<20:38, 2.88s/it]
73%|███████▎ | 1141/1570 [1:26:37<20:36, 2.88s/it]
73%|███████▎ | 1142/1570 [1:26:40<20:31, 2.88s/it]
73%|███████▎ | 1143/1570 [1:26:42<20:26, 2.87s/it]
73%|███████▎ | 1144/1570 [1:26:45<20:27, 2.88s/it]
73%|███████▎ | 1145/1570 [1:26:48<20:25, 2.88s/it]
73%|███████▎ | 1146/1570 [1:26:51<20:23, 2.89s/it]
73%|███████▎ | 1147/1570 [1:26:54<20:20, 2.89s/it]
73%|███████▎ | 1148/1570 [1:26:57<20:17, 2.89s/it]
73%|███████▎ | 1149/1570 [1:27:00<20:12, 2.88s/it]
73%|███████▎ | 1150/1570 [1:27:03<20:06, 2.87s/it]
73%|███████▎ | 1151/1570 [1:27:06<19:59, 2.86s/it]
73%|███████▎ | 1152/1570 [1:27:08<19:56, 2.86s/it]
73%|███████▎ | 1153/1570 [1:27:11<19:53, 2.86s/it]
74%|███████▎ | 1154/1570 [1:27:14<19:53, 2.87s/it]
74%|███████▎ | 1155/1570 [1:27:17<19:55, 2.88s/it]
74%|███████▎ | 1156/1570 [1:27:20<19:45, 2.86s/it]
74%|███████▎ | 1157/1570 [1:27:23<19:43, 2.87s/it]
74%|███████▍ | 1158/1570 [1:27:26<19:46, 2.88s/it]
74%|███████▍ | 1159/1570 [1:27:28<19:41, 2.88s/it]
74%|███████▍ | 1160/1570 [1:27:31<19:39, 2.88s/it]
{'loss': '0.006552', 'grad_norm': '0.0656', 'learning_rate': '1.606e-05', 'epoch': '3.696'}
+
74%|███████▍ | 1160/1570 [1:27:31<19:39, 2.88s/it]
74%|███████▍ | 1161/1570 [1:27:34<19:38, 2.88s/it]
74%|███████▍ | 1162/1570 [1:27:37<19:36, 2.88s/it]
74%|███████▍ | 1163/1570 [1:27:40<19:33, 2.88s/it]
74%|███████▍ | 1164/1570 [1:27:43<19:26, 2.87s/it]
74%|███████▍ | 1165/1570 [1:27:46<19:25, 2.88s/it]
74%|███████▍ | 1166/1570 [1:27:49<19:25, 2.88s/it]
74%|███████▍ | 1167/1570 [1:27:52<19:15, 2.87s/it]
74%|███████▍ | 1168/1570 [1:27:54<19:16, 2.88s/it]
74%|███████▍ | 1169/1570 [1:27:57<19:12, 2.87s/it]
75%|███████▍ | 1170/1570 [1:28:00<19:12, 2.88s/it]
75%|███████▍ | 1171/1570 [1:28:03<19:08, 2.88s/it]
75%|███████▍ | 1172/1570 [1:28:06<19:03, 2.87s/it]
75%|███████▍ | 1173/1570 [1:28:09<19:02, 2.88s/it]
75%|███████▍ | 1174/1570 [1:28:12<19:05, 2.89s/it]
75%|███████▍ | 1175/1570 [1:28:15<19:01, 2.89s/it]
75%|███████▍ | 1176/1570 [1:28:17<18:59, 2.89s/it]
75%|███████▍ | 1177/1570 [1:28:20<19:01, 2.90s/it]
75%|███████▌ | 1178/1570 [1:28:23<18:51, 2.89s/it]
75%|███████▌ | 1179/1570 [1:28:26<18:49, 2.89s/it]
75%|███████▌ | 1180/1570 [1:28:29<18:47, 2.89s/it]
{'loss': '0.002468', 'grad_norm': '0.04897', 'learning_rate': '1.462e-05', 'epoch': '3.759'}
+
75%|███████▌ | 1180/1570 [1:28:29<18:47, 2.89s/it]
75%|███████▌ | 1181/1570 [1:28:32<18:51, 2.91s/it]
75%|███████▌ | 1182/1570 [1:28:35<18:47, 2.91s/it]
75%|███████▌ | 1183/1570 [1:28:38<18:43, 2.90s/it]
75%|███████▌ | 1184/1570 [1:28:41<18:38, 2.90s/it]
75%|███████▌ | 1185/1570 [1:28:44<18:32, 2.89s/it]
76%|███████▌ | 1186/1570 [1:28:46<18:34, 2.90s/it]
76%|███████▌ | 1187/1570 [1:28:49<18:33, 2.91s/it]
76%|███████▌ | 1188/1570 [1:28:52<18:34, 2.92s/it]
76%|███████▌ | 1189/1570 [1:28:55<18:33, 2.92s/it]
76%|███████▌ | 1190/1570 [1:28:58<18:33, 2.93s/it]
76%|███████▌ | 1191/1570 [1:29:01<18:24, 2.91s/it]
76%|███████▌ | 1192/1570 [1:29:04<18:21, 2.91s/it]
76%|███████▌ | 1193/1570 [1:29:07<18:16, 2.91s/it]
76%|███████▌ | 1194/1570 [1:29:10<18:11, 2.90s/it]
76%|███████▌ | 1195/1570 [1:29:13<18:06, 2.90s/it]
76%|███████▌ | 1196/1570 [1:29:16<18:03, 2.90s/it]
76%|███████▌ | 1197/1570 [1:29:18<17:57, 2.89s/it]
76%|███████▋ | 1198/1570 [1:29:21<17:55, 2.89s/it]
76%|███████▋ | 1199/1570 [1:29:24<17:55, 2.90s/it]
76%|███████▋ | 1200/1570 [1:29:27<17:53, 2.90s/it]
{'loss': '0.002888', 'grad_norm': '0.03442', 'learning_rate': '1.324e-05', 'epoch': '3.823'}
+
76%|███████▋ | 1200/1570 [1:29:27<17:53, 2.90s/it]
76%|███████▋ | 1201/1570 [1:29:30<17:52, 2.91s/it]
77%|███████▋ | 1202/1570 [1:29:33<17:50, 2.91s/it]
77%|███████▋ | 1203/1570 [1:29:36<17:44, 2.90s/it]
77%|███████▋ | 1204/1570 [1:29:39<17:46, 2.91s/it]
77%|███████▋ | 1205/1570 [1:29:42<17:39, 2.90s/it]
77%|███████▋ | 1206/1570 [1:29:45<17:35, 2.90s/it]
77%|███████▋ | 1207/1570 [1:29:48<17:31, 2.90s/it]
77%|███████▋ | 1208/1570 [1:29:50<17:30, 2.90s/it]
77%|███████▋ | 1209/1570 [1:29:53<17:27, 2.90s/it]
77%|███████▋ | 1210/1570 [1:29:56<17:25, 2.90s/it]
77%|███████▋ | 1211/1570 [1:29:59<17:24, 2.91s/it]
77%|███████▋ | 1212/1570 [1:30:02<17:18, 2.90s/it]
77%|███████▋ | 1213/1570 [1:30:05<17:09, 2.88s/it]
77%|███████▋ | 1214/1570 [1:30:08<17:07, 2.89s/it]
77%|███████▋ | 1215/1570 [1:30:11<17:04, 2.89s/it]
77%|███████▋ | 1216/1570 [1:30:14<17:05, 2.90s/it]
78%|███████▊ | 1217/1570 [1:30:16<17:02, 2.90s/it]
78%|███████▊ | 1218/1570 [1:30:19<16:58, 2.89s/it]
78%|███████▊ | 1219/1570 [1:30:22<16:59, 2.90s/it]
78%|███████▊ | 1220/1570 [1:30:25<16:51, 2.89s/it]
{'loss': '0.005654', 'grad_norm': '0.1818', 'learning_rate': '1.192e-05', 'epoch': '3.887'}
+
78%|███████▊ | 1220/1570 [1:30:25<16:51, 2.89s/it]
78%|███████▊ | 1221/1570 [1:30:28<16:49, 2.89s/it]
78%|███████▊ | 1222/1570 [1:30:31<16:44, 2.89s/it]
78%|███████▊ | 1223/1570 [1:30:34<16:40, 2.88s/it]
78%|███████▊ | 1224/1570 [1:30:37<16:38, 2.89s/it]
78%|███████▊ | 1225/1570 [1:30:40<16:33, 2.88s/it]
78%|███████▊ | 1226/1570 [1:30:42<16:30, 2.88s/it]
78%|███████▊ | 1227/1570 [1:30:45<16:37, 2.91s/it]
78%|███████▊ | 1228/1570 [1:30:48<16:32, 2.90s/it]
78%|███████▊ | 1229/1570 [1:30:51<16:29, 2.90s/it]
78%|███████▊ | 1230/1570 [1:30:54<16:29, 2.91s/it]
78%|███████▊ | 1231/1570 [1:30:57<16:26, 2.91s/it]
78%|███████▊ | 1232/1570 [1:31:00<16:21, 2.90s/it]
79%|███████▊ | 1233/1570 [1:31:03<16:16, 2.90s/it]
79%|███████▊ | 1234/1570 [1:31:06<16:12, 2.89s/it]
79%|███████▊ | 1235/1570 [1:31:09<16:10, 2.90s/it]
79%|███████▊ | 1236/1570 [1:31:12<16:34, 2.98s/it]
79%|███████▉ | 1237/1570 [1:31:15<17:29, 3.15s/it]
79%|███████▉ | 1238/1570 [1:31:19<18:29, 3.34s/it]
79%|███████▉ | 1239/1570 [1:31:23<18:53, 3.42s/it]
79%|███████▉ | 1240/1570 [1:31:26<19:23, 3.53s/it]
{'loss': '0.0008344', 'grad_norm': '0.01794', 'learning_rate': '1.066e-05', 'epoch': '3.951'}
+
79%|███████▉ | 1240/1570 [1:31:26<19:23, 3.53s/it]
79%|███████▉ | 1241/1570 [1:31:30<19:22, 3.53s/it]
79%|███████▉ | 1242/1570 [1:31:34<19:18, 3.53s/it]
79%|███████▉ | 1243/1570 [1:31:37<19:13, 3.53s/it]
79%|███████▉ | 1244/1570 [1:31:40<18:52, 3.47s/it]
79%|███████▉ | 1245/1570 [1:31:43<17:56, 3.31s/it]
79%|███████▉ | 1246/1570 [1:31:46<17:26, 3.23s/it]
79%|███████▉ | 1247/1570 [1:31:49<17:04, 3.17s/it]
79%|███████▉ | 1248/1570 [1:31:52<16:50, 3.14s/it]
80%|███████▉ | 1249/1570 [1:31:55<16:26, 3.07s/it]
80%|███████▉ | 1250/1570 [1:31:58<16:12, 3.04s/it]
80%|███████▉ | 1251/1570 [1:32:01<15:56, 3.00s/it]
80%|███████▉ | 1252/1570 [1:32:04<15:45, 2.97s/it]
80%|███████▉ | 1253/1570 [1:32:07<15:38, 2.96s/it]
80%|███████▉ | 1254/1570 [1:32:10<15:29, 2.94s/it]
80%|███████▉ | 1255/1570 [1:32:13<15:28, 2.95s/it]
80%|████████ | 1256/1570 [1:32:14<12:42, 2.43s/it][INFO|trainer.py:2668] 2026-08-22 14:11:42,560 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 14:11:42,560 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 14:11:42,560 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/rkl/checkpoint-1256
+[INFO|configuration_utils.py:780] 2026-08-22 14:23:36,730 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 14:23:36,733 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 14:23:36,770 >> chat template saved in results/qwen3/rkl/checkpoint-1256/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 14:23:36,770 >> tokenizer config file saved in results/qwen3/rkl/checkpoint-1256/tokenizer_config.json
+
80%|████████ | 1257/1570 [1:44:12<18:52:47, 217.15s/it]
80%|████████ | 1258/1570 [1:44:15<13:14:53, 152.86s/it]
80%|████████ | 1259/1570 [1:44:18<9:19:05, 107.86s/it]
80%|████████ | 1260/1570 [1:44:21<6:34:33, 76.37s/it]
{'loss': '0.002629', 'grad_norm': '0.685', 'learning_rate': '9.464e-06', 'epoch': '4.013'}
+
80%|████████ | 1260/1570 [1:44:21<6:34:33, 76.37s/it]
80%|████████ | 1261/1570 [1:44:24<4:39:42, 54.31s/it]
80%|████████ | 1262/1570 [1:44:27<3:19:31, 38.87s/it]
80%|████████ | 1263/1570 [1:44:30<2:23:37, 28.07s/it]
81%|████████ | 1264/1570 [1:44:32<1:44:34, 20.51s/it]
81%|████████ | 1265/1570 [1:44:35<1:17:16, 15.20s/it]
81%|████████ | 1266/1570 [1:44:38<58:15, 11.50s/it]
81%|████████ | 1267/1570 [1:44:41<44:59, 8.91s/it]
81%|████████ | 1268/1570 [1:44:44<35:41, 7.09s/it]
81%|████████ | 1269/1570 [1:44:47<29:09, 5.81s/it]
81%|████████ | 1270/1570 [1:44:49<24:36, 4.92s/it]
81%|████████ | 1271/1570 [1:44:52<21:27, 4.31s/it]
81%|████████ | 1272/1570 [1:44:55<19:13, 3.87s/it]
81%|████████ | 1273/1570 [1:44:58<17:39, 3.57s/it]
81%|████████ | 1274/1570 [1:45:01<16:31, 3.35s/it]
81%|████████ | 1275/1570 [1:45:04<15:48, 3.22s/it]
81%|████████▏ | 1276/1570 [1:45:07<15:12, 3.10s/it]
81%|████████▏ | 1277/1570 [1:45:09<14:47, 3.03s/it]
81%|████████▏ | 1278/1570 [1:45:12<14:24, 2.96s/it]
81%|████████▏ | 1279/1570 [1:45:15<14:10, 2.92s/it]
82%|████████▏ | 1280/1570 [1:45:18<14:00, 2.90s/it]
{'loss': '0.0009219', 'grad_norm': '0.1672', 'learning_rate': '8.332e-06', 'epoch': '4.077'}
+
82%|████████▏ | 1280/1570 [1:45:18<14:00, 2.90s/it]
82%|████████▏ | 1281/1570 [1:45:21<13:51, 2.88s/it]
82%|████████▏ | 1282/1570 [1:45:24<13:44, 2.86s/it]
82%|████████▏ | 1283/1570 [1:45:26<13:37, 2.85s/it]
82%|████████▏ | 1284/1570 [1:45:29<13:36, 2.85s/it]
82%|████████▏ | 1285/1570 [1:45:32<13:28, 2.84s/it]
82%|████████▏ | 1286/1570 [1:45:35<13:21, 2.82s/it]
82%|████████▏ | 1287/1570 [1:45:38<13:15, 2.81s/it]
82%|████████▏ | 1288/1570 [1:45:40<13:10, 2.80s/it]
82%|████████▏ | 1289/1570 [1:45:43<13:05, 2.79s/it]
82%|████████▏ | 1290/1570 [1:45:46<13:06, 2.81s/it]
82%|████████▏ | 1291/1570 [1:45:49<13:04, 2.81s/it]
82%|████████▏ | 1292/1570 [1:45:52<13:02, 2.81s/it]
82%|████████▏ | 1293/1570 [1:45:55<13:02, 2.82s/it]
82%|████████▏ | 1294/1570 [1:45:57<12:57, 2.82s/it]
82%|████████▏ | 1295/1570 [1:46:00<12:56, 2.82s/it]
83%|████████▎ | 1296/1570 [1:46:03<12:54, 2.83s/it]
83%|████████▎ | 1297/1570 [1:46:06<12:53, 2.83s/it]
83%|████████▎ | 1298/1570 [1:46:09<12:45, 2.82s/it]
83%|████████▎ | 1299/1570 [1:46:11<12:44, 2.82s/it]
83%|████████▎ | 1300/1570 [1:46:14<12:39, 2.81s/it]
{'loss': '0.000969', 'grad_norm': '0.0727', 'learning_rate': '7.266e-06', 'epoch': '4.14'}
+
83%|████████▎ | 1300/1570 [1:46:14<12:39, 2.81s/it]
83%|████████▎ | 1301/1570 [1:46:17<12:40, 2.83s/it]
83%|████████▎ | 1302/1570 [1:46:20<12:36, 2.82s/it]
83%|████████▎ | 1303/1570 [1:46:23<12:34, 2.83s/it]
83%|████████▎ | 1304/1570 [1:46:26<12:31, 2.82s/it]
83%|████████▎ | 1305/1570 [1:46:28<12:31, 2.83s/it]
83%|████████▎ | 1306/1570 [1:46:31<12:25, 2.82s/it]
83%|████████▎ | 1307/1570 [1:46:34<12:23, 2.83s/it]
83%|████████▎ | 1308/1570 [1:46:37<12:20, 2.83s/it]
83%|████████▎ | 1309/1570 [1:46:40<12:18, 2.83s/it]
83%|████████▎ | 1310/1570 [1:46:43<12:16, 2.83s/it]
84%|████████▎ | 1311/1570 [1:46:45<12:13, 2.83s/it]
84%|████████▎ | 1312/1570 [1:46:48<12:09, 2.83s/it]
84%|████████▎ | 1313/1570 [1:46:51<12:03, 2.81s/it]
84%|████████▎ | 1314/1570 [1:46:54<11:57, 2.80s/it]
84%|████████▍ | 1315/1570 [1:46:57<11:57, 2.82s/it]
84%|████████▍ | 1316/1570 [1:46:59<11:55, 2.82s/it]
84%|████████▍ | 1317/1570 [1:47:02<11:49, 2.81s/it]
84%|████████▍ | 1318/1570 [1:47:05<11:49, 2.81s/it]
84%|████████▍ | 1319/1570 [1:47:08<11:45, 2.81s/it]
84%|████████▍ | 1320/1570 [1:47:11<11:46, 2.83s/it]
{'loss': '0.001098', 'grad_norm': '0.0144', 'learning_rate': '6.269e-06', 'epoch': '4.204'}
+
84%|████████▍ | 1320/1570 [1:47:11<11:46, 2.83s/it]
84%|████████▍ | 1321/1570 [1:47:14<11:44, 2.83s/it]
84%|████████▍ | 1322/1570 [1:47:16<11:40, 2.82s/it]
84%|████████▍ | 1323/1570 [1:47:19<11:37, 2.82s/it]
84%|████████▍ | 1324/1570 [1:47:22<11:34, 2.82s/it]
84%|████████▍ | 1325/1570 [1:47:25<11:35, 2.84s/it]
84%|████████▍ | 1326/1570 [1:47:28<11:33, 2.84s/it]
85%|████████▍ | 1327/1570 [1:47:31<11:32, 2.85s/it]
85%|████████▍ | 1328/1570 [1:47:33<11:27, 2.84s/it]
85%|████████▍ | 1329/1570 [1:47:36<11:28, 2.86s/it]
85%|████████▍ | 1330/1570 [1:47:39<11:21, 2.84s/it]
85%|████████▍ | 1331/1570 [1:47:42<11:12, 2.81s/it]
85%|████████▍ | 1332/1570 [1:47:45<11:15, 2.84s/it]
85%|████████▍ | 1333/1570 [1:47:48<11:10, 2.83s/it]
85%|████████▍ | 1334/1570 [1:47:50<11:07, 2.83s/it]
85%|████████▌ | 1335/1570 [1:47:53<11:02, 2.82s/it]
85%|████████▌ | 1336/1570 [1:47:56<10:55, 2.80s/it]
85%|████████▌ | 1337/1570 [1:47:59<10:56, 2.82s/it]
85%|████████▌ | 1338/1570 [1:48:02<10:56, 2.83s/it]
85%|████████▌ | 1339/1570 [1:48:05<10:56, 2.84s/it]
85%|████████▌ | 1340/1570 [1:48:07<10:54, 2.84s/it]
{'loss': '0.0009998', 'grad_norm': '0.03588', 'learning_rate': '5.342e-06', 'epoch': '4.268'}
+
85%|████████▌ | 1340/1570 [1:48:07<10:54, 2.84s/it]
85%|████████▌ | 1341/1570 [1:48:10<10:51, 2.84s/it]
85%|████████▌ | 1342/1570 [1:48:13<10:48, 2.84s/it]
86%|████████▌ | 1343/1570 [1:48:16<10:44, 2.84s/it]
86%|████████▌ | 1344/1570 [1:48:19<10:42, 2.85s/it]
86%|████████▌ | 1345/1570 [1:48:22<10:40, 2.85s/it]
86%|████████▌ | 1346/1570 [1:48:25<10:40, 2.86s/it]
86%|████████▌ | 1347/1570 [1:48:27<10:36, 2.85s/it]
86%|████████▌ | 1348/1570 [1:48:30<10:30, 2.84s/it]
86%|████████▌ | 1349/1570 [1:48:33<10:26, 2.84s/it]
86%|████████▌ | 1350/1570 [1:48:36<10:24, 2.84s/it]
86%|████████▌ | 1351/1570 [1:48:39<10:55, 2.99s/it]
86%|████████▌ | 1352/1570 [1:48:42<10:44, 2.96s/it]
86%|████████▌ | 1353/1570 [1:48:45<10:35, 2.93s/it]
86%|████████▌ | 1354/1570 [1:48:48<10:29, 2.91s/it]
86%|████████▋ | 1355/1570 [1:48:51<10:19, 2.88s/it]
86%|████████▋ | 1356/1570 [1:48:53<10:13, 2.86s/it]
86%|████████▋ | 1357/1570 [1:48:56<10:09, 2.86s/it]
86%|████████▋ | 1358/1570 [1:48:59<10:02, 2.84s/it]
87%|████████▋ | 1359/1570 [1:49:02<09:58, 2.84s/it]
87%|████████▋ | 1360/1570 [1:49:05<10:00, 2.86s/it]
{'loss': '0.006878', 'grad_norm': '2.991', 'learning_rate': '4.486e-06', 'epoch': '4.332'}
+
87%|████████▋ | 1360/1570 [1:49:05<10:00, 2.86s/it]
87%|████████▋ | 1361/1570 [1:49:08<09:56, 2.86s/it]
87%|████████▋ | 1362/1570 [1:49:11<09:53, 2.85s/it]
87%|████████▋ | 1363/1570 [1:49:13<09:49, 2.85s/it]
87%|████████▋ | 1364/1570 [1:49:16<09:48, 2.86s/it]
87%|████████▋ | 1365/1570 [1:49:19<09:45, 2.86s/it]
87%|████████▋ | 1366/1570 [1:49:22<09:41, 2.85s/it]
87%|████████▋ | 1367/1570 [1:49:25<09:39, 2.85s/it]
87%|████████▋ | 1368/1570 [1:49:28<09:36, 2.85s/it]
87%|████████▋ | 1369/1570 [1:49:30<09:32, 2.85s/it]
87%|████████▋ | 1370/1570 [1:49:33<09:30, 2.85s/it]
87%|████████▋ | 1371/1570 [1:49:36<09:29, 2.86s/it]
87%|████████▋ | 1372/1570 [1:49:39<09:27, 2.87s/it]
87%|████████▋ | 1373/1570 [1:49:42<09:24, 2.86s/it]
88%|████████▊ | 1374/1570 [1:49:45<09:22, 2.87s/it]
88%|████████▊ | 1375/1570 [1:49:48<09:21, 2.88s/it]
88%|████████▊ | 1376/1570 [1:49:51<09:17, 2.87s/it]
88%|████████▊ | 1377/1570 [1:49:53<09:11, 2.86s/it]
88%|████████▊ | 1378/1570 [1:49:56<09:12, 2.88s/it]
88%|████████▊ | 1379/1570 [1:49:59<09:09, 2.88s/it]
88%|████████▊ | 1380/1570 [1:50:02<09:05, 2.87s/it]
{'loss': '0.00488', 'grad_norm': '0.09274', 'learning_rate': '3.704e-06', 'epoch': '4.396'}
+
88%|████████▊ | 1380/1570 [1:50:02<09:05, 2.87s/it]
88%|████████▊ | 1381/1570 [1:50:05<09:06, 2.89s/it]
88%|████████▊ | 1382/1570 [1:50:08<09:02, 2.89s/it]
88%|████████▊ | 1383/1570 [1:50:11<08:59, 2.89s/it]
88%|████████▊ | 1384/1570 [1:50:14<08:55, 2.88s/it]
88%|████████▊ | 1385/1570 [1:50:17<08:54, 2.89s/it]
88%|████████▊ | 1386/1570 [1:50:19<08:49, 2.88s/it]
88%|████████▊ | 1387/1570 [1:50:22<08:49, 2.89s/it]
88%|████████▊ | 1388/1570 [1:50:25<08:46, 2.89s/it]
88%|████████▊ | 1389/1570 [1:50:28<08:44, 2.90s/it]
89%|████████▊ | 1390/1570 [1:50:31<09:01, 3.01s/it]
89%|████████▊ | 1391/1570 [1:50:34<08:51, 2.97s/it]
89%|████████▊ | 1392/1570 [1:50:37<08:39, 2.92s/it]
89%|████████▊ | 1393/1570 [1:50:40<08:31, 2.89s/it]
89%|████████▉ | 1394/1570 [1:50:43<08:27, 2.88s/it]
89%|████████▉ | 1395/1570 [1:50:46<08:23, 2.88s/it]
89%|████████▉ | 1396/1570 [1:50:48<08:18, 2.86s/it]
89%|████████▉ | 1397/1570 [1:50:51<08:14, 2.86s/it]
89%|████████▉ | 1398/1570 [1:50:54<08:10, 2.85s/it]
89%|████████▉ | 1399/1570 [1:50:57<08:08, 2.86s/it]
89%|████████▉ | 1400/1570 [1:51:00<08:06, 2.86s/it]
{'loss': '0.002034', 'grad_norm': '0.03078', 'learning_rate': '2.996e-06', 'epoch': '4.46'}
+
89%|████████▉ | 1400/1570 [1:51:00<08:06, 2.86s/it]
89%|████████▉ | 1401/1570 [1:51:03<08:00, 2.84s/it]
89%|████████▉ | 1402/1570 [1:51:06<07:59, 2.85s/it]
89%|████████▉ | 1403/1570 [1:51:08<07:57, 2.86s/it]
89%|████████▉ | 1404/1570 [1:51:11<07:56, 2.87s/it]
89%|████████▉ | 1405/1570 [1:51:14<07:53, 2.87s/it]
90%|████████▉ | 1406/1570 [1:51:17<07:49, 2.86s/it]
90%|████████▉ | 1407/1570 [1:51:20<07:45, 2.86s/it]
90%|████████▉ | 1408/1570 [1:51:23<07:42, 2.86s/it]
90%|████████▉ | 1409/1570 [1:51:26<07:39, 2.85s/it]
90%|████████▉ | 1410/1570 [1:51:28<07:37, 2.86s/it]
90%|████████▉ | 1411/1570 [1:51:31<07:33, 2.85s/it]
90%|████████▉ | 1412/1570 [1:51:34<07:32, 2.87s/it]
90%|█████████ | 1413/1570 [1:51:37<07:30, 2.87s/it]
90%|█████████ | 1414/1570 [1:51:40<07:26, 2.86s/it]
90%|█████████ | 1415/1570 [1:51:43<07:24, 2.87s/it]
90%|█████████ | 1416/1570 [1:51:46<07:20, 2.86s/it]
90%|█████████ | 1417/1570 [1:51:48<07:16, 2.85s/it]
90%|█████████ | 1418/1570 [1:51:51<07:11, 2.84s/it]
90%|█████████ | 1419/1570 [1:51:54<07:08, 2.84s/it]
90%|█████████ | 1420/1570 [1:51:57<07:08, 2.86s/it]
{'loss': '0.001902', 'grad_norm': '0.1409', 'learning_rate': '2.363e-06', 'epoch': '4.523'}
+
90%|█████████ | 1420/1570 [1:51:57<07:08, 2.86s/it]
91%|█████████ | 1421/1570 [1:52:00<07:04, 2.85s/it]
91%|█████████ | 1422/1570 [1:52:03<07:00, 2.84s/it]
91%|█████████ | 1423/1570 [1:52:05<06:56, 2.84s/it]
91%|█████████ | 1424/1570 [1:52:08<06:55, 2.85s/it]
91%|█████████ | 1425/1570 [1:52:11<06:54, 2.86s/it]
91%|█████████ | 1426/1570 [1:52:14<06:52, 2.86s/it]
91%|█████████ | 1427/1570 [1:52:17<06:47, 2.85s/it]
91%|█████████ | 1428/1570 [1:52:20<06:46, 2.86s/it]
91%|█████████ | 1429/1570 [1:52:23<06:43, 2.86s/it]
91%|█████████ | 1430/1570 [1:52:26<06:39, 2.86s/it]
91%|█████████ | 1431/1570 [1:52:28<06:37, 2.86s/it]
91%|█████████ | 1432/1570 [1:52:31<06:35, 2.86s/it]
91%|█████████▏| 1433/1570 [1:52:34<06:31, 2.86s/it]
91%|█████████▏| 1434/1570 [1:52:37<06:30, 2.87s/it]
91%|█████████▏| 1435/1570 [1:52:40<06:28, 2.87s/it]
91%|█████████▏| 1436/1570 [1:52:43<06:23, 2.86s/it]
92%|█████████▏| 1437/1570 [1:52:46<06:20, 2.86s/it]
92%|█████████▏| 1438/1570 [1:52:49<06:20, 2.88s/it]
92%|█████████▏| 1439/1570 [1:52:51<06:16, 2.88s/it]
92%|█████████▏| 1440/1570 [1:52:54<06:14, 2.88s/it]
{'loss': '0.001717', 'grad_norm': '0.01342', 'learning_rate': '1.806e-06', 'epoch': '4.587'}
+
92%|█████████▏| 1440/1570 [1:52:54<06:14, 2.88s/it]
92%|█████████▏| 1441/1570 [1:52:57<06:10, 2.87s/it]
92%|█████████▏| 1442/1570 [1:53:00<06:05, 2.85s/it]
92%|█████████▏| 1443/1570 [1:53:03<06:02, 2.86s/it]
92%|█████████▏| 1444/1570 [1:53:06<06:01, 2.87s/it]
92%|█████████▏| 1445/1570 [1:53:09<05:56, 2.85s/it]
92%|█████████▏| 1446/1570 [1:53:11<05:55, 2.86s/it]
92%|█████████▏| 1447/1570 [1:53:14<05:50, 2.85s/it]
92%|█████████▏| 1448/1570 [1:53:17<05:47, 2.85s/it]
92%|█████████▏| 1449/1570 [1:53:20<05:44, 2.85s/it]
92%|█████████▏| 1450/1570 [1:53:23<05:41, 2.85s/it]
92%|█████████▏| 1451/1570 [1:53:26<05:40, 2.86s/it]
92%|█████████▏| 1452/1570 [1:53:29<05:37, 2.86s/it]
93%|█████████▎| 1453/1570 [1:53:31<05:35, 2.87s/it]
93%|█████████▎| 1454/1570 [1:53:34<05:32, 2.87s/it]
93%|█████████▎| 1455/1570 [1:53:37<05:28, 2.86s/it]
93%|█████████▎| 1456/1570 [1:53:40<05:24, 2.85s/it]
93%|█████████▎| 1457/1570 [1:53:43<05:21, 2.85s/it]
93%|█████████▎| 1458/1570 [1:53:46<05:21, 2.87s/it]
93%|█████████▎| 1459/1570 [1:53:49<05:18, 2.87s/it]
93%|█████████▎| 1460/1570 [1:53:51<05:15, 2.87s/it]
{'loss': '0.0006548', 'grad_norm': '0.01554', 'learning_rate': '1.327e-06', 'epoch': '4.651'}
+
93%|█████████▎| 1460/1570 [1:53:51<05:15, 2.87s/it]
93%|█████████▎| 1461/1570 [1:53:54<05:11, 2.86s/it]
93%|█████████▎| 1462/1570 [1:53:57<05:08, 2.86s/it]
93%|█████████▎| 1463/1570 [1:54:00<05:07, 2.87s/it]
93%|█████████▎| 1464/1570 [1:54:03<05:04, 2.87s/it]
93%|█████████▎| 1465/1570 [1:54:06<05:00, 2.86s/it]
93%|█████████▎| 1466/1570 [1:54:09<04:58, 2.87s/it]
93%|█████████▎| 1467/1570 [1:54:12<04:56, 2.87s/it]
94%|█████████▎| 1468/1570 [1:54:14<04:52, 2.87s/it]
94%|█████████▎| 1469/1570 [1:54:17<04:49, 2.87s/it]
94%|█████████▎| 1470/1570 [1:54:20<04:46, 2.86s/it]
94%|█████████▎| 1471/1570 [1:54:23<04:42, 2.86s/it]
94%|█████████▍| 1472/1570 [1:54:26<04:39, 2.86s/it]
94%|█████████▍| 1473/1570 [1:54:29<04:37, 2.86s/it]
94%|█████████▍| 1474/1570 [1:54:32<04:34, 2.86s/it]
94%|█████████▍| 1475/1570 [1:54:34<04:32, 2.87s/it]
94%|█████████▍| 1476/1570 [1:54:37<04:29, 2.87s/it]
94%|█████████▍| 1477/1570 [1:54:40<04:25, 2.85s/it]
94%|█████████▍| 1478/1570 [1:54:43<04:22, 2.85s/it]
94%|█████████▍| 1479/1570 [1:54:46<04:18, 2.84s/it]
94%|█████████▍| 1480/1570 [1:54:49<04:16, 2.85s/it]
{'loss': '0.0006419', 'grad_norm': '0.0196', 'learning_rate': '9.258e-07', 'epoch': '4.715'}
+
94%|█████████▍| 1480/1570 [1:54:49<04:16, 2.85s/it]
94%|█████████▍| 1481/1570 [1:54:52<04:14, 2.86s/it]
94%|█████████▍| 1482/1570 [1:54:54<04:09, 2.83s/it]
94%|█████████▍| 1483/1570 [1:54:57<04:07, 2.84s/it]
95%|█████████▍| 1484/1570 [1:55:00<04:04, 2.84s/it]
95%|█████████▍| 1485/1570 [1:55:03<04:00, 2.83s/it]
95%|█████████▍| 1486/1570 [1:55:06<03:57, 2.83s/it]
95%|█████████▍| 1487/1570 [1:55:08<03:54, 2.83s/it]
95%|█████████▍| 1488/1570 [1:55:11<03:53, 2.85s/it]
95%|█████████▍| 1489/1570 [1:55:14<03:51, 2.85s/it]
95%|█████████▍| 1490/1570 [1:55:17<03:47, 2.85s/it]
95%|█████████▍| 1491/1570 [1:55:20<03:44, 2.85s/it]
95%|█████████▌| 1492/1570 [1:55:23<03:42, 2.85s/it]
95%|█████████▌| 1493/1570 [1:55:26<03:39, 2.85s/it]
95%|█████████▌| 1494/1570 [1:55:28<03:37, 2.86s/it]
95%|█████████▌| 1495/1570 [1:55:31<03:33, 2.85s/it]
95%|█████████▌| 1496/1570 [1:55:34<03:31, 2.85s/it]
95%|█████████▌| 1497/1570 [1:55:37<03:28, 2.86s/it]
95%|█████████▌| 1498/1570 [1:55:40<03:25, 2.85s/it]
95%|█████████▌| 1499/1570 [1:55:43<03:21, 2.83s/it]
96%|█████████▌| 1500/1570 [1:55:45<03:17, 2.82s/it]
{'loss': '0.001546', 'grad_norm': '0.01885', 'learning_rate': '6.033e-07', 'epoch': '4.779'}
+
96%|█████████▌| 1500/1570 [1:55:45<03:17, 2.82s/it]
96%|█████████▌| 1501/1570 [1:55:48<03:15, 2.84s/it]
96%|█████████▌| 1502/1570 [1:55:51<03:11, 2.82s/it]
96%|█████████▌| 1503/1570 [1:55:54<03:09, 2.83s/it]
96%|█████████▌| 1504/1570 [1:55:57<03:07, 2.84s/it]
96%|█████████▌| 1505/1570 [1:56:00<03:04, 2.84s/it]
96%|█████████▌| 1506/1570 [1:56:03<03:02, 2.85s/it]
96%|█████████▌| 1507/1570 [1:56:05<02:59, 2.85s/it]
96%|█████████▌| 1508/1570 [1:56:08<02:56, 2.85s/it]
96%|█████████▌| 1509/1570 [1:56:11<02:53, 2.85s/it]
96%|█████████▌| 1510/1570 [1:56:14<02:51, 2.86s/it]
96%|█████████▌| 1511/1570 [1:56:17<02:47, 2.85s/it]
96%|█████████▋| 1512/1570 [1:56:20<02:45, 2.85s/it]
96%|█████████▋| 1513/1570 [1:56:22<02:41, 2.84s/it]
96%|█████████▋| 1514/1570 [1:56:25<02:39, 2.84s/it]
96%|█████████▋| 1515/1570 [1:56:28<02:36, 2.84s/it]
97%|█████████▋| 1516/1570 [1:56:31<02:33, 2.84s/it]
97%|█████████▋| 1517/1570 [1:56:34<02:30, 2.84s/it]
97%|█████████▋| 1518/1570 [1:56:37<02:28, 2.85s/it]
97%|█████████▋| 1519/1570 [1:56:40<02:25, 2.85s/it]
97%|█████████▋| 1520/1570 [1:56:42<02:21, 2.84s/it]
{'loss': '0.009023', 'grad_norm': '7.28', 'learning_rate': '3.599e-07', 'epoch': '4.842'}
+
97%|█████████▋| 1520/1570 [1:56:42<02:21, 2.84s/it]
97%|█████████▋| 1521/1570 [1:56:45<02:19, 2.84s/it]
97%|█████████▋| 1522/1570 [1:56:48<02:16, 2.85s/it]
97%|█████████▋| 1523/1570 [1:56:51<02:13, 2.85s/it]
97%|█████████▋| 1524/1570 [1:56:54<02:11, 2.85s/it]
97%|█████████▋| 1525/1570 [1:56:57<02:08, 2.85s/it]
97%|█████████▋| 1526/1570 [1:56:59<02:05, 2.85s/it]
97%|█████████▋| 1527/1570 [1:57:02<02:02, 2.85s/it]
97%|█████████▋| 1528/1570 [1:57:05<01:59, 2.85s/it]
97%|█████████▋| 1529/1570 [1:57:08<01:56, 2.84s/it]
97%|█████████▋| 1530/1570 [1:57:11<01:54, 2.85s/it]
98%|█████████▊| 1531/1570 [1:57:14<01:51, 2.86s/it]
98%|█████████▊| 1532/1570 [1:57:17<01:48, 2.85s/it]
98%|█████████▊| 1533/1570 [1:57:19<01:45, 2.84s/it]
98%|█████████▊| 1534/1570 [1:57:22<01:42, 2.84s/it]
98%|█████████▊| 1535/1570 [1:57:25<01:40, 2.86s/it]
98%|█████████▊| 1536/1570 [1:57:28<01:37, 2.86s/it]
98%|█████████▊| 1537/1570 [1:57:31<01:34, 2.86s/it]
98%|█████████▊| 1538/1570 [1:57:34<01:31, 2.86s/it]
98%|█████████▊| 1539/1570 [1:57:37<01:28, 2.86s/it]
98%|█████████▊| 1540/1570 [1:57:39<01:25, 2.84s/it]
{'loss': '0.0006238', 'grad_norm': '0.08475', 'learning_rate': '1.961e-07', 'epoch': '4.906'}
+
98%|█████████▊| 1540/1570 [1:57:39<01:25, 2.84s/it]
98%|█████████▊| 1541/1570 [1:57:42<01:22, 2.85s/it]
98%|█████████▊| 1542/1570 [1:57:45<01:19, 2.85s/it]
98%|█████████▊| 1543/1570 [1:57:48<01:17, 2.86s/it]
98%|█████████▊| 1544/1570 [1:57:51<01:14, 2.85s/it]
98%|█████████▊| 1545/1570 [1:57:54<01:11, 2.86s/it]
98%|█████████▊| 1546/1570 [1:57:56<01:08, 2.85s/it]
99%|█████████▊| 1547/1570 [1:57:59<01:05, 2.85s/it]
99%|█████████▊| 1548/1570 [1:58:02<01:02, 2.84s/it]
99%|█████████▊| 1549/1570 [1:58:05<00:59, 2.85s/it]
99%|█████████▊| 1550/1570 [1:58:08<00:57, 2.87s/it]
99%|█████████▉| 1551/1570 [1:58:11<00:54, 2.86s/it]
99%|█████████▉| 1552/1570 [1:58:14<00:51, 2.86s/it]
99%|█████████▉| 1553/1570 [1:58:17<00:48, 2.86s/it]
99%|█████████▉| 1554/1570 [1:58:19<00:45, 2.85s/it]
99%|█████████▉| 1555/1570 [1:58:22<00:42, 2.85s/it]
99%|█████████▉| 1556/1570 [1:58:25<00:39, 2.85s/it]
99%|█████████▉| 1557/1570 [1:58:28<00:36, 2.84s/it]
99%|█████████▉| 1558/1570 [1:58:31<00:34, 2.85s/it]
99%|█████████▉| 1559/1570 [1:58:34<00:31, 2.83s/it]
99%|█████████▉| 1560/1570 [1:58:36<00:28, 2.84s/it]
{'loss': '0.002313', 'grad_norm': '0.04515', 'learning_rate': '1.121e-07', 'epoch': '4.97'}
+
99%|█████████▉| 1560/1570 [1:58:36<00:28, 2.84s/it]
99%|█████████▉| 1561/1570 [1:58:39<00:25, 2.84s/it]
99%|█████████▉| 1562/1570 [1:58:42<00:22, 2.85s/it]
100%|█████████▉| 1563/1570 [1:58:45<00:19, 2.85s/it]
100%|█████████▉| 1564/1570 [1:58:48<00:17, 2.84s/it]
100%|█████████▉| 1565/1570 [1:58:51<00:14, 2.84s/it]
100%|█████████▉| 1566/1570 [1:58:53<00:11, 2.85s/it]
100%|█████████▉| 1567/1570 [1:58:56<00:08, 2.85s/it]
100%|█████████▉| 1568/1570 [1:58:59<00:05, 2.85s/it]
100%|█████████▉| 1569/1570 [1:59:02<00:02, 2.84s/it]
100%|██████████| 1570/1570 [1:59:03<00:00, 2.33s/it][INFO|trainer.py:2668] 2026-08-22 14:38:31,521 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 14:38:31,521 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 14:38:31,521 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/rkl/checkpoint-1570
+[INFO|configuration_utils.py:780] 2026-08-22 14:48:33,300 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 14:48:33,301 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 14:48:33,337 >> chat template saved in results/qwen3/rkl/checkpoint-1570/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 14:48:33,337 >> tokenizer config file saved in results/qwen3/rkl/checkpoint-1570/tokenizer_config.json
+[INFO|trainer.py:1823] 2026-08-22 14:48:34,187 >>
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
{'train_runtime': '7746', 'train_samples_per_second': '6.472', 'train_steps_per_second': '0.203', 'train_loss': '0.051', 'epoch': '5'}
+
100%|██████████| 1570/1570 [2:09:06<00:00, 2.33s/it]
100%|██████████| 1570/1570 [2:09:06<00:00, 4.93s/it]
+[INFO|trainer.py:3815] 2026-08-22 14:48:34,700 >> Saving model checkpoint to results/qwen3/rkl
+[INFO|configuration_utils.py:780] 2026-08-22 14:48:35,191 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 14:48:35,194 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 14:48:35,229 >> chat template saved in results/qwen3/rkl/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 14:48:35,229 >> tokenizer config file saved in results/qwen3/rkl/tokenizer_config.json
+[transformers] The following generation flags are not valid and may be ignored: ['temperature', 'top_p']. Set `TRANSFORMERS_VERBOSITY=info` for more details.
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+***** train metrics *****
+ epoch = 5.0
+ total_flos = 54682091GF
+ train_loss = 0.051
+ train_runtime = 2:09:06.33
+ train_samples_per_second = 6.472
+ train_steps_per_second = 0.203
+[INFO|trainer.py:2668] 2026-08-22 14:48:35,380 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 14:48:35,380 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 14:48:35,380 >> Batch size = 16
+[WARNING|logging.py:340] 2026-08-22 14:48:35,495 >> The following generation flags are not valid and may be ignored: ['temperature', 'top_p'].
+[INFO|logging.py:355] 2026-08-22 14:48:35,495 >> - `temperature`: `do_sample` is not set to `True`. However, `temperature` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.
+- `top_p`: `do_sample` is not set to `True`. However, `top_p` is set to `0.7` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.
+If you're using a pretrained model, note that some of these attributes may be set through the model's `generation_config.json` file.
+[WARNING|utils.py:1642] 2026-08-22 14:48:35,498 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
0%| | 0/107 [00:00> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
2%|▏ | 2/107 [00:05<04:50, 2.77s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:48:46,669 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
3%|▎ | 3/107 [00:11<06:44, 3.89s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:48:52,132 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
4%|▎ | 4/107 [00:16<07:43, 4.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:48:57,640 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
5%|▍ | 5/107 [00:22<08:14, 4.85s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:49:03,160 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
6%|▌ | 6/107 [00:26<08:07, 4.83s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:49:07,946 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 7/107 [00:32<08:23, 5.04s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:49:13,419 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 8/107 [00:37<08:28, 5.13s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:49:18,763 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
8%|▊ | 9/107 [00:43<08:46, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:49:24,682 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
9%|▉ | 10/107 [00:49<08:51, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:49:30,384 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
10%|█ | 11/107 [00:53<08:21, 5.23s/it][WARNING|utils.py:1642] 2026-08-22 14:49:35,045 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
11%|█ | 12/107 [00:59<08:32, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:49:40,807 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
12%|█▏ | 13/107 [01:05<08:30, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:49:46,323 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
13%|█▎ | 14/107 [01:10<08:21, 5.39s/it][WARNING|utils.py:1642] 2026-08-22 14:49:51,635 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
14%|█▍ | 15/107 [01:16<08:19, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:49:57,164 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
15%|█▍ | 16/107 [01:21<08:17, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:50:02,715 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
16%|█▌ | 17/107 [01:27<08:21, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:50:08,514 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
17%|█▋ | 18/107 [01:33<08:17, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:50:14,143 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
18%|█▊ | 19/107 [01:38<08:04, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:50:19,467 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
19%|█▊ | 20/107 [01:43<07:55, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:50:24,834 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
20%|█▉ | 21/107 [01:48<07:33, 5.27s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:50:29,656 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██ | 22/107 [01:54<07:37, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:50:35,299 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██▏ | 23/107 [01:59<07:34, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:50:40,767 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
22%|██▏ | 24/107 [02:05<07:28, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:50:46,161 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
23%|██▎ | 25/107 [02:11<07:46, 5.69s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:50:52,502 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
24%|██▍ | 26/107 [02:17<07:42, 5.71s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:50:58,272 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
25%|██▌ | 27/107 [02:22<07:31, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:51:03,759 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
26%|██▌ | 28/107 [02:28<07:27, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:51:09,453 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
27%|██▋ | 29/107 [02:34<07:30, 5.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:51:15,511 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
28%|██▊ | 30/107 [02:39<07:15, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:51:20,889 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
29%|██▉ | 31/107 [02:44<06:44, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:51:25,448 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
30%|██▉ | 32/107 [02:49<06:42, 5.36s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:51:30,888 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
31%|███ | 33/107 [02:55<06:40, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:51:36,427 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
32%|███▏ | 34/107 [03:01<06:53, 5.67s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:51:42,678 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
33%|███▎ | 35/107 [03:07<06:48, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:51:48,384 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
34%|███▎ | 36/107 [03:12<06:36, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:51:53,729 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
35%|███▍ | 37/107 [03:17<06:13, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:51:58,479 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▌ | 38/107 [03:22<06:13, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:52:04,099 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▋ | 39/107 [03:27<05:58, 5.28s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:52:09,047 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
37%|███▋ | 40/107 [03:33<05:53, 5.27s/it][WARNING|utils.py:1642] 2026-08-22 14:52:14,312 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
38%|███▊ | 41/107 [03:38<05:55, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:52:19,947 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
39%|███▉ | 42/107 [03:44<06:03, 5.59s/it][WARNING|utils.py:1642] 2026-08-22 14:52:26,018 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
40%|████ | 43/107 [03:50<06:03, 5.67s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:52:31,887 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
41%|████ | 44/107 [03:56<05:53, 5.60s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:52:37,333 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
42%|████▏ | 45/107 [04:01<05:38, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:52:42,445 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
43%|████▎ | 46/107 [04:06<05:31, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:52:47,833 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
44%|████▍ | 47/107 [04:12<05:35, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:52:53,794 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
45%|████▍ | 48/107 [04:18<05:31, 5.62s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:52:59,469 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
46%|████▌ | 49/107 [04:23<05:21, 5.54s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:53:04,820 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
47%|████▋ | 50/107 [04:29<05:14, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:53:10,282 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
48%|████▊ | 51/107 [04:34<05:13, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:53:16,052 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
49%|████▊ | 52/107 [04:40<05:00, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:53:21,231 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|████▉ | 53/107 [04:46<05:02, 5.60s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:53:27,140 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|█████ | 54/107 [04:51<04:51, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:53:32,415 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
51%|█████▏ | 55/107 [04:57<04:49, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:53:38,142 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
52%|█████▏ | 56/107 [05:02<04:46, 5.62s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:53:43,895 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
53%|█████▎ | 57/107 [05:08<04:42, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:53:49,585 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
54%|█████▍ | 58/107 [05:13<04:34, 5.60s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:53:55,065 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
55%|█████▌ | 59/107 [05:19<04:27, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:54:00,558 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
56%|█████▌ | 60/107 [05:24<04:19, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:54:06,010 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
57%|█████▋ | 61/107 [05:29<04:05, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:54:10,887 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
58%|█████▊ | 62/107 [05:35<03:59, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:54:16,153 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
59%|█████▉ | 63/107 [05:40<03:55, 5.36s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:54:21,613 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
60%|█████▉ | 64/107 [05:45<03:49, 5.35s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:54:26,939 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
61%|██████ | 65/107 [05:51<03:42, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:54:32,158 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
62%|██████▏ | 66/107 [05:56<03:36, 5.29s/it][WARNING|utils.py:1642] 2026-08-22 14:54:37,388 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
63%|██████▎ | 67/107 [06:01<03:31, 5.28s/it][WARNING|utils.py:1642] 2026-08-22 14:54:42,660 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▎ | 68/107 [06:06<03:19, 5.12s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:54:47,415 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▍ | 69/107 [06:11<03:13, 5.10s/it][WARNING|utils.py:1642] 2026-08-22 14:54:52,459 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
65%|██████▌ | 70/107 [06:17<03:17, 5.34s/it][WARNING|utils.py:1642] 2026-08-22 14:54:58,370 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
66%|██████▋ | 71/107 [06:22<03:15, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:55:04,050 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
67%|██████▋ | 72/107 [06:28<03:08, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:55:09,281 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
68%|██████▊ | 73/107 [06:33<03:04, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:55:14,769 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
69%|██████▉ | 74/107 [06:39<02:58, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:55:20,134 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
70%|███████ | 75/107 [06:44<02:51, 5.35s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:55:25,370 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
71%|███████ | 76/107 [06:49<02:46, 5.37s/it][WARNING|utils.py:1642] 2026-08-22 14:55:30,785 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
72%|███████▏ | 77/107 [06:55<02:41, 5.39s/it][WARNING|utils.py:1642] 2026-08-22 14:55:36,210 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
73%|███████▎ | 78/107 [07:00<02:38, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:55:41,805 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
74%|███████▍ | 79/107 [07:06<02:32, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:55:47,193 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
75%|███████▍ | 80/107 [07:11<02:29, 5.52s/it][WARNING|utils.py:1642] 2026-08-22 14:55:52,933 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
76%|███████▌ | 81/107 [07:17<02:23, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:55:58,415 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
77%|███████▋ | 82/107 [07:22<02:16, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:56:03,767 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
78%|███████▊ | 83/107 [07:27<02:09, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:56:08,947 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▊ | 84/107 [07:32<02:01, 5.30s/it][WARNING|utils.py:1642] 2026-08-22 14:56:14,066 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▉ | 85/107 [07:38<02:00, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:56:19,951 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
80%|████████ | 86/107 [07:44<01:55, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:56:25,549 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
81%|████████▏ | 87/107 [07:49<01:44, 5.24s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:56:30,151 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
82%|████████▏ | 88/107 [07:54<01:40, 5.27s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:56:35,477 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
83%|████████▎ | 89/107 [08:00<01:41, 5.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:56:41,903 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
84%|████████▍ | 90/107 [08:06<01:34, 5.53s/it][WARNING|utils.py:1642] 2026-08-22 14:56:47,237 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
85%|████████▌ | 91/107 [08:11<01:26, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:56:52,278 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
86%|████████▌ | 92/107 [08:16<01:19, 5.28s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:56:57,331 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
87%|████████▋ | 93/107 [08:21<01:14, 5.30s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:57:02,682 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
88%|████████▊ | 94/107 [08:27<01:10, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:57:08,277 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
89%|████████▉ | 95/107 [08:32<01:04, 5.37s/it][WARNING|utils.py:1642] 2026-08-22 14:57:13,595 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
90%|████████▉ | 96/107 [08:37<00:58, 5.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:57:18,751 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
91%|█████████ | 97/107 [08:43<00:53, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:57:24,332 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
92%|█████████▏| 98/107 [08:48<00:49, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:57:29,928 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 99/107 [08:54<00:43, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:57:35,504 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 100/107 [08:59<00:37, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:57:40,771 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
94%|█████████▍| 101/107 [09:05<00:33, 5.55s/it][WARNING|utils.py:1642] 2026-08-22 14:57:46,633 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
95%|█████████▌| 102/107 [09:11<00:27, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:57:52,269 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
96%|█████████▋| 103/107 [09:16<00:22, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:57:57,607 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
97%|█████████▋| 104/107 [09:22<00:16, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:58:03,175 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
98%|█████████▊| 105/107 [09:27<00:11, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:58:08,910 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
99%|█████████▉| 106/107 [09:33<00:05, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 14:58:14,507 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
100%|██████████| 107/107 [09:39<00:00, 5.62s/it]
100%|██████████| 107/107 [09:40<00:00, 5.42s/it]
+***** eval metrics *****
+ epoch = 5.0
+ eval_generator_count = 1712.0
+ eval_generator_exact_match = 96.1449
+ eval_generator_rouge1 = 99.9473
+ eval_generator_rouge2 = 99.8921
+ eval_generator_rougeL = 99.7553
+ eval_loss = 0.0039
+ eval_runtime = 0:09:45.95
+ eval_samples_per_second = 5.826
+ eval_selector_accuracy = 98.7734
+ eval_selector_count = 1712.0
+ eval_steps_per_second = 0.183
+[INFO|modelcard.py:263] 2026-08-22 14:58:21,334 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
diff --git a/qwen3/run_all_logs/20260822_031516/sfkl.log b/qwen3/run_all_logs/20260822_031516/sfkl.log
new file mode 100644
index 0000000000000000000000000000000000000000..9ac1115392280ba282cb2ef1893713e9d7da27e5
--- /dev/null
+++ b/qwen3/run_all_logs/20260822_031516/sfkl.log
@@ -0,0 +1,2100 @@
+W0822 14:58:25.076000 1390402 torch/distributed/run.py:803]
+W0822 14:58:25.076000 1390402 torch/distributed/run.py:803] *****************************************
+W0822 14:58:25.076000 1390402 torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+W0822 14:58:25.076000 1390402 torch/distributed/run.py:803] *****************************************
+[INFO|2026-08-22 14:58:32] llamafactory.hparams.parser:651 >> Process rank: 1, world size: 2, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16
+[INFO|2026-08-22 14:58:32] llamafactory.hparams.parser:144 >> Set `ddp_find_unused_parameters` to False in DDP training since LoRA is enabled.
+[INFO|2026-08-22 14:58:32] llamafactory.hparams.parser:651 >> Process rank: 0, world size: 2, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16
+Starting sfkl with LlamaFactory and DeepSpeed=True.
+[INFO|configuration_utils.py:780] 2026-08-22 14:58:33,237 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 14:58:33,242 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 14:58:36,435 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 14:58:36,438 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 14:58:36,678 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 14:58:36,681 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 14:58:38] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_train.jsonl...
+[INFO|2026-08-22 14:58:38] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_eval.jsonl...
+training example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 1947, 19029, 10592, 311, 35592, 304, 279, 17328, 1947, 1707, 429, 9482, 1573, 279, 1042, 220, 16, 21, 15, 15, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 9286, 38050, 756, 414, 330, 13193, 788, 341, 286, 330, 408, 14645, 788, 330, 68689, 756, 286, 330, 2468, 14645, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 37525, 756, 414, 330, 13193, 788, 5613, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 18098, 287, 756, 414, 330, 13193, 788, 341, 286, 330, 11141, 3575, 34043, 788, 330, 16468, 756, 286, 330, 37375, 14645, 788, 330, 68689, 698, 414, 456, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 9286, 38050, 756, 414, 330, 1313, 788, 330, 49087, 2354, 698, 262, 1153, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 37525, 756, 414, 330, 1313, 788, 330, 4648, 37525, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of art movements linked to paintings in the genre art style that ended before the year 1600?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "ArtMovement",
+ "properties": {
+ "end_year": "INTEGER",
+ "start_year": "INTEGER"
+ }
+ },
+ {
+ "label": "Genre",
+ "properties": {}
+ },
+ {
+ "label": "Painting",
+ "properties": {
+ "country_of_origin": "STRING",
+ "creation_year": "INTEGER"
+ }
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "ArtMovement",
+ "type": "associatedWith"
+ },
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "Genre",
+ "type": "hasGenre"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+eval example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 9893, 1483, 304, 2987, 279, 220, 17, 15, 17, 16, 55218, 18144, 73379, 476, 279, 23007, 24847, 7755, 9257, 36331, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 13193, 788, 341, 286, 330, 1028, 788, 330, 7097, 756, 286, 330, 31309, 788, 330, 22852, 756, 286, 330, 4082, 3575, 2259, 26286, 788, 330, 68689, 756, 286, 330, 4082, 3575, 1243, 89827, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 28214, 756, 414, 330, 13193, 788, 5613, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 5657, 788, 330, 28214, 756, 414, 330, 1313, 788, 330, 30210, 1047, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of weapons used in either the 2021 Baghdad suicide bombings or the Botroseya Church bombing?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "TerroristAttack",
+ "properties": {
+ "date": "DATE",
+ "locations": "LIST",
+ "number_of_deaths": "INTEGER",
+ "number_of_injuries": "INTEGER"
+ }
+ },
+ {
+ "label": "Weapon",
+ "properties": {}
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "TerroristAttack",
+ "target": "Weapon",
+ "type": "employs"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+[INFO|configuration_utils.py:780] 2026-08-22 14:58:42,596 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 14:58:42,599 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 14:58:42] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training.
+[INFO|modeling_utils.py:769] 2026-08-22 14:58:43,355 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/model.safetensors
+[INFO|modeling_utils.py:839] 2026-08-22 14:58:43,355 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 14:58:43,357 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": false
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 14:58:43,985 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95
+}
+
+[INFO|2026-08-22 14:58:43] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled.
+[INFO|2026-08-22 14:58:43] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-22 14:58:43] llamafactory.model.adapter:144 >> Upcasting trainable params to float32.
+[INFO|2026-08-22 14:58:43] llamafactory.model.adapter:144 >> Fine-tuning method: LoRA
+[INFO|2026-08-22 14:58:43] llamafactory.model.model_utils.misc:144 >> Found linear modules: up_proj,q_proj,o_proj,gate_proj,k_proj,v_proj,down_proj
+[INFO|2026-08-22 14:58:44] llamafactory.model.loader:144 >> trainable params: 20,185,088 || all params: 616,235,008 || trainable%: 3.2756
+[INFO|configuration_utils.py:780] 2026-08-22 14:58:44,902 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 14:58:44,905 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 14:58:48,067 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 14:58:48,306 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 14:58:48,309 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 14:58:50,306 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 14:58:50,312 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 14:58:50] llamafactory.model.model_utils.kv_cache:144 >> KV cache is enabled for faster generation.
+[INFO|modeling_utils.py:769] 2026-08-22 14:58:50,551 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model.safetensors.index.json
+[INFO|modeling_utils.py:839] 2026-08-22 14:58:50,551 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 14:58:50,552 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": true
+}
+
+
Loading weights: 0%| | 0/398 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 14:58:52,671 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.7,
+ "top_k": 20,
+ "top_p": 0.8
+}
+
+[INFO|2026-08-22 14:58:52] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+
Loading weights: 0%| | 0/398 [00:00> Merged 1 adapter(s).
+[INFO|2026-08-22 14:58:53] llamafactory.model.adapter:144 >> Loaded adapter(s): results/qwen3/teacher_lora
+[INFO|2026-08-22 14:58:53] llamafactory.model.loader:144 >> all params: 4,022,468,096
+[INFO|2026-08-22 14:58:53] llamafactory.train.trainer_utils:144 >> Created reference model from Qwen/Qwen3-4B-Instruct-2507
+
Loading weights: 0%| | 1/398 [00:00<00:42, 9.42it/s]
Loading weights: 7%|▋ | 27/398 [00:00<00:02, 150.27it/s]
Loading weights: 15%|█▌ | 60/398 [00:00<00:01, 229.09it/s]
Loading weights: 23%|██▎ | 93/398 [00:00<00:01, 267.05it/s]
Loading weights: 32%|███▏ | 126/398 [00:00<00:00, 286.56it/s]
Loading weights: 40%|███▉ | 159/398 [00:00<00:00, 298.30it/s]
Loading weights: 48%|████▊ | 191/398 [00:00<00:00, 293.17it/s]
Loading weights: 57%|█████▋ | 225/398 [00:00<00:00, 300.64it/s]
Loading weights: 65%|██████▍ | 258/398 [00:00<00:00, 306.74it/s]
Loading weights: 73%|███████▎ | 290/398 [00:01<00:00, 306.83it/s]
Loading weights: 81%|████████▏ | 324/398 [00:01<00:00, 312.40it/s]
Loading weights: 89%|████████▉ | 356/398 [00:01<00:00, 306.52it/s]
Loading weights: 98%|█████████▊| 390/398 [00:01<00:00, 311.89it/s]
Loading weights: 100%|██████████| 398/398 [00:01<00:00, 289.51it/s]
+[transformers] The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}.
+begin bf16_optimizer
+MA 8.71 GB Max_MA 9.15 GB CA 9.37 GB Max_CA 9 GB
+CPU Virtual Memory: used = 10.6 GB, percent = 2.3%
+end bf16_ optimizer
+MA 8.71 GB Max_MA 8.71 GB CA 9.37 GB Max_CA 9 GB
+CPU Virtual Memory: used = 10.6 GB, percent = 2.3%
+[WARNING|trainer_utils.py:1238] 2026-08-22 14:58:57,849 >> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}.
+Gradient accumulation steps mismatch: GradientAccumulationPlugin has 1, DeepSpeed config has 8. Using DeepSpeed's value.
+[INFO|trainer.py:1469] 2026-08-22 14:58:58,428 >> ***** Running training *****
+[INFO|trainer.py:1470] 2026-08-22 14:58:58,428 >> Num examples = 10,027
+[INFO|trainer.py:1471] 2026-08-22 14:58:58,428 >> Num Epochs = 5
+[INFO|trainer.py:1472] 2026-08-22 14:58:58,428 >> Num update steps per epoch = 314
+[INFO|trainer.py:1473] 2026-08-22 14:58:58,428 >> Instantaneous batch size per device = 2
+[INFO|trainer.py:1476] 2026-08-22 14:58:58,428 >> Total train batch size (w. parallel, distributed & accumulation) = 32
+[INFO|trainer.py:1477] 2026-08-22 14:58:58,428 >> Gradient Accumulation steps = 8
+[INFO|trainer.py:1478] 2026-08-22 14:58:58,428 >> Total optimization steps = 1,570
+[INFO|trainer.py:1479] 2026-08-22 14:58:58,432 >> Number of trainable parameters = 20,185,088
+
0%| | 0/1570 [00:00>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 15:14:01,914 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 15:14:01,914 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/sfkl/checkpoint-314
+[INFO|configuration_utils.py:780] 2026-08-22 15:24:00,825 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 15:24:00,828 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 15:24:00,862 >> chat template saved in results/qwen3/sfkl/checkpoint-314/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 15:24:00,862 >> tokenizer config file saved in results/qwen3/sfkl/checkpoint-314/tokenizer_config.json
+
20%|██ | 315/1570 [25:06<63:36:32, 182.46s/it]
20%|██ | 316/1570 [25:09<44:47:15, 128.58s/it]
20%|██ | 317/1570 [25:11<31:37:23, 90.86s/it]
20%|██ | 318/1570 [25:14<22:25:10, 64.47s/it]
20%|██ | 319/1570 [25:17<15:58:41, 45.98s/it]
20%|██ | 320/1570 [25:20<11:28:13, 33.03s/it]
{'loss': '0.00897', 'grad_norm': '1.706', 'learning_rate': '9.016e-05', 'epoch': '1.019'}
+
20%|██ | 320/1570 [25:20<11:28:13, 33.03s/it]
20%|██ | 321/1570 [25:23<8:19:06, 23.98s/it]
21%|██ | 322/1570 [25:26<6:06:25, 17.62s/it]
21%|██ | 323/1570 [25:28<4:33:50, 13.18s/it]
21%|██ | 324/1570 [25:31<3:29:21, 10.08s/it]
21%|██ | 325/1570 [25:34<2:44:03, 7.91s/it]
21%|██ | 326/1570 [25:37<2:12:13, 6.38s/it]
21%|██ | 327/1570 [25:40<1:50:11, 5.32s/it]
21%|██ | 328/1570 [25:43<1:34:47, 4.58s/it]
21%|██ | 329/1570 [25:45<1:24:01, 4.06s/it]
21%|██ | 330/1570 [25:48<1:16:33, 3.70s/it]
21%|██ | 331/1570 [25:51<1:11:08, 3.44s/it]
21%|██ | 332/1570 [25:54<1:07:04, 3.25s/it]
21%|██ | 333/1570 [25:57<1:04:46, 3.14s/it]
21%|██▏ | 334/1570 [26:00<1:03:02, 3.06s/it]
21%|██▏ | 335/1570 [26:03<1:01:31, 2.99s/it]
21%|██▏ | 336/1570 [26:05<1:00:28, 2.94s/it]
21%|██▏ | 337/1570 [26:08<59:29, 2.90s/it]
22%|██▏ | 338/1570 [26:11<58:47, 2.86s/it]
22%|██▏ | 339/1570 [26:14<58:26, 2.85s/it]
22%|██▏ | 340/1570 [26:17<58:31, 2.85s/it]
{'loss': '0.003804', 'grad_norm': '0.1012', 'learning_rate': '8.894e-05', 'epoch': '1.083'}
+
22%|██▏ | 340/1570 [26:17<58:31, 2.85s/it]
22%|██▏ | 341/1570 [26:20<58:31, 2.86s/it]
22%|██▏ | 342/1570 [26:22<58:28, 2.86s/it]
22%|██▏ | 343/1570 [26:25<58:07, 2.84s/it]
22%|██▏ | 344/1570 [26:28<57:56, 2.84s/it]
22%|██▏ | 345/1570 [26:31<57:44, 2.83s/it]
22%|██▏ | 346/1570 [26:34<57:42, 2.83s/it]
22%|██▏ | 347/1570 [26:36<57:45, 2.83s/it]
22%|██▏ | 348/1570 [26:39<57:43, 2.83s/it]
22%|██▏ | 349/1570 [26:42<57:35, 2.83s/it]
22%|██▏ | 350/1570 [26:45<57:33, 2.83s/it]
22%|██▏ | 351/1570 [26:48<57:25, 2.83s/it]
22%|██▏ | 352/1570 [26:51<57:23, 2.83s/it]
22%|██▏ | 353/1570 [26:53<57:24, 2.83s/it]
23%|██▎ | 354/1570 [26:56<57:30, 2.84s/it]
23%|██▎ | 355/1570 [26:59<57:32, 2.84s/it]
23%|██▎ | 356/1570 [27:02<57:34, 2.85s/it]
23%|██▎ | 357/1570 [27:05<57:32, 2.85s/it]
23%|██▎ | 358/1570 [27:08<57:28, 2.85s/it]
23%|██▎ | 359/1570 [27:11<57:34, 2.85s/it]
23%|██▎ | 360/1570 [27:13<57:09, 2.83s/it]
{'loss': '0.005675', 'grad_norm': '0.3312', 'learning_rate': '8.766e-05', 'epoch': '1.147'}
+
23%|██▎ | 360/1570 [27:13<57:09, 2.83s/it]
23%|██▎ | 361/1570 [27:16<57:16, 2.84s/it]
23%|██▎ | 362/1570 [27:19<57:23, 2.85s/it]
23%|██▎ | 363/1570 [27:22<57:01, 2.83s/it]
23%|██▎ | 364/1570 [27:25<56:59, 2.84s/it]
23%|██▎ | 365/1570 [27:28<56:46, 2.83s/it]
23%|██▎ | 366/1570 [27:30<56:49, 2.83s/it]
23%|██▎ | 367/1570 [27:33<56:39, 2.83s/it]
23%|██▎ | 368/1570 [27:36<56:38, 2.83s/it]
24%|██▎ | 369/1570 [27:39<56:40, 2.83s/it]
24%|██▎ | 370/1570 [27:42<56:34, 2.83s/it]
24%|██▎ | 371/1570 [27:44<56:24, 2.82s/it]
24%|██▎ | 372/1570 [27:47<56:20, 2.82s/it]
24%|██▍ | 373/1570 [27:50<56:27, 2.83s/it]
24%|██▍ | 374/1570 [27:53<56:19, 2.83s/it]
24%|██▍ | 375/1570 [27:56<56:46, 2.85s/it]
24%|██▍ | 376/1570 [27:59<56:46, 2.85s/it]
24%|██▍ | 377/1570 [28:02<56:33, 2.84s/it]
24%|██▍ | 378/1570 [28:04<56:33, 2.85s/it]
24%|██▍ | 379/1570 [28:07<56:43, 2.86s/it]
24%|██▍ | 380/1570 [28:10<56:23, 2.84s/it]
{'loss': '0.004399', 'grad_norm': '0.2123', 'learning_rate': '8.631e-05', 'epoch': '1.211'}
+
24%|██▍ | 380/1570 [28:10<56:23, 2.84s/it]
24%|██▍ | 381/1570 [28:13<56:26, 2.85s/it]
24%|██▍ | 382/1570 [28:16<56:18, 2.84s/it]
24%|██▍ | 383/1570 [28:19<56:12, 2.84s/it]
24%|██▍ | 384/1570 [28:21<56:13, 2.84s/it]
25%|██▍ | 385/1570 [28:24<56:09, 2.84s/it]
25%|██▍ | 386/1570 [28:27<55:57, 2.84s/it]
25%|██▍ | 387/1570 [28:30<56:13, 2.85s/it]
25%|██▍ | 388/1570 [28:33<55:58, 2.84s/it]
25%|██▍ | 389/1570 [28:36<55:52, 2.84s/it]
25%|██▍ | 390/1570 [28:39<55:57, 2.85s/it]
25%|██▍ | 391/1570 [28:41<55:59, 2.85s/it]
25%|██▍ | 392/1570 [28:44<55:51, 2.85s/it]
25%|██▌ | 393/1570 [28:47<55:36, 2.84s/it]
25%|██▌ | 394/1570 [28:50<55:57, 2.86s/it]
25%|██▌ | 395/1570 [28:53<56:01, 2.86s/it]
25%|██▌ | 396/1570 [28:56<55:57, 2.86s/it]
25%|██▌ | 397/1570 [28:59<56:02, 2.87s/it]
25%|██▌ | 398/1570 [29:01<55:48, 2.86s/it]
25%|██▌ | 399/1570 [29:04<55:33, 2.85s/it]
25%|██▌ | 400/1570 [29:07<55:28, 2.84s/it]
{'loss': '0.006617', 'grad_norm': '0.1173', 'learning_rate': '8.491e-05', 'epoch': '1.274'}
+
25%|██▌ | 400/1570 [29:07<55:28, 2.84s/it]
26%|██▌ | 401/1570 [29:10<55:41, 2.86s/it]
26%|██▌ | 402/1570 [29:13<55:26, 2.85s/it]
26%|██▌ | 403/1570 [29:16<55:28, 2.85s/it]
26%|██▌ | 404/1570 [29:18<55:23, 2.85s/it]
26%|██▌ | 405/1570 [29:21<55:22, 2.85s/it]
26%|██▌ | 406/1570 [29:24<55:31, 2.86s/it]
26%|██▌ | 407/1570 [29:27<55:34, 2.87s/it]
26%|██▌ | 408/1570 [29:30<55:09, 2.85s/it]
26%|██▌ | 409/1570 [29:33<55:19, 2.86s/it]
26%|██▌ | 410/1570 [29:36<55:13, 2.86s/it]
26%|██▌ | 411/1570 [29:39<55:09, 2.86s/it]
26%|██▌ | 412/1570 [29:41<54:59, 2.85s/it]
26%|██▋ | 413/1570 [29:44<54:47, 2.84s/it]
26%|██▋ | 414/1570 [29:47<55:09, 2.86s/it]
26%|██▋ | 415/1570 [29:50<54:54, 2.85s/it]
26%|██▋ | 416/1570 [29:53<54:57, 2.86s/it]
27%|██▋ | 417/1570 [29:56<54:53, 2.86s/it]
27%|██▋ | 418/1570 [29:58<54:48, 2.85s/it]
27%|██▋ | 419/1570 [30:01<54:28, 2.84s/it]
27%|██▋ | 420/1570 [30:04<54:29, 2.84s/it]
{'loss': '0.005979', 'grad_norm': '0.1508', 'learning_rate': '8.345e-05', 'epoch': '1.338'}
+
27%|██▋ | 420/1570 [30:04<54:29, 2.84s/it]
27%|██▋ | 421/1570 [30:07<54:41, 2.86s/it]
27%|██▋ | 422/1570 [30:10<54:22, 2.84s/it]
27%|██▋ | 423/1570 [30:13<54:19, 2.84s/it]
27%|██▋ | 424/1570 [30:16<54:12, 2.84s/it]
27%|██▋ | 425/1570 [30:18<54:17, 2.84s/it]
27%|██▋ | 426/1570 [30:21<54:04, 2.84s/it]
27%|██▋ | 427/1570 [30:24<54:14, 2.85s/it]
27%|██▋ | 428/1570 [30:27<54:04, 2.84s/it]
27%|██▋ | 429/1570 [30:30<54:02, 2.84s/it]
27%|██▋ | 430/1570 [30:33<54:09, 2.85s/it]
27%|██▋ | 431/1570 [30:35<54:05, 2.85s/it]
28%|██▊ | 432/1570 [30:38<54:14, 2.86s/it]
28%|██▊ | 433/1570 [30:41<54:06, 2.86s/it]
28%|██▊ | 434/1570 [30:44<53:50, 2.84s/it]
28%|██▊ | 435/1570 [30:47<53:54, 2.85s/it]
28%|██▊ | 436/1570 [30:50<53:49, 2.85s/it]
28%|██▊ | 437/1570 [30:53<54:02, 2.86s/it]
28%|██▊ | 438/1570 [30:55<54:01, 2.86s/it]
28%|██▊ | 439/1570 [30:58<53:52, 2.86s/it]
28%|██▊ | 440/1570 [31:01<53:54, 2.86s/it]
{'loss': '0.004834', 'grad_norm': '0.03421', 'learning_rate': '8.194e-05', 'epoch': '1.402'}
+
28%|██▊ | 440/1570 [31:01<53:54, 2.86s/it]
28%|██▊ | 441/1570 [31:04<53:38, 2.85s/it]
28%|██▊ | 442/1570 [31:07<53:28, 2.84s/it]
28%|██▊ | 443/1570 [31:10<53:18, 2.84s/it]
28%|██▊ | 444/1570 [31:12<53:10, 2.83s/it]
28%|██▊ | 445/1570 [31:15<52:58, 2.83s/it]
28%|██▊ | 446/1570 [31:18<53:00, 2.83s/it]
28%|██▊ | 447/1570 [31:21<53:08, 2.84s/it]
29%|██▊ | 448/1570 [31:24<53:02, 2.84s/it]
29%|██▊ | 449/1570 [31:27<53:22, 2.86s/it]
29%|██▊ | 450/1570 [31:30<53:19, 2.86s/it]
29%|██▊ | 451/1570 [31:32<53:27, 2.87s/it]
29%|██▉ | 452/1570 [31:35<53:19, 2.86s/it]
29%|██▉ | 453/1570 [31:38<53:12, 2.86s/it]
29%|██▉ | 454/1570 [31:41<53:04, 2.85s/it]
29%|██▉ | 455/1570 [31:44<53:09, 2.86s/it]
29%|██▉ | 456/1570 [31:47<53:15, 2.87s/it]
29%|██▉ | 457/1570 [31:50<52:59, 2.86s/it]
29%|██▉ | 458/1570 [31:52<53:00, 2.86s/it]
29%|██▉ | 459/1570 [31:55<52:55, 2.86s/it]
29%|██▉ | 460/1570 [31:58<52:25, 2.83s/it]
{'loss': '0.006534', 'grad_norm': '0.3611', 'learning_rate': '8.037e-05', 'epoch': '1.466'}
+
29%|██▉ | 460/1570 [31:58<52:25, 2.83s/it]
29%|██▉ | 461/1570 [32:01<52:24, 2.84s/it]
29%|██▉ | 462/1570 [32:04<52:24, 2.84s/it]
29%|██▉ | 463/1570 [32:07<52:39, 2.85s/it]
30%|██▉ | 464/1570 [32:10<52:46, 2.86s/it]
30%|██▉ | 465/1570 [32:12<52:39, 2.86s/it]
30%|██▉ | 466/1570 [32:15<52:09, 2.83s/it]
30%|██▉ | 467/1570 [32:18<52:05, 2.83s/it]
30%|██▉ | 468/1570 [32:21<52:05, 2.84s/it]
30%|██▉ | 469/1570 [32:24<52:11, 2.84s/it]
30%|██▉ | 470/1570 [32:27<52:07, 2.84s/it]
30%|███ | 471/1570 [32:29<51:53, 2.83s/it]
30%|███ | 472/1570 [32:32<51:43, 2.83s/it]
30%|███ | 473/1570 [32:35<51:56, 2.84s/it]
30%|███ | 474/1570 [32:38<51:42, 2.83s/it]
30%|███ | 475/1570 [32:41<51:41, 2.83s/it]
30%|███ | 476/1570 [32:44<51:47, 2.84s/it]
30%|███ | 477/1570 [32:46<51:45, 2.84s/it]
30%|███ | 478/1570 [32:49<51:37, 2.84s/it]
31%|███ | 479/1570 [32:52<51:34, 2.84s/it]
31%|███ | 480/1570 [32:55<51:36, 2.84s/it]
{'loss': '0.004453', 'grad_norm': '0.3489', 'learning_rate': '7.876e-05', 'epoch': '1.53'}
+
31%|███ | 480/1570 [32:55<51:36, 2.84s/it]
31%|███ | 481/1570 [32:58<51:40, 2.85s/it]
31%|███ | 482/1570 [33:01<51:31, 2.84s/it]
31%|███ | 483/1570 [33:03<51:26, 2.84s/it]
31%|███ | 484/1570 [33:06<51:25, 2.84s/it]
31%|███ | 485/1570 [33:09<51:30, 2.85s/it]
31%|███ | 486/1570 [33:12<51:32, 2.85s/it]
31%|███ | 487/1570 [33:15<51:27, 2.85s/it]
31%|███ | 488/1570 [33:18<51:21, 2.85s/it]
31%|███ | 489/1570 [33:21<51:14, 2.84s/it]
31%|███ | 490/1570 [33:23<51:13, 2.85s/it]
31%|███▏ | 491/1570 [33:26<51:19, 2.85s/it]
31%|███▏ | 492/1570 [33:29<51:21, 2.86s/it]
31%|███▏ | 493/1570 [33:32<51:17, 2.86s/it]
31%|███▏ | 494/1570 [33:35<50:53, 2.84s/it]
32%|███▏ | 495/1570 [33:38<50:48, 2.84s/it]
32%|███▏ | 496/1570 [33:40<50:33, 2.82s/it]
32%|███▏ | 497/1570 [33:43<50:35, 2.83s/it]
32%|███▏ | 498/1570 [33:46<50:41, 2.84s/it]
32%|███▏ | 499/1570 [33:49<50:40, 2.84s/it]
32%|███▏ | 500/1570 [33:52<50:27, 2.83s/it]
{'loss': '0.008047', 'grad_norm': '0.07788', 'learning_rate': '7.71e-05', 'epoch': '1.594'}
+
32%|███▏ | 500/1570 [33:52<50:27, 2.83s/it]
32%|███▏ | 501/1570 [33:55<50:22, 2.83s/it]
32%|███▏ | 502/1570 [33:57<50:25, 2.83s/it]
32%|███▏ | 503/1570 [34:00<50:21, 2.83s/it]
32%|███▏ | 504/1570 [34:03<50:40, 2.85s/it]
32%|███▏ | 505/1570 [34:06<50:35, 2.85s/it]
32%|███▏ | 506/1570 [34:09<50:14, 2.83s/it]
32%|███▏ | 507/1570 [34:12<49:51, 2.81s/it]
32%|███▏ | 508/1570 [34:14<49:54, 2.82s/it]
32%|███▏ | 509/1570 [34:17<49:53, 2.82s/it]
32%|███▏ | 510/1570 [34:20<50:00, 2.83s/it]
33%|███▎ | 511/1570 [34:23<49:56, 2.83s/it]
33%|███▎ | 512/1570 [34:26<49:57, 2.83s/it]
33%|███▎ | 513/1570 [34:29<50:07, 2.84s/it]
33%|███▎ | 514/1570 [34:31<49:52, 2.83s/it]
33%|███▎ | 515/1570 [34:34<49:54, 2.84s/it]
33%|███▎ | 516/1570 [34:37<49:39, 2.83s/it]
33%|███▎ | 517/1570 [34:40<49:29, 2.82s/it]
33%|███▎ | 518/1570 [34:43<49:23, 2.82s/it]
33%|███▎ | 519/1570 [34:46<49:45, 2.84s/it]
33%|███▎ | 520/1570 [34:48<49:46, 2.84s/it]
{'loss': '0.006671', 'grad_norm': '0.05649', 'learning_rate': '7.54e-05', 'epoch': '1.657'}
+
33%|███▎ | 520/1570 [34:48<49:46, 2.84s/it]
33%|███▎ | 521/1570 [34:51<49:46, 2.85s/it]
33%|███▎ | 522/1570 [34:54<49:44, 2.85s/it]
33%|███▎ | 523/1570 [34:57<49:38, 2.84s/it]
33%|███▎ | 524/1570 [35:00<49:46, 2.85s/it]
33%|███▎ | 525/1570 [35:03<49:43, 2.86s/it]
34%|███▎ | 526/1570 [35:06<49:44, 2.86s/it]
34%|███▎ | 527/1570 [35:08<49:38, 2.86s/it]
34%|███▎ | 528/1570 [35:11<49:18, 2.84s/it]
34%|███▎ | 529/1570 [35:14<49:10, 2.83s/it]
34%|███▍ | 530/1570 [35:17<49:03, 2.83s/it]
34%|███▍ | 531/1570 [35:20<49:15, 2.84s/it]
34%|███▍ | 532/1570 [35:23<49:20, 2.85s/it]
34%|███▍ | 533/1570 [35:25<49:20, 2.85s/it]
34%|███▍ | 534/1570 [35:28<49:24, 2.86s/it]
34%|███▍ | 535/1570 [35:31<49:22, 2.86s/it]
34%|███▍ | 536/1570 [35:34<49:19, 2.86s/it]
34%|███▍ | 537/1570 [35:37<49:00, 2.85s/it]
34%|███▍ | 538/1570 [35:40<49:07, 2.86s/it]
34%|███▍ | 539/1570 [35:43<48:47, 2.84s/it]
34%|███▍ | 540/1570 [35:45<48:47, 2.84s/it]
{'loss': '0.007859', 'grad_norm': '0.1628', 'learning_rate': '7.366e-05', 'epoch': '1.721'}
+
34%|███▍ | 540/1570 [35:45<48:47, 2.84s/it]
34%|███▍ | 541/1570 [35:48<48:54, 2.85s/it]
35%|███▍ | 542/1570 [35:51<48:54, 2.85s/it]
35%|███▍ | 543/1570 [35:54<48:48, 2.85s/it]
35%|███▍ | 544/1570 [35:57<48:43, 2.85s/it]
35%|███▍ | 545/1570 [36:00<48:42, 2.85s/it]
35%|███▍ | 546/1570 [36:03<48:30, 2.84s/it]
35%|███▍ | 547/1570 [36:05<48:23, 2.84s/it]
35%|███▍ | 548/1570 [36:08<48:06, 2.82s/it]
35%|███▍ | 549/1570 [36:11<47:47, 2.81s/it]
35%|███▌ | 550/1570 [36:14<47:45, 2.81s/it]
35%|███▌ | 551/1570 [36:17<48:06, 2.83s/it]
35%|███▌ | 552/1570 [36:19<47:58, 2.83s/it]
35%|███▌ | 553/1570 [36:22<47:49, 2.82s/it]
35%|███▌ | 554/1570 [36:25<47:55, 2.83s/it]
35%|███▌ | 555/1570 [36:28<47:59, 2.84s/it]
35%|███▌ | 556/1570 [36:31<48:01, 2.84s/it]
35%|███▌ | 557/1570 [36:34<47:57, 2.84s/it]
36%|███▌ | 558/1570 [36:36<47:59, 2.84s/it]
36%|███▌ | 559/1570 [36:39<47:53, 2.84s/it]
36%|███▌ | 560/1570 [36:42<47:43, 2.84s/it]
{'loss': '0.001981', 'grad_norm': '0.01645', 'learning_rate': '7.188e-05', 'epoch': '1.785'}
+
36%|███▌ | 560/1570 [36:42<47:43, 2.84s/it]
36%|███▌ | 561/1570 [36:45<47:41, 2.84s/it]
36%|███▌ | 562/1570 [36:48<47:36, 2.83s/it]
36%|███▌ | 563/1570 [36:51<47:20, 2.82s/it]
36%|███▌ | 564/1570 [36:53<47:20, 2.82s/it]
36%|███▌ | 565/1570 [36:56<47:40, 2.85s/it]
36%|███▌ | 566/1570 [36:59<47:30, 2.84s/it]
36%|███▌ | 567/1570 [37:02<47:21, 2.83s/it]
36%|███▌ | 568/1570 [37:05<47:15, 2.83s/it]
36%|███▌ | 569/1570 [37:08<47:14, 2.83s/it]
36%|███▋ | 570/1570 [37:10<47:05, 2.83s/it]
36%|███▋ | 571/1570 [37:13<46:59, 2.82s/it]
36%|███▋ | 572/1570 [37:16<46:53, 2.82s/it]
36%|███▋ | 573/1570 [37:19<47:08, 2.84s/it]
37%|███▋ | 574/1570 [37:22<46:57, 2.83s/it]
37%|███▋ | 575/1570 [37:25<46:53, 2.83s/it]
37%|███▋ | 576/1570 [37:27<46:35, 2.81s/it]
37%|███▋ | 577/1570 [37:30<46:48, 2.83s/it]
37%|███▋ | 578/1570 [37:33<46:41, 2.82s/it]
37%|███▋ | 579/1570 [37:36<46:42, 2.83s/it]
37%|███▋ | 580/1570 [37:39<47:00, 2.85s/it]
{'loss': '0.009945', 'grad_norm': '0.4171', 'learning_rate': '7.006e-05', 'epoch': '1.849'}
+
37%|███▋ | 580/1570 [37:39<47:00, 2.85s/it]
37%|███▋ | 581/1570 [37:42<46:48, 2.84s/it]
37%|███▋ | 582/1570 [37:44<46:42, 2.84s/it]
37%|███▋ | 583/1570 [37:47<46:41, 2.84s/it]
37%|███▋ | 584/1570 [37:50<46:46, 2.85s/it]
37%|███▋ | 585/1570 [37:53<46:44, 2.85s/it]
37%|███▋ | 586/1570 [37:56<46:39, 2.84s/it]
37%|███▋ | 587/1570 [37:59<46:26, 2.83s/it]
37%|███▋ | 588/1570 [38:01<46:03, 2.81s/it]
38%|███▊ | 589/1570 [38:04<46:22, 2.84s/it]
38%|███▊ | 590/1570 [38:07<46:20, 2.84s/it]
38%|███▊ | 591/1570 [38:10<46:18, 2.84s/it]
38%|███▊ | 592/1570 [38:13<46:21, 2.84s/it]
38%|███▊ | 593/1570 [38:16<46:04, 2.83s/it]
38%|███▊ | 594/1570 [38:18<45:50, 2.82s/it]
38%|███▊ | 595/1570 [38:21<45:48, 2.82s/it]
38%|███▊ | 596/1570 [38:24<46:01, 2.84s/it]
38%|███▊ | 597/1570 [38:27<46:05, 2.84s/it]
38%|███▊ | 598/1570 [38:30<46:02, 2.84s/it]
38%|███▊ | 599/1570 [38:33<46:01, 2.84s/it]
38%|███▊ | 600/1570 [38:35<45:46, 2.83s/it]
{'loss': '0.008132', 'grad_norm': '0.2327', 'learning_rate': '6.821e-05', 'epoch': '1.913'}
+
38%|███▊ | 600/1570 [38:35<45:46, 2.83s/it]
38%|███▊ | 601/1570 [38:38<45:42, 2.83s/it]
38%|███▊ | 602/1570 [38:41<45:33, 2.82s/it]
38%|███▊ | 603/1570 [38:44<45:41, 2.83s/it]
38%|███▊ | 604/1570 [38:47<45:44, 2.84s/it]
39%|███▊ | 605/1570 [38:50<45:35, 2.83s/it]
39%|███▊ | 606/1570 [38:52<45:42, 2.84s/it]
39%|███▊ | 607/1570 [38:55<45:43, 2.85s/it]
39%|███▊ | 608/1570 [38:58<45:34, 2.84s/it]
39%|███▉ | 609/1570 [39:01<45:16, 2.83s/it]
39%|███▉ | 610/1570 [39:04<45:18, 2.83s/it]
39%|███▉ | 611/1570 [39:07<45:15, 2.83s/it]
39%|███▉ | 612/1570 [39:09<45:14, 2.83s/it]
39%|███▉ | 613/1570 [39:12<45:03, 2.82s/it]
39%|███▉ | 614/1570 [39:15<44:54, 2.82s/it]
39%|███▉ | 615/1570 [39:18<44:58, 2.83s/it]
39%|███▉ | 616/1570 [39:21<45:06, 2.84s/it]
39%|███▉ | 617/1570 [39:24<45:11, 2.85s/it]
39%|███▉ | 618/1570 [39:26<45:05, 2.84s/it]
39%|███▉ | 619/1570 [39:29<45:16, 2.86s/it]
39%|███▉ | 620/1570 [39:32<45:13, 2.86s/it]
{'loss': '0.002966', 'grad_norm': '0.04735', 'learning_rate': '6.634e-05', 'epoch': '1.976'}
+
39%|███▉ | 620/1570 [39:32<45:13, 2.86s/it]
40%|███▉ | 621/1570 [39:35<45:15, 2.86s/it]
40%|███▉ | 622/1570 [39:38<44:44, 2.83s/it]
40%|███▉ | 623/1570 [39:41<44:41, 2.83s/it]
40%|███▉ | 624/1570 [39:44<44:54, 2.85s/it]
40%|███▉ | 625/1570 [39:46<44:57, 2.85s/it]
40%|███▉ | 626/1570 [39:49<44:41, 2.84s/it]
40%|███▉ | 627/1570 [39:52<44:38, 2.84s/it]
40%|████ | 628/1570 [39:53<36:33, 2.33s/it][INFO|trainer.py:2668] 2026-08-22 15:38:52,171 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 15:38:52,171 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 15:38:52,171 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/sfkl/checkpoint-628
+[INFO|configuration_utils.py:780] 2026-08-22 15:48:49,246 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 15:48:49,247 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 15:48:49,282 >> chat template saved in results/qwen3/sfkl/checkpoint-628/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 15:48:49,282 >> tokenizer config file saved in results/qwen3/sfkl/checkpoint-628/tokenizer_config.json
+
40%|████ | 629/1570 [49:54<47:32:40, 181.89s/it]
40%|████ | 630/1570 [49:57<33:28:04, 128.17s/it]
40%|████ | 631/1570 [50:00<23:37:24, 90.57s/it]
40%|████ | 632/1570 [50:03<16:44:16, 64.24s/it]
40%|████ | 633/1570 [50:05<11:55:26, 45.81s/it]
40%|████ | 634/1570 [50:08<8:33:27, 32.91s/it]
40%|████ | 635/1570 [50:11<6:12:38, 23.91s/it]
41%|████ | 636/1570 [50:14<4:33:44, 17.58s/it]
41%|████ | 637/1570 [50:17<3:24:33, 13.15s/it]
41%|████ | 638/1570 [50:20<2:36:16, 10.06s/it]
41%|████ | 639/1570 [50:22<2:02:18, 7.88s/it]
41%|████ | 640/1570 [50:25<1:38:57, 6.38s/it]
{'loss': '0.002854', 'grad_norm': '0.1173', 'learning_rate': '6.444e-05', 'epoch': '2.038'}
+
41%|████ | 640/1570 [50:25<1:38:57, 6.38s/it]
41%|████ | 641/1570 [50:28<1:22:29, 5.33s/it]
41%|████ | 642/1570 [50:31<1:10:54, 4.58s/it]
41%|████ | 643/1570 [50:34<1:02:43, 4.06s/it]
41%|████ | 644/1570 [50:37<56:59, 3.69s/it]
41%|████ | 645/1570 [50:40<53:06, 3.44s/it]
41%|████ | 646/1570 [50:42<50:11, 3.26s/it]
41%|████ | 647/1570 [50:45<48:12, 3.13s/it]
41%|████▏ | 648/1570 [50:48<46:42, 3.04s/it]
41%|████▏ | 649/1570 [50:51<45:48, 2.98s/it]
41%|████▏ | 650/1570 [50:54<45:07, 2.94s/it]
41%|████▏ | 651/1570 [50:57<44:39, 2.92s/it]
42%|████▏ | 652/1570 [50:59<44:13, 2.89s/it]
42%|████▏ | 653/1570 [51:02<44:07, 2.89s/it]
42%|████▏ | 654/1570 [51:05<43:25, 2.84s/it]
42%|████▏ | 655/1570 [51:08<43:10, 2.83s/it]
42%|████▏ | 656/1570 [51:11<43:00, 2.82s/it]
42%|████▏ | 657/1570 [51:13<42:47, 2.81s/it]
42%|████▏ | 658/1570 [51:16<42:45, 2.81s/it]
42%|████▏ | 659/1570 [51:19<42:42, 2.81s/it]
42%|████▏ | 660/1570 [51:22<42:39, 2.81s/it]
{'loss': '0.002621', 'grad_norm': '0.2079', 'learning_rate': '6.251e-05', 'epoch': '2.102'}
+
42%|████▏ | 660/1570 [51:22<42:39, 2.81s/it]
42%|████▏ | 661/1570 [51:25<42:43, 2.82s/it]
42%|████▏ | 662/1570 [51:27<42:33, 2.81s/it]
42%|████▏ | 663/1570 [51:30<42:27, 2.81s/it]
42%|████▏ | 664/1570 [51:33<42:22, 2.81s/it]
42%|████▏ | 665/1570 [51:36<42:21, 2.81s/it]
42%|████▏ | 666/1570 [51:39<42:24, 2.81s/it]
42%|████▏ | 667/1570 [51:42<42:23, 2.82s/it]
43%|████▎ | 668/1570 [51:44<42:19, 2.82s/it]
43%|████▎ | 669/1570 [51:47<42:18, 2.82s/it]
43%|████▎ | 670/1570 [51:50<42:10, 2.81s/it]
43%|████▎ | 671/1570 [51:53<42:17, 2.82s/it]
43%|████▎ | 672/1570 [51:56<42:18, 2.83s/it]
43%|████▎ | 673/1570 [51:58<42:01, 2.81s/it]
43%|████▎ | 674/1570 [52:01<42:16, 2.83s/it]
43%|████▎ | 675/1570 [52:04<42:26, 2.85s/it]
43%|████▎ | 676/1570 [52:07<42:18, 2.84s/it]
43%|████▎ | 677/1570 [52:10<42:02, 2.82s/it]
43%|████▎ | 678/1570 [52:13<42:02, 2.83s/it]
43%|████▎ | 679/1570 [52:15<42:00, 2.83s/it]
43%|████▎ | 680/1570 [52:18<41:58, 2.83s/it]
{'loss': '0.001722', 'grad_norm': '0.1209', 'learning_rate': '6.057e-05', 'epoch': '2.166'}
+
43%|████▎ | 680/1570 [52:18<41:58, 2.83s/it]
43%|████▎ | 681/1570 [52:21<41:56, 2.83s/it]
43%|████▎ | 682/1570 [52:24<41:58, 2.84s/it]
44%|████▎ | 683/1570 [52:27<41:44, 2.82s/it]
44%|████▎ | 684/1570 [52:30<41:41, 2.82s/it]
44%|████▎ | 685/1570 [52:32<41:53, 2.84s/it]
44%|████▎ | 686/1570 [52:35<41:48, 2.84s/it]
44%|████▍ | 687/1570 [52:38<41:39, 2.83s/it]
44%|████▍ | 688/1570 [52:41<41:44, 2.84s/it]
44%|████▍ | 689/1570 [52:44<41:41, 2.84s/it]
44%|████▍ | 690/1570 [52:47<41:29, 2.83s/it]
44%|████▍ | 691/1570 [52:49<41:27, 2.83s/it]
44%|████▍ | 692/1570 [52:52<41:24, 2.83s/it]
44%|████▍ | 693/1570 [52:55<41:21, 2.83s/it]
44%|████▍ | 694/1570 [52:58<41:25, 2.84s/it]
44%|████▍ | 695/1570 [53:01<41:28, 2.84s/it]
44%|████▍ | 696/1570 [53:04<41:15, 2.83s/it]
44%|████▍ | 697/1570 [53:06<41:01, 2.82s/it]
44%|████▍ | 698/1570 [53:09<40:53, 2.81s/it]
45%|████▍ | 699/1570 [53:12<40:54, 2.82s/it]
45%|████▍ | 700/1570 [53:15<40:51, 2.82s/it]
{'loss': '0.001958', 'grad_norm': '0.01757', 'learning_rate': '5.86e-05', 'epoch': '2.23'}
+
45%|████▍ | 700/1570 [53:15<40:51, 2.82s/it]
45%|████▍ | 701/1570 [53:18<40:52, 2.82s/it]
45%|████▍ | 702/1570 [53:21<40:57, 2.83s/it]
45%|████▍ | 703/1570 [53:23<40:51, 2.83s/it]
45%|████▍ | 704/1570 [53:26<40:47, 2.83s/it]
45%|████▍ | 705/1570 [53:29<40:41, 2.82s/it]
45%|████▍ | 706/1570 [53:32<40:59, 2.85s/it]
45%|████▌ | 707/1570 [53:35<40:44, 2.83s/it]
45%|████▌ | 708/1570 [53:38<40:48, 2.84s/it]
45%|████▌ | 709/1570 [53:40<40:50, 2.85s/it]
45%|████▌ | 710/1570 [53:43<40:31, 2.83s/it]
45%|████▌ | 711/1570 [53:46<40:32, 2.83s/it]
45%|████▌ | 712/1570 [53:49<40:27, 2.83s/it]
45%|████▌ | 713/1570 [53:52<40:30, 2.84s/it]
45%|████▌ | 714/1570 [53:55<40:23, 2.83s/it]
46%|████▌ | 715/1570 [53:57<40:23, 2.83s/it]
46%|████▌ | 716/1570 [54:00<40:23, 2.84s/it]
46%|████▌ | 717/1570 [54:03<40:20, 2.84s/it]
46%|████▌ | 718/1570 [54:06<40:23, 2.84s/it]
46%|████▌ | 719/1570 [54:09<40:22, 2.85s/it]
46%|████▌ | 720/1570 [54:12<40:22, 2.85s/it]
{'loss': '0.005093', 'grad_norm': '0.1678', 'learning_rate': '5.663e-05', 'epoch': '2.294'}
+
46%|████▌ | 720/1570 [54:12<40:22, 2.85s/it]
46%|████▌ | 721/1570 [54:14<40:04, 2.83s/it]
46%|████▌ | 722/1570 [54:17<39:57, 2.83s/it]
46%|████▌ | 723/1570 [54:20<39:55, 2.83s/it]
46%|████▌ | 724/1570 [54:23<39:58, 2.84s/it]
46%|████▌ | 725/1570 [54:26<40:11, 2.85s/it]
46%|████▌ | 726/1570 [54:29<39:55, 2.84s/it]
46%|████▋ | 727/1570 [54:31<39:53, 2.84s/it]
46%|████▋ | 728/1570 [54:34<40:01, 2.85s/it]
46%|████▋ | 729/1570 [54:37<39:53, 2.85s/it]
46%|████▋ | 730/1570 [54:40<39:39, 2.83s/it]
47%|████▋ | 731/1570 [54:43<39:32, 2.83s/it]
47%|████▋ | 732/1570 [54:46<39:36, 2.84s/it]
47%|████▋ | 733/1570 [54:49<39:31, 2.83s/it]
47%|████▋ | 734/1570 [54:51<39:31, 2.84s/it]
47%|████▋ | 735/1570 [54:54<39:19, 2.83s/it]
47%|████▋ | 736/1570 [54:57<39:21, 2.83s/it]
47%|████▋ | 737/1570 [55:00<39:13, 2.83s/it]
47%|████▋ | 738/1570 [55:03<39:15, 2.83s/it]
47%|████▋ | 739/1570 [55:05<39:07, 2.82s/it]
47%|████▋ | 740/1570 [55:08<38:59, 2.82s/it]
{'loss': '0.003297', 'grad_norm': '0.02727', 'learning_rate': '5.464e-05', 'epoch': '2.357'}
+
47%|████▋ | 740/1570 [55:08<38:59, 2.82s/it]
47%|████▋ | 741/1570 [55:11<39:06, 2.83s/it]
47%|████▋ | 742/1570 [55:14<39:09, 2.84s/it]
47%|████▋ | 743/1570 [55:17<39:08, 2.84s/it]
47%|████▋ | 744/1570 [55:20<39:17, 2.85s/it]
47%|████▋ | 745/1570 [55:23<39:09, 2.85s/it]
48%|████▊ | 746/1570 [55:25<38:55, 2.83s/it]
48%|████▊ | 747/1570 [55:28<39:02, 2.85s/it]
48%|████▊ | 748/1570 [55:31<38:55, 2.84s/it]
48%|████▊ | 749/1570 [55:34<39:01, 2.85s/it]
48%|████▊ | 750/1570 [55:37<38:42, 2.83s/it]
48%|████▊ | 751/1570 [55:40<38:47, 2.84s/it]
48%|████▊ | 752/1570 [55:42<38:44, 2.84s/it]
48%|████▊ | 753/1570 [55:45<38:44, 2.85s/it]
48%|████▊ | 754/1570 [55:48<38:39, 2.84s/it]
48%|████▊ | 755/1570 [55:51<38:38, 2.84s/it]
48%|████▊ | 756/1570 [55:54<38:34, 2.84s/it]
48%|████▊ | 757/1570 [55:57<38:29, 2.84s/it]
48%|████▊ | 758/1570 [55:59<38:27, 2.84s/it]
48%|████▊ | 759/1570 [56:02<38:23, 2.84s/it]
48%|████▊ | 760/1570 [56:05<38:12, 2.83s/it]
{'loss': '0.001565', 'grad_norm': '0.03072', 'learning_rate': '5.265e-05', 'epoch': '2.421'}
+
48%|████▊ | 760/1570 [56:05<38:12, 2.83s/it]
48%|████▊ | 761/1570 [56:08<38:09, 2.83s/it]
49%|████▊ | 762/1570 [56:11<38:07, 2.83s/it]
49%|████▊ | 763/1570 [56:14<37:52, 2.82s/it]
49%|████▊ | 764/1570 [56:16<37:54, 2.82s/it]
49%|████▊ | 765/1570 [56:19<37:40, 2.81s/it]
49%|████▉ | 766/1570 [56:22<37:39, 2.81s/it]
49%|████▉ | 767/1570 [56:25<37:33, 2.81s/it]
49%|████▉ | 768/1570 [56:28<37:33, 2.81s/it]
49%|████▉ | 769/1570 [56:30<37:23, 2.80s/it]
49%|████▉ | 770/1570 [56:33<37:31, 2.81s/it]
49%|████▉ | 771/1570 [56:36<37:29, 2.82s/it]
49%|████▉ | 772/1570 [56:39<37:34, 2.82s/it]
49%|████▉ | 773/1570 [56:42<37:26, 2.82s/it]
49%|████▉ | 774/1570 [56:44<37:16, 2.81s/it]
49%|████▉ | 775/1570 [56:47<37:23, 2.82s/it]
49%|████▉ | 776/1570 [56:50<37:25, 2.83s/it]
49%|████▉ | 777/1570 [56:53<37:23, 2.83s/it]
50%|████▉ | 778/1570 [56:56<37:17, 2.83s/it]
50%|████▉ | 779/1570 [56:59<37:09, 2.82s/it]
50%|████▉ | 780/1570 [57:01<36:58, 2.81s/it]
{'loss': '0.002582', 'grad_norm': '0.02817', 'learning_rate': '5.065e-05', 'epoch': '2.485'}
+
50%|████▉ | 780/1570 [57:01<36:58, 2.81s/it]
50%|████▉ | 781/1570 [57:04<36:59, 2.81s/it]
50%|████▉ | 782/1570 [57:07<37:02, 2.82s/it]
50%|████▉ | 783/1570 [57:10<37:00, 2.82s/it]
50%|████▉ | 784/1570 [57:13<36:59, 2.82s/it]
50%|█████ | 785/1570 [57:16<36:56, 2.82s/it]
50%|█████ | 786/1570 [57:18<36:51, 2.82s/it]
50%|█████ | 787/1570 [57:21<36:47, 2.82s/it]
50%|█████ | 788/1570 [57:24<36:39, 2.81s/it]
50%|█████ | 789/1570 [57:27<36:40, 2.82s/it]
50%|█████ | 790/1570 [57:30<36:47, 2.83s/it]
50%|█████ | 791/1570 [57:33<36:46, 2.83s/it]
50%|█████ | 792/1570 [57:35<36:39, 2.83s/it]
51%|█████ | 793/1570 [57:38<36:32, 2.82s/it]
51%|█████ | 794/1570 [57:41<36:29, 2.82s/it]
51%|█████ | 795/1570 [57:44<36:25, 2.82s/it]
51%|█████ | 796/1570 [57:47<36:28, 2.83s/it]
51%|█████ | 797/1570 [57:49<36:22, 2.82s/it]
51%|█████ | 798/1570 [57:52<36:18, 2.82s/it]
51%|█████ | 799/1570 [57:55<36:22, 2.83s/it]
51%|█████ | 800/1570 [57:58<36:36, 2.85s/it]
{'loss': '0.005924', 'grad_norm': '0.1058', 'learning_rate': '4.865e-05', 'epoch': '2.549'}
+
51%|█████ | 800/1570 [57:58<36:36, 2.85s/it]
51%|█████ | 801/1570 [58:01<36:34, 2.85s/it]
51%|█████ | 802/1570 [58:04<36:26, 2.85s/it]
51%|█████ | 803/1570 [58:07<36:23, 2.85s/it]
51%|█████ | 804/1570 [58:09<36:12, 2.84s/it]
51%|█████▏ | 805/1570 [58:12<36:04, 2.83s/it]
51%|█████▏ | 806/1570 [58:15<36:01, 2.83s/it]
51%|█████▏ | 807/1570 [58:18<35:52, 2.82s/it]
51%|█████▏ | 808/1570 [58:21<35:44, 2.81s/it]
52%|█████▏ | 809/1570 [58:23<35:42, 2.81s/it]
52%|█████▏ | 810/1570 [58:26<35:41, 2.82s/it]
52%|█████▏ | 811/1570 [58:29<35:36, 2.81s/it]
52%|█████▏ | 812/1570 [58:32<35:38, 2.82s/it]
52%|█████▏ | 813/1570 [58:35<35:38, 2.82s/it]
52%|█████▏ | 814/1570 [58:38<35:34, 2.82s/it]
52%|█████▏ | 815/1570 [58:40<35:25, 2.81s/it]
52%|█████▏ | 816/1570 [58:43<35:37, 2.83s/it]
52%|█████▏ | 817/1570 [58:46<35:36, 2.84s/it]
52%|█████▏ | 818/1570 [58:49<35:36, 2.84s/it]
52%|█████▏ | 819/1570 [58:52<35:38, 2.85s/it]
52%|█████▏ | 820/1570 [58:55<35:35, 2.85s/it]
{'loss': '0.004035', 'grad_norm': '0.2381', 'learning_rate': '4.665e-05', 'epoch': '2.613'}
+
52%|█████▏ | 820/1570 [58:55<35:35, 2.85s/it]
52%|█████▏ | 821/1570 [58:57<35:37, 2.85s/it]
52%|█████▏ | 822/1570 [59:00<35:27, 2.84s/it]
52%|█████▏ | 823/1570 [59:03<35:25, 2.84s/it]
52%|█████▏ | 824/1570 [59:06<35:17, 2.84s/it]
53%|█████▎ | 825/1570 [59:09<35:08, 2.83s/it]
53%|█████▎ | 826/1570 [59:12<35:18, 2.85s/it]
53%|█████▎ | 827/1570 [59:15<35:11, 2.84s/it]
53%|█████▎ | 828/1570 [59:17<35:12, 2.85s/it]
53%|█████▎ | 829/1570 [59:20<35:02, 2.84s/it]
53%|█████▎ | 830/1570 [59:23<35:06, 2.85s/it]
53%|█████▎ | 831/1570 [59:26<35:00, 2.84s/it]
53%|█████▎ | 832/1570 [59:29<34:51, 2.83s/it]
53%|█████▎ | 833/1570 [59:32<34:59, 2.85s/it]
53%|█████▎ | 834/1570 [59:34<34:57, 2.85s/it]
53%|█████▎ | 835/1570 [59:37<34:55, 2.85s/it]
53%|█████▎ | 836/1570 [59:40<34:41, 2.84s/it]
53%|█████▎ | 837/1570 [59:43<34:37, 2.83s/it]
53%|█████▎ | 838/1570 [59:46<34:35, 2.84s/it]
53%|█████▎ | 839/1570 [59:49<34:27, 2.83s/it]
54%|█████▎ | 840/1570 [59:51<34:22, 2.82s/it]
{'loss': '0.002211', 'grad_norm': '0.05524', 'learning_rate': '4.466e-05', 'epoch': '2.677'}
+
54%|█████▎ | 840/1570 [59:51<34:22, 2.82s/it]
54%|█████▎ | 841/1570 [59:54<34:12, 2.82s/it]
54%|█████▎ | 842/1570 [59:57<34:18, 2.83s/it]
54%|█████▎ | 843/1570 [1:00:00<34:15, 2.83s/it]
54%|█████▍ | 844/1570 [1:00:03<34:22, 2.84s/it]
54%|█████▍ | 845/1570 [1:00:06<34:12, 2.83s/it]
54%|█████▍ | 846/1570 [1:00:08<34:18, 2.84s/it]
54%|█████▍ | 847/1570 [1:00:11<34:16, 2.84s/it]
54%|█████▍ | 848/1570 [1:00:14<34:08, 2.84s/it]
54%|█████▍ | 849/1570 [1:00:17<34:01, 2.83s/it]
54%|█████▍ | 850/1570 [1:00:20<34:05, 2.84s/it]
54%|█████▍ | 851/1570 [1:00:23<33:59, 2.84s/it]
54%|█████▍ | 852/1570 [1:00:25<33:58, 2.84s/it]
54%|█████▍ | 853/1570 [1:00:28<33:54, 2.84s/it]
54%|█████▍ | 854/1570 [1:00:31<34:01, 2.85s/it]
54%|█████▍ | 855/1570 [1:00:34<33:54, 2.85s/it]
55%|█████▍ | 856/1570 [1:00:37<33:45, 2.84s/it]
55%|█████▍ | 857/1570 [1:00:40<33:51, 2.85s/it]
55%|█████▍ | 858/1570 [1:00:43<33:45, 2.84s/it]
55%|█████▍ | 859/1570 [1:00:45<33:35, 2.83s/it]
55%|█████▍ | 860/1570 [1:00:48<33:39, 2.85s/it]
{'loss': '0.005412', 'grad_norm': '0.07276', 'learning_rate': '4.268e-05', 'epoch': '2.74'}
+
55%|█████▍ | 860/1570 [1:00:48<33:39, 2.85s/it]
55%|█████▍ | 861/1570 [1:00:51<33:26, 2.83s/it]
55%|█████▍ | 862/1570 [1:00:54<33:21, 2.83s/it]
55%|█████▍ | 863/1570 [1:00:57<33:17, 2.82s/it]
55%|█████▌ | 864/1570 [1:00:59<33:20, 2.83s/it]
55%|█████▌ | 865/1570 [1:01:02<33:15, 2.83s/it]
55%|█████▌ | 866/1570 [1:01:05<32:58, 2.81s/it]
55%|█████▌ | 867/1570 [1:01:08<32:48, 2.80s/it]
55%|█████▌ | 868/1570 [1:01:11<32:40, 2.79s/it]
55%|█████▌ | 869/1570 [1:01:13<32:47, 2.81s/it]
55%|█████▌ | 870/1570 [1:01:16<32:44, 2.81s/it]
55%|█████▌ | 871/1570 [1:01:19<32:49, 2.82s/it]
56%|█████▌ | 872/1570 [1:01:22<32:43, 2.81s/it]
56%|█████▌ | 873/1570 [1:01:25<32:37, 2.81s/it]
56%|█████▌ | 874/1570 [1:01:28<32:34, 2.81s/it]
56%|█████▌ | 875/1570 [1:01:30<32:35, 2.81s/it]
56%|█████▌ | 876/1570 [1:01:33<32:31, 2.81s/it]
56%|█████▌ | 877/1570 [1:01:36<32:33, 2.82s/it]
56%|█████▌ | 878/1570 [1:01:39<32:33, 2.82s/it]
56%|█████▌ | 879/1570 [1:01:42<32:37, 2.83s/it]
56%|█████▌ | 880/1570 [1:01:45<32:35, 2.83s/it]
{'loss': '0.001901', 'grad_norm': '0.03675', 'learning_rate': '4.071e-05', 'epoch': '2.804'}
+
56%|█████▌ | 880/1570 [1:01:45<32:35, 2.83s/it]
56%|█████▌ | 881/1570 [1:01:47<32:24, 2.82s/it]
56%|█████▌ | 882/1570 [1:01:50<32:18, 2.82s/it]
56%|█████▌ | 883/1570 [1:01:53<32:18, 2.82s/it]
56%|█████▋ | 884/1570 [1:01:56<32:14, 2.82s/it]
56%|█████▋ | 885/1570 [1:01:59<32:00, 2.80s/it]
56%|█████▋ | 886/1570 [1:02:01<32:09, 2.82s/it]
56%|█████▋ | 887/1570 [1:02:04<32:14, 2.83s/it]
57%|█████▋ | 888/1570 [1:02:07<32:17, 2.84s/it]
57%|█████▋ | 889/1570 [1:02:10<32:13, 2.84s/it]
57%|█████▋ | 890/1570 [1:02:13<31:59, 2.82s/it]
57%|█████▋ | 891/1570 [1:02:16<31:57, 2.82s/it]
57%|█████▋ | 892/1570 [1:02:18<31:57, 2.83s/it]
57%|█████▋ | 893/1570 [1:02:21<31:49, 2.82s/it]
57%|█████▋ | 894/1570 [1:02:24<31:58, 2.84s/it]
57%|█████▋ | 895/1570 [1:02:27<31:50, 2.83s/it]
57%|█████▋ | 896/1570 [1:02:30<31:45, 2.83s/it]
57%|█████▋ | 897/1570 [1:02:33<31:47, 2.83s/it]
57%|█████▋ | 898/1570 [1:02:35<31:54, 2.85s/it]
57%|█████▋ | 899/1570 [1:02:38<31:40, 2.83s/it]
57%|█████▋ | 900/1570 [1:02:41<31:39, 2.83s/it]
{'loss': '0.001375', 'grad_norm': '0.1189', 'learning_rate': '3.875e-05', 'epoch': '2.868'}
+
57%|█████▋ | 900/1570 [1:02:41<31:39, 2.83s/it]
57%|█████▋ | 901/1570 [1:02:44<31:34, 2.83s/it]
57%|█████▋ | 902/1570 [1:02:47<31:32, 2.83s/it]
58%|█████▊ | 903/1570 [1:02:50<31:27, 2.83s/it]
58%|█████▊ | 904/1570 [1:02:52<31:25, 2.83s/it]
58%|█████▊ | 905/1570 [1:02:55<31:27, 2.84s/it]
58%|█████▊ | 906/1570 [1:02:58<31:15, 2.82s/it]
58%|█████▊ | 907/1570 [1:03:01<31:12, 2.82s/it]
58%|█████▊ | 908/1570 [1:03:04<31:08, 2.82s/it]
58%|█████▊ | 909/1570 [1:03:06<31:04, 2.82s/it]
58%|█████▊ | 910/1570 [1:03:09<31:07, 2.83s/it]
58%|█████▊ | 911/1570 [1:03:12<31:05, 2.83s/it]
58%|█████▊ | 912/1570 [1:03:15<31:08, 2.84s/it]
58%|█████▊ | 913/1570 [1:03:18<30:56, 2.83s/it]
58%|█████▊ | 914/1570 [1:03:21<30:57, 2.83s/it]
58%|█████▊ | 915/1570 [1:03:24<30:56, 2.83s/it]
58%|█████▊ | 916/1570 [1:03:26<30:55, 2.84s/it]
58%|█████▊ | 917/1570 [1:03:29<30:40, 2.82s/it]
58%|█████▊ | 918/1570 [1:03:32<30:35, 2.82s/it]
59%|█████▊ | 919/1570 [1:03:35<30:30, 2.81s/it]
59%|█████▊ | 920/1570 [1:03:38<30:34, 2.82s/it]
{'loss': '0.003346', 'grad_norm': '0.004523', 'learning_rate': '3.682e-05', 'epoch': '2.932'}
+
59%|█████▊ | 920/1570 [1:03:38<30:34, 2.82s/it]
59%|█████▊ | 921/1570 [1:03:40<30:35, 2.83s/it]
59%|█████▊ | 922/1570 [1:03:43<30:35, 2.83s/it]
59%|█████▉ | 923/1570 [1:03:46<30:34, 2.84s/it]
59%|█████▉ | 924/1570 [1:03:49<30:28, 2.83s/it]
59%|█████▉ | 925/1570 [1:03:52<30:27, 2.83s/it]
59%|█████▉ | 926/1570 [1:03:55<30:19, 2.83s/it]
59%|█████▉ | 927/1570 [1:03:57<30:13, 2.82s/it]
59%|█████▉ | 928/1570 [1:04:00<30:06, 2.81s/it]
59%|█████▉ | 929/1570 [1:04:03<30:13, 2.83s/it]
59%|█████▉ | 930/1570 [1:04:06<30:21, 2.85s/it]
59%|█████▉ | 931/1570 [1:04:09<30:15, 2.84s/it]
59%|█████▉ | 932/1570 [1:04:12<30:16, 2.85s/it]
59%|█████▉ | 933/1570 [1:04:15<30:21, 2.86s/it]
59%|█████▉ | 934/1570 [1:04:17<30:24, 2.87s/it]
60%|█████▉ | 935/1570 [1:04:20<30:17, 2.86s/it]
60%|█████▉ | 936/1570 [1:04:23<30:14, 2.86s/it]
60%|█████▉ | 937/1570 [1:04:26<30:04, 2.85s/it]
60%|█████▉ | 938/1570 [1:04:29<29:57, 2.84s/it]
60%|█████▉ | 939/1570 [1:04:32<29:58, 2.85s/it]
60%|█████▉ | 940/1570 [1:04:34<29:54, 2.85s/it]
{'loss': '0.001428', 'grad_norm': '0.08859', 'learning_rate': '3.49e-05', 'epoch': '2.996'}
+
60%|█████▉ | 940/1570 [1:04:34<29:54, 2.85s/it]
60%|█████▉ | 941/1570 [1:04:37<29:49, 2.85s/it]
60%|██████ | 942/1570 [1:04:38<24:28, 2.34s/it][INFO|trainer.py:2668] 2026-08-22 16:03:37,420 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 16:03:37,421 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 16:03:37,421 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/sfkl/checkpoint-942
+[INFO|configuration_utils.py:780] 2026-08-22 16:13:33,622 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 16:13:33,624 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 16:13:33,659 >> chat template saved in results/qwen3/sfkl/checkpoint-942/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 16:13:33,659 >> tokenizer config file saved in results/qwen3/sfkl/checkpoint-942/tokenizer_config.json
+
60%|██████ | 943/1570 [1:14:39<31:38:17, 181.66s/it]
60%|██████ | 944/1570 [1:14:41<22:15:32, 128.01s/it]
60%|██████ | 945/1570 [1:14:44<15:42:22, 90.47s/it]
60%|██████ | 946/1570 [1:14:47<11:07:25, 64.18s/it]
60%|██████ | 947/1570 [1:14:50<7:55:16, 45.77s/it]
60%|██████ | 948/1570 [1:14:53<5:40:56, 32.89s/it]
60%|██████ | 949/1570 [1:14:56<4:07:05, 23.87s/it]
61%|██████ | 950/1570 [1:14:58<3:01:27, 17.56s/it]
61%|██████ | 951/1570 [1:15:01<2:15:37, 13.15s/it]
61%|██████ | 952/1570 [1:15:04<1:43:32, 10.05s/it]
61%|██████ | 953/1570 [1:15:07<1:21:12, 7.90s/it]
61%|██████ | 954/1570 [1:15:10<1:05:31, 6.38s/it]
61%|██████ | 955/1570 [1:15:13<54:35, 5.33s/it]
61%|██████ | 956/1570 [1:15:16<46:55, 4.58s/it]
61%|██████ | 957/1570 [1:15:18<41:29, 4.06s/it]
61%|██████ | 958/1570 [1:15:21<37:36, 3.69s/it]
61%|██████ | 959/1570 [1:15:24<34:55, 3.43s/it]
61%|██████ | 960/1570 [1:15:27<33:04, 3.25s/it]
{'loss': '0.003326', 'grad_norm': '0.03319', 'learning_rate': '3.301e-05', 'epoch': '3.057'}
+
61%|██████ | 960/1570 [1:15:27<33:04, 3.25s/it]
61%|██████ | 961/1570 [1:15:30<31:37, 3.12s/it]
61%|██████▏ | 962/1570 [1:15:32<30:41, 3.03s/it]
61%|██████▏ | 963/1570 [1:15:35<30:05, 2.98s/it]
61%|██████▏ | 964/1570 [1:15:38<29:46, 2.95s/it]
61%|██████▏ | 965/1570 [1:15:41<29:27, 2.92s/it]
62%|██████▏ | 966/1570 [1:15:44<29:12, 2.90s/it]
62%|██████▏ | 967/1570 [1:15:47<28:51, 2.87s/it]
62%|██████▏ | 968/1570 [1:15:50<28:49, 2.87s/it]
62%|██████▏ | 969/1570 [1:15:52<28:42, 2.87s/it]
62%|██████▏ | 970/1570 [1:15:55<28:42, 2.87s/it]
62%|██████▏ | 971/1570 [1:15:58<28:28, 2.85s/it]
62%|██████▏ | 972/1570 [1:16:01<28:16, 2.84s/it]
62%|██████▏ | 973/1570 [1:16:04<28:08, 2.83s/it]
62%|██████▏ | 974/1570 [1:16:07<28:08, 2.83s/it]
62%|██████▏ | 975/1570 [1:16:09<28:05, 2.83s/it]
62%|██████▏ | 976/1570 [1:16:12<27:55, 2.82s/it]
62%|██████▏ | 977/1570 [1:16:15<28:00, 2.83s/it]
62%|██████▏ | 978/1570 [1:16:18<28:00, 2.84s/it]
62%|██████▏ | 979/1570 [1:16:21<27:51, 2.83s/it]
62%|██████▏ | 980/1570 [1:16:24<27:55, 2.84s/it]
{'loss': '0.002738', 'grad_norm': '0.03868', 'learning_rate': '3.114e-05', 'epoch': '3.121'}
+
62%|██████▏ | 980/1570 [1:16:24<27:55, 2.84s/it]
62%|██████▏ | 981/1570 [1:16:26<27:50, 2.84s/it]
63%|██████▎ | 982/1570 [1:16:29<27:50, 2.84s/it]
63%|██████▎ | 983/1570 [1:16:32<27:55, 2.85s/it]
63%|██████▎ | 984/1570 [1:16:35<27:45, 2.84s/it]
63%|██████▎ | 985/1570 [1:16:38<27:38, 2.84s/it]
63%|██████▎ | 986/1570 [1:16:41<27:36, 2.84s/it]
63%|██████▎ | 987/1570 [1:16:43<27:32, 2.83s/it]
63%|██████▎ | 988/1570 [1:16:46<27:36, 2.85s/it]
63%|██████▎ | 989/1570 [1:16:49<27:37, 2.85s/it]
63%|██████▎ | 990/1570 [1:16:52<27:25, 2.84s/it]
63%|██████▎ | 991/1570 [1:16:55<27:30, 2.85s/it]
63%|██████▎ | 992/1570 [1:16:58<27:28, 2.85s/it]
63%|██████▎ | 993/1570 [1:17:01<27:22, 2.85s/it]
63%|██████▎ | 994/1570 [1:17:03<27:14, 2.84s/it]
63%|██████▎ | 995/1570 [1:17:06<27:09, 2.83s/it]
63%|██████▎ | 996/1570 [1:17:09<27:05, 2.83s/it]
64%|██████▎ | 997/1570 [1:17:12<26:56, 2.82s/it]
64%|██████▎ | 998/1570 [1:17:15<26:50, 2.82s/it]
64%|██████▎ | 999/1570 [1:17:18<26:56, 2.83s/it]
64%|██████▎ | 1000/1570 [1:17:20<26:54, 2.83s/it]
{'loss': '0.001316', 'grad_norm': '0.02454', 'learning_rate': '2.931e-05', 'epoch': '3.185'}
+
64%|██████▎ | 1000/1570 [1:17:20<26:54, 2.83s/it]
64%|██████▍ | 1001/1570 [1:17:23<26:48, 2.83s/it]
64%|██████▍ | 1002/1570 [1:17:26<26:40, 2.82s/it]
64%|██████▍ | 1003/1570 [1:17:29<26:52, 2.84s/it]
64%|██████▍ | 1004/1570 [1:17:32<26:44, 2.84s/it]
64%|██████▍ | 1005/1570 [1:17:35<26:43, 2.84s/it]
64%|██████▍ | 1006/1570 [1:17:37<26:44, 2.84s/it]
64%|██████▍ | 1007/1570 [1:17:40<26:43, 2.85s/it]
64%|██████▍ | 1008/1570 [1:17:43<26:35, 2.84s/it]
64%|██████▍ | 1009/1570 [1:17:46<26:28, 2.83s/it]
64%|██████▍ | 1010/1570 [1:17:49<26:19, 2.82s/it]
64%|██████▍ | 1011/1570 [1:17:52<26:23, 2.83s/it]
64%|██████▍ | 1012/1570 [1:17:54<26:23, 2.84s/it]
65%|██████▍ | 1013/1570 [1:17:57<26:20, 2.84s/it]
65%|██████▍ | 1014/1570 [1:18:00<26:21, 2.84s/it]
65%|██████▍ | 1015/1570 [1:18:03<26:16, 2.84s/it]
65%|██████▍ | 1016/1570 [1:18:06<26:16, 2.85s/it]
65%|██████▍ | 1017/1570 [1:18:09<26:15, 2.85s/it]
65%|██████▍ | 1018/1570 [1:18:11<26:04, 2.83s/it]
65%|██████▍ | 1019/1570 [1:18:14<25:56, 2.82s/it]
65%|██████▍ | 1020/1570 [1:18:17<25:57, 2.83s/it]
{'loss': '0.003594', 'grad_norm': '0.02027', 'learning_rate': '2.751e-05', 'epoch': '3.249'}
+
65%|██████▍ | 1020/1570 [1:18:17<25:57, 2.83s/it]
65%|██████▌ | 1021/1570 [1:18:20<26:01, 2.84s/it]
65%|██████▌ | 1022/1570 [1:18:23<25:55, 2.84s/it]
65%|██████▌ | 1023/1570 [1:18:26<25:53, 2.84s/it]
65%|██████▌ | 1024/1570 [1:18:28<25:51, 2.84s/it]
65%|██████▌ | 1025/1570 [1:18:31<25:47, 2.84s/it]
65%|██████▌ | 1026/1570 [1:18:34<25:44, 2.84s/it]
65%|██████▌ | 1027/1570 [1:18:37<25:44, 2.85s/it]
65%|██████▌ | 1028/1570 [1:18:40<25:36, 2.83s/it]
66%|██████▌ | 1029/1570 [1:18:43<25:33, 2.83s/it]
66%|██████▌ | 1030/1570 [1:18:45<25:25, 2.83s/it]
66%|██████▌ | 1031/1570 [1:18:48<25:20, 2.82s/it]
66%|██████▌ | 1032/1570 [1:18:51<25:25, 2.83s/it]
66%|██████▌ | 1033/1570 [1:18:54<25:25, 2.84s/it]
66%|██████▌ | 1034/1570 [1:18:57<25:27, 2.85s/it]
66%|██████▌ | 1035/1570 [1:19:00<25:25, 2.85s/it]
66%|██████▌ | 1036/1570 [1:19:03<25:23, 2.85s/it]
66%|██████▌ | 1037/1570 [1:19:05<25:32, 2.88s/it]
66%|██████▌ | 1038/1570 [1:19:08<25:31, 2.88s/it]
66%|██████▌ | 1039/1570 [1:19:11<25:32, 2.89s/it]
66%|██████▌ | 1040/1570 [1:19:14<25:23, 2.87s/it]
{'loss': '0.002231', 'grad_norm': '0.2636', 'learning_rate': '2.574e-05', 'epoch': '3.313'}
+
66%|██████▌ | 1040/1570 [1:19:14<25:23, 2.87s/it]
66%|██████▋ | 1041/1570 [1:19:17<25:08, 2.85s/it]
66%|██████▋ | 1042/1570 [1:19:20<25:02, 2.84s/it]
66%|██████▋ | 1043/1570 [1:19:23<25:01, 2.85s/it]
66%|██████▋ | 1044/1570 [1:19:25<25:01, 2.85s/it]
67%|██████▋ | 1045/1570 [1:19:28<24:49, 2.84s/it]
67%|██████▋ | 1046/1570 [1:19:31<24:40, 2.83s/it]
67%|██████▋ | 1047/1570 [1:19:34<24:41, 2.83s/it]
67%|██████▋ | 1048/1570 [1:19:37<24:37, 2.83s/it]
67%|██████▋ | 1049/1570 [1:19:40<24:43, 2.85s/it]
67%|██████▋ | 1050/1570 [1:19:42<24:43, 2.85s/it]
67%|██████▋ | 1051/1570 [1:19:45<24:36, 2.84s/it]
67%|██████▋ | 1052/1570 [1:19:48<24:37, 2.85s/it]
67%|██████▋ | 1053/1570 [1:19:51<24:38, 2.86s/it]
67%|██████▋ | 1054/1570 [1:19:54<24:32, 2.85s/it]
67%|██████▋ | 1055/1570 [1:19:57<24:20, 2.84s/it]
67%|██████▋ | 1056/1570 [1:20:00<24:20, 2.84s/it]
67%|██████▋ | 1057/1570 [1:20:02<24:08, 2.82s/it]
67%|██████▋ | 1058/1570 [1:20:05<24:12, 2.84s/it]
67%|██████▋ | 1059/1570 [1:20:08<24:11, 2.84s/it]
68%|██████▊ | 1060/1570 [1:20:11<24:10, 2.84s/it]
{'loss': '0.002583', 'grad_norm': '0.02567', 'learning_rate': '2.402e-05', 'epoch': '3.377'}
+
68%|██████▊ | 1060/1570 [1:20:11<24:10, 2.84s/it]
68%|██████▊ | 1061/1570 [1:20:14<24:15, 2.86s/it]
68%|██████▊ | 1062/1570 [1:20:17<24:14, 2.86s/it]
68%|██████▊ | 1063/1570 [1:20:19<24:03, 2.85s/it]
68%|██████▊ | 1064/1570 [1:20:22<24:02, 2.85s/it]
68%|██████▊ | 1065/1570 [1:20:25<24:01, 2.86s/it]
68%|██████▊ | 1066/1570 [1:20:28<23:52, 2.84s/it]
68%|██████▊ | 1067/1570 [1:20:31<23:48, 2.84s/it]
68%|██████▊ | 1068/1570 [1:20:34<23:49, 2.85s/it]
68%|██████▊ | 1069/1570 [1:20:37<23:55, 2.86s/it]
68%|██████▊ | 1070/1570 [1:20:39<23:48, 2.86s/it]
68%|██████▊ | 1071/1570 [1:20:42<23:50, 2.87s/it]
68%|██████▊ | 1072/1570 [1:20:45<23:49, 2.87s/it]
68%|██████▊ | 1073/1570 [1:20:48<23:44, 2.87s/it]
68%|██████▊ | 1074/1570 [1:20:51<23:43, 2.87s/it]
68%|██████▊ | 1075/1570 [1:20:54<23:38, 2.86s/it]
69%|██████▊ | 1076/1570 [1:20:57<23:29, 2.85s/it]
69%|██████▊ | 1077/1570 [1:20:59<23:21, 2.84s/it]
69%|██████▊ | 1078/1570 [1:21:02<23:24, 2.85s/it]
69%|██████▊ | 1079/1570 [1:21:05<23:07, 2.83s/it]
69%|██████▉ | 1080/1570 [1:21:08<23:01, 2.82s/it]
{'loss': '0.003804', 'grad_norm': '0.06274', 'learning_rate': '2.233e-05', 'epoch': '3.44'}
+
69%|██████▉ | 1080/1570 [1:21:08<23:01, 2.82s/it]
69%|██████▉ | 1081/1570 [1:21:11<22:58, 2.82s/it]
69%|██████▉ | 1082/1570 [1:21:14<22:54, 2.82s/it]
69%|██████▉ | 1083/1570 [1:21:16<22:49, 2.81s/it]
69%|██████▉ | 1084/1570 [1:21:19<22:53, 2.83s/it]
69%|██████▉ | 1085/1570 [1:21:22<22:47, 2.82s/it]
69%|██████▉ | 1086/1570 [1:21:25<22:40, 2.81s/it]
69%|██████▉ | 1087/1570 [1:21:28<22:39, 2.81s/it]
69%|██████▉ | 1088/1570 [1:21:30<22:35, 2.81s/it]
69%|██████▉ | 1089/1570 [1:21:33<22:34, 2.82s/it]
69%|██████▉ | 1090/1570 [1:21:36<22:28, 2.81s/it]
69%|██████▉ | 1091/1570 [1:21:39<22:22, 2.80s/it]
70%|██████▉ | 1092/1570 [1:21:42<22:22, 2.81s/it]
70%|██████▉ | 1093/1570 [1:21:44<22:19, 2.81s/it]
70%|██████▉ | 1094/1570 [1:21:47<22:19, 2.81s/it]
70%|██████▉ | 1095/1570 [1:21:50<22:11, 2.80s/it]
70%|██████▉ | 1096/1570 [1:21:53<22:12, 2.81s/it]
70%|██████▉ | 1097/1570 [1:21:56<22:11, 2.82s/it]
70%|██████▉ | 1098/1570 [1:21:59<22:13, 2.82s/it]
70%|███████ | 1099/1570 [1:22:01<22:20, 2.85s/it]
70%|███████ | 1100/1570 [1:22:04<22:16, 2.84s/it]
{'loss': '0.001325', 'grad_norm': '0.005196', 'learning_rate': '2.069e-05', 'epoch': '3.504'}
+
70%|███████ | 1100/1570 [1:22:04<22:16, 2.84s/it]
70%|███████ | 1101/1570 [1:22:07<22:18, 2.85s/it]
70%|███████ | 1102/1570 [1:22:10<22:15, 2.85s/it]
70%|███████ | 1103/1570 [1:22:13<22:07, 2.84s/it]
70%|███████ | 1104/1570 [1:22:16<21:58, 2.83s/it]
70%|███████ | 1105/1570 [1:22:18<21:55, 2.83s/it]
70%|███████ | 1106/1570 [1:22:21<21:54, 2.83s/it]
71%|███████ | 1107/1570 [1:22:24<21:56, 2.84s/it]
71%|███████ | 1108/1570 [1:22:27<21:47, 2.83s/it]
71%|███████ | 1109/1570 [1:22:30<21:45, 2.83s/it]
71%|███████ | 1110/1570 [1:22:33<21:39, 2.83s/it]
71%|███████ | 1111/1570 [1:22:35<21:33, 2.82s/it]
71%|███████ | 1112/1570 [1:22:38<21:30, 2.82s/it]
71%|███████ | 1113/1570 [1:22:41<21:33, 2.83s/it]
71%|███████ | 1114/1570 [1:22:44<21:25, 2.82s/it]
71%|███████ | 1115/1570 [1:22:47<21:26, 2.83s/it]
71%|███████ | 1116/1570 [1:22:50<21:21, 2.82s/it]
71%|███████ | 1117/1570 [1:22:52<21:17, 2.82s/it]
71%|███████ | 1118/1570 [1:22:55<21:12, 2.81s/it]
71%|███████▏ | 1119/1570 [1:22:58<21:16, 2.83s/it]
71%|███████▏ | 1120/1570 [1:23:01<21:07, 2.82s/it]
{'loss': '0.00113', 'grad_norm': '0.08129', 'learning_rate': '1.91e-05', 'epoch': '3.568'}
+
71%|███████▏ | 1120/1570 [1:23:01<21:07, 2.82s/it]
71%|███████▏ | 1121/1570 [1:23:04<21:09, 2.83s/it]
71%|███████▏ | 1122/1570 [1:23:07<21:13, 2.84s/it]
72%|███████▏ | 1123/1570 [1:23:09<21:09, 2.84s/it]
72%|███████▏ | 1124/1570 [1:23:12<20:57, 2.82s/it]
72%|███████▏ | 1125/1570 [1:23:15<20:52, 2.82s/it]
72%|███████▏ | 1126/1570 [1:23:18<20:53, 2.82s/it]
72%|███████▏ | 1127/1570 [1:23:21<20:46, 2.81s/it]
72%|███████▏ | 1128/1570 [1:23:23<20:47, 2.82s/it]
72%|███████▏ | 1129/1570 [1:23:26<20:46, 2.83s/it]
72%|███████▏ | 1130/1570 [1:23:29<20:48, 2.84s/it]
72%|███████▏ | 1131/1570 [1:23:32<20:46, 2.84s/it]
72%|███████▏ | 1132/1570 [1:23:35<20:39, 2.83s/it]
72%|███████▏ | 1133/1570 [1:23:38<20:36, 2.83s/it]
72%|███████▏ | 1134/1570 [1:23:40<20:32, 2.83s/it]
72%|███████▏ | 1135/1570 [1:23:43<20:34, 2.84s/it]
72%|███████▏ | 1136/1570 [1:23:46<20:31, 2.84s/it]
72%|███████▏ | 1137/1570 [1:23:49<20:22, 2.82s/it]
72%|███████▏ | 1138/1570 [1:23:52<20:23, 2.83s/it]
73%|███████▎ | 1139/1570 [1:23:55<20:14, 2.82s/it]
73%|███████▎ | 1140/1570 [1:23:57<20:20, 2.84s/it]
{'loss': '0.0009622', 'grad_norm': '0.08928', 'learning_rate': '1.755e-05', 'epoch': '3.632'}
+
73%|███████▎ | 1140/1570 [1:23:57<20:20, 2.84s/it]
73%|███████▎ | 1141/1570 [1:24:00<20:22, 2.85s/it]
73%|███████▎ | 1142/1570 [1:24:03<20:14, 2.84s/it]
73%|███████▎ | 1143/1570 [1:24:06<20:10, 2.83s/it]
73%|███████▎ | 1144/1570 [1:24:09<20:09, 2.84s/it]
73%|███████▎ | 1145/1570 [1:24:12<20:07, 2.84s/it]
73%|███████▎ | 1146/1570 [1:24:15<20:04, 2.84s/it]
73%|███████▎ | 1147/1570 [1:24:17<20:01, 2.84s/it]
73%|███████▎ | 1148/1570 [1:24:20<20:00, 2.84s/it]
73%|███████▎ | 1149/1570 [1:24:23<19:53, 2.84s/it]
73%|███████▎ | 1150/1570 [1:24:26<19:48, 2.83s/it]
73%|███████▎ | 1151/1570 [1:24:29<19:43, 2.82s/it]
73%|███████▎ | 1152/1570 [1:24:31<19:36, 2.81s/it]
73%|███████▎ | 1153/1570 [1:24:34<19:32, 2.81s/it]
74%|███████▎ | 1154/1570 [1:24:37<19:29, 2.81s/it]
74%|███████▎ | 1155/1570 [1:24:40<19:30, 2.82s/it]
74%|███████▎ | 1156/1570 [1:24:43<19:22, 2.81s/it]
74%|███████▎ | 1157/1570 [1:24:45<19:17, 2.80s/it]
74%|███████▍ | 1158/1570 [1:24:48<19:18, 2.81s/it]
74%|███████▍ | 1159/1570 [1:24:51<19:19, 2.82s/it]
74%|███████▍ | 1160/1570 [1:24:54<19:16, 2.82s/it]
{'loss': '0.004592', 'grad_norm': '0.5374', 'learning_rate': '1.606e-05', 'epoch': '3.696'}
+
74%|███████▍ | 1160/1570 [1:24:54<19:16, 2.82s/it]
74%|███████▍ | 1161/1570 [1:24:57<19:12, 2.82s/it]
74%|███████▍ | 1162/1570 [1:25:00<19:14, 2.83s/it]
74%|███████▍ | 1163/1570 [1:25:02<19:10, 2.83s/it]
74%|███████▍ | 1164/1570 [1:25:05<19:06, 2.82s/it]
74%|███████▍ | 1165/1570 [1:25:08<19:06, 2.83s/it]
74%|███████▍ | 1166/1570 [1:25:11<19:04, 2.83s/it]
74%|███████▍ | 1167/1570 [1:25:14<18:54, 2.82s/it]
74%|███████▍ | 1168/1570 [1:25:17<18:50, 2.81s/it]
74%|███████▍ | 1169/1570 [1:25:19<18:46, 2.81s/it]
75%|███████▍ | 1170/1570 [1:25:22<18:43, 2.81s/it]
75%|███████▍ | 1171/1570 [1:25:25<18:42, 2.81s/it]
75%|███████▍ | 1172/1570 [1:25:28<18:40, 2.81s/it]
75%|███████▍ | 1173/1570 [1:25:31<18:35, 2.81s/it]
75%|███████▍ | 1174/1570 [1:25:33<18:37, 2.82s/it]
75%|███████▍ | 1175/1570 [1:25:36<18:35, 2.82s/it]
75%|███████▍ | 1176/1570 [1:25:39<18:39, 2.84s/it]
75%|███████▍ | 1177/1570 [1:25:42<18:36, 2.84s/it]
75%|███████▌ | 1178/1570 [1:25:45<18:26, 2.82s/it]
75%|███████▌ | 1179/1570 [1:25:48<18:22, 2.82s/it]
75%|███████▌ | 1180/1570 [1:25:50<18:18, 2.82s/it]
{'loss': '0.002302', 'grad_norm': '0.03226', 'learning_rate': '1.462e-05', 'epoch': '3.759'}
+
75%|███████▌ | 1180/1570 [1:25:50<18:18, 2.82s/it]
75%|███████▌ | 1181/1570 [1:25:53<18:22, 2.83s/it]
75%|███████▌ | 1182/1570 [1:25:56<18:15, 2.82s/it]
75%|███████▌ | 1183/1570 [1:25:59<18:12, 2.82s/it]
75%|███████▌ | 1184/1570 [1:26:02<18:05, 2.81s/it]
75%|███████▌ | 1185/1570 [1:26:04<18:02, 2.81s/it]
76%|███████▌ | 1186/1570 [1:26:07<18:01, 2.82s/it]
76%|███████▌ | 1187/1570 [1:26:10<17:56, 2.81s/it]
76%|███████▌ | 1188/1570 [1:26:13<17:56, 2.82s/it]
76%|███████▌ | 1189/1570 [1:26:16<17:57, 2.83s/it]
76%|███████▌ | 1190/1570 [1:26:19<17:55, 2.83s/it]
76%|███████▌ | 1191/1570 [1:26:21<17:51, 2.83s/it]
76%|███████▌ | 1192/1570 [1:26:24<17:50, 2.83s/it]
76%|███████▌ | 1193/1570 [1:26:27<17:51, 2.84s/it]
76%|███████▌ | 1194/1570 [1:26:30<17:49, 2.84s/it]
76%|███████▌ | 1195/1570 [1:26:33<17:39, 2.83s/it]
76%|███████▌ | 1196/1570 [1:26:36<17:36, 2.83s/it]
76%|███████▌ | 1197/1570 [1:26:38<17:34, 2.83s/it]
76%|███████▋ | 1198/1570 [1:26:41<17:31, 2.83s/it]
76%|███████▋ | 1199/1570 [1:26:44<17:31, 2.83s/it]
76%|███████▋ | 1200/1570 [1:26:47<17:26, 2.83s/it]
{'loss': '0.002388', 'grad_norm': '0.005963', 'learning_rate': '1.324e-05', 'epoch': '3.823'}
+
76%|███████▋ | 1200/1570 [1:26:47<17:26, 2.83s/it]
76%|███████▋ | 1201/1570 [1:26:50<17:26, 2.84s/it]
77%|███████▋ | 1202/1570 [1:26:53<17:24, 2.84s/it]
77%|███████▋ | 1203/1570 [1:26:55<17:17, 2.83s/it]
77%|███████▋ | 1204/1570 [1:26:58<17:17, 2.83s/it]
77%|███████▋ | 1205/1570 [1:27:01<17:08, 2.82s/it]
77%|███████▋ | 1206/1570 [1:27:04<17:04, 2.81s/it]
77%|███████▋ | 1207/1570 [1:27:07<17:01, 2.81s/it]
77%|███████▋ | 1208/1570 [1:27:10<17:00, 2.82s/it]
77%|███████▋ | 1209/1570 [1:27:12<17:02, 2.83s/it]
77%|███████▋ | 1210/1570 [1:27:15<16:59, 2.83s/it]
77%|███████▋ | 1211/1570 [1:27:18<16:56, 2.83s/it]
77%|███████▋ | 1212/1570 [1:27:21<16:55, 2.84s/it]
77%|███████▋ | 1213/1570 [1:27:24<16:49, 2.83s/it]
77%|███████▋ | 1214/1570 [1:27:27<16:50, 2.84s/it]
77%|███████▋ | 1215/1570 [1:27:29<16:48, 2.84s/it]
77%|███████▋ | 1216/1570 [1:27:32<16:47, 2.85s/it]
78%|███████▊ | 1217/1570 [1:27:35<16:39, 2.83s/it]
78%|███████▊ | 1218/1570 [1:27:38<16:36, 2.83s/it]
78%|███████▊ | 1219/1570 [1:27:41<16:33, 2.83s/it]
78%|███████▊ | 1220/1570 [1:27:44<16:28, 2.82s/it]
{'loss': '0.002433', 'grad_norm': '0.07597', 'learning_rate': '1.192e-05', 'epoch': '3.887'}
+
78%|███████▊ | 1220/1570 [1:27:44<16:28, 2.82s/it]
78%|███████▊ | 1221/1570 [1:27:46<16:25, 2.82s/it]
78%|███████▊ | 1222/1570 [1:27:49<16:23, 2.83s/it]
78%|███████▊ | 1223/1570 [1:27:52<16:18, 2.82s/it]
78%|███████▊ | 1224/1570 [1:27:55<16:18, 2.83s/it]
78%|███████▊ | 1225/1570 [1:27:58<16:20, 2.84s/it]
78%|███████▊ | 1226/1570 [1:28:01<16:16, 2.84s/it]
78%|███████▊ | 1227/1570 [1:28:03<16:19, 2.86s/it]
78%|███████▊ | 1228/1570 [1:28:06<16:18, 2.86s/it]
78%|███████▊ | 1229/1570 [1:28:09<16:14, 2.86s/it]
78%|███████▊ | 1230/1570 [1:28:12<16:09, 2.85s/it]
78%|███████▊ | 1231/1570 [1:28:15<16:03, 2.84s/it]
78%|███████▊ | 1232/1570 [1:28:18<16:01, 2.84s/it]
79%|███████▊ | 1233/1570 [1:28:20<15:55, 2.84s/it]
79%|███████▊ | 1234/1570 [1:28:23<15:53, 2.84s/it]
79%|███████▊ | 1235/1570 [1:28:26<15:50, 2.84s/it]
79%|███████▊ | 1236/1570 [1:28:29<15:47, 2.84s/it]
79%|███████▉ | 1237/1570 [1:28:32<15:39, 2.82s/it]
79%|███████▉ | 1238/1570 [1:28:35<15:36, 2.82s/it]
79%|███████▉ | 1239/1570 [1:28:37<15:35, 2.83s/it]
79%|███████▉ | 1240/1570 [1:28:40<15:34, 2.83s/it]
{'loss': '0.0009138', 'grad_norm': '0.02545', 'learning_rate': '1.066e-05', 'epoch': '3.951'}
+
79%|███████▉ | 1240/1570 [1:28:40<15:34, 2.83s/it]
79%|███████▉ | 1241/1570 [1:28:43<15:35, 2.84s/it]
79%|███████▉ | 1242/1570 [1:28:46<15:28, 2.83s/it]
79%|███████▉ | 1243/1570 [1:28:49<15:29, 2.84s/it]
79%|███████▉ | 1244/1570 [1:28:52<15:24, 2.84s/it]
79%|███████▉ | 1245/1570 [1:28:54<15:23, 2.84s/it]
79%|███████▉ | 1246/1570 [1:28:57<15:25, 2.86s/it]
79%|███████▉ | 1247/1570 [1:29:00<15:24, 2.86s/it]
79%|███████▉ | 1248/1570 [1:29:03<15:22, 2.87s/it]
80%|███████▉ | 1249/1570 [1:29:06<15:17, 2.86s/it]
80%|███████▉ | 1250/1570 [1:29:09<15:17, 2.87s/it]
80%|███████▉ | 1251/1570 [1:29:12<15:08, 2.85s/it]
80%|███████▉ | 1252/1570 [1:29:14<15:03, 2.84s/it]
80%|███████▉ | 1253/1570 [1:29:17<14:57, 2.83s/it]
80%|███████▉ | 1254/1570 [1:29:20<14:54, 2.83s/it]
80%|███████▉ | 1255/1570 [1:29:23<14:55, 2.84s/it]
80%|████████ | 1256/1570 [1:29:24<12:10, 2.33s/it][INFO|trainer.py:2668] 2026-08-22 16:28:23,070 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 16:28:23,070 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 16:28:23,070 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/sfkl/checkpoint-1256
+[INFO|configuration_utils.py:780] 2026-08-22 16:38:29,153 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 16:38:29,155 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 16:38:29,189 >> chat template saved in results/qwen3/sfkl/checkpoint-1256/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 16:38:29,190 >> tokenizer config file saved in results/qwen3/sfkl/checkpoint-1256/tokenizer_config.json
+
80%|████████ | 1257/1570 [1:39:35<16:04:50, 184.95s/it]
80%|████████ | 1258/1570 [1:39:39<11:19:27, 130.66s/it]
80%|████████ | 1259/1570 [1:39:43<8:00:06, 92.62s/it]
80%|████████ | 1260/1570 [1:39:47<5:41:12, 66.04s/it]
{'loss': '0.002068', 'grad_norm': '0.3483', 'learning_rate': '9.464e-06', 'epoch': '4.013'}
+
80%|████████ | 1260/1570 [1:39:47<5:41:12, 66.04s/it]
80%|████████ | 1261/1570 [1:39:51<4:03:39, 47.31s/it]
80%|████████ | 1262/1570 [1:39:54<2:55:30, 34.19s/it]
80%|████████ | 1263/1570 [1:39:58<2:08:05, 25.03s/it]
81%|████████ | 1264/1570 [1:40:01<1:34:39, 18.56s/it]
81%|████████ | 1265/1570 [1:40:05<1:11:03, 13.98s/it]
81%|████████ | 1266/1570 [1:40:08<54:47, 10.81s/it]
81%|████████ | 1267/1570 [1:40:12<43:24, 8.59s/it]
81%|████████ | 1268/1570 [1:40:15<35:23, 7.03s/it]
81%|████████ | 1269/1570 [1:40:18<29:53, 5.96s/it]
81%|████████ | 1270/1570 [1:40:22<26:02, 5.21s/it]
81%|████████ | 1271/1570 [1:40:25<23:32, 4.72s/it]
81%|████████ | 1272/1570 [1:40:28<20:43, 4.17s/it]
81%|████████ | 1273/1570 [1:40:31<18:45, 3.79s/it]
81%|████████ | 1274/1570 [1:40:34<17:20, 3.51s/it]
81%|████████ | 1275/1570 [1:40:37<16:24, 3.34s/it]
81%|████████▏ | 1276/1570 [1:40:40<15:53, 3.24s/it]
81%|████████▏ | 1277/1570 [1:40:43<15:36, 3.20s/it]
81%|████████▏ | 1278/1570 [1:40:46<15:03, 3.09s/it]
81%|████████▏ | 1279/1570 [1:40:49<14:56, 3.08s/it]
82%|████████▏ | 1280/1570 [1:40:52<14:55, 3.09s/it]
{'loss': '0.0007737', 'grad_norm': '0.1926', 'learning_rate': '8.332e-06', 'epoch': '4.077'}
+
82%|████████▏ | 1280/1570 [1:40:52<14:55, 3.09s/it]
82%|████████▏ | 1281/1570 [1:40:55<15:18, 3.18s/it]
82%|████████▏ | 1282/1570 [1:40:59<15:48, 3.29s/it]
82%|████████▏ | 1283/1570 [1:41:03<16:25, 3.43s/it]
82%|████████▏ | 1284/1570 [1:41:07<16:57, 3.56s/it]
82%|████████▏ | 1285/1570 [1:41:11<17:22, 3.66s/it]
82%|████████▏ | 1286/1570 [1:41:14<17:36, 3.72s/it]
82%|████████▏ | 1287/1570 [1:41:18<18:01, 3.82s/it]
82%|████████▏ | 1288/1570 [1:41:22<17:47, 3.79s/it]
82%|████████▏ | 1289/1570 [1:41:26<18:02, 3.85s/it]
82%|████████▏ | 1290/1570 [1:41:30<18:10, 3.90s/it]
82%|████████▏ | 1291/1570 [1:41:34<18:10, 3.91s/it]
82%|████████▏ | 1292/1570 [1:41:38<18:03, 3.90s/it]
82%|████████▏ | 1293/1570 [1:41:42<18:12, 3.94s/it]
82%|████████▏ | 1294/1570 [1:41:46<17:57, 3.91s/it]
82%|████████▏ | 1295/1570 [1:41:50<17:59, 3.92s/it]
83%|████████▎ | 1296/1570 [1:41:54<17:58, 3.94s/it]
83%|████████▎ | 1297/1570 [1:41:58<17:59, 3.95s/it]
83%|████████▎ | 1298/1570 [1:42:02<17:54, 3.95s/it]
83%|████████▎ | 1299/1570 [1:42:06<17:54, 3.96s/it]
83%|████████▎ | 1300/1570 [1:42:10<17:38, 3.92s/it]
{'loss': '0.001195', 'grad_norm': '0.03017', 'learning_rate': '7.266e-06', 'epoch': '4.14'}
+
83%|████████▎ | 1300/1570 [1:42:10<17:38, 3.92s/it]
83%|████████▎ | 1301/1570 [1:42:14<17:45, 3.96s/it]
83%|████████▎ | 1302/1570 [1:42:17<17:36, 3.94s/it]
83%|████████▎ | 1303/1570 [1:42:22<17:42, 3.98s/it]
83%|████████▎ | 1304/1570 [1:42:25<17:25, 3.93s/it]
83%|████████▎ | 1305/1570 [1:42:29<17:15, 3.91s/it]
83%|████████▎ | 1306/1570 [1:42:33<16:57, 3.86s/it]
83%|████████▎ | 1307/1570 [1:42:37<16:53, 3.85s/it]
83%|████████▎ | 1308/1570 [1:42:41<16:46, 3.84s/it]
83%|████████▎ | 1309/1570 [1:42:45<16:54, 3.89s/it]
83%|████████▎ | 1310/1570 [1:42:48<16:38, 3.84s/it]
84%|████████▎ | 1311/1570 [1:42:52<16:43, 3.88s/it]
84%|████████▎ | 1312/1570 [1:42:56<16:48, 3.91s/it]
84%|████████▎ | 1313/1570 [1:43:00<16:36, 3.88s/it]
84%|████████▎ | 1314/1570 [1:43:04<16:30, 3.87s/it]
84%|████████▍ | 1315/1570 [1:43:08<16:37, 3.91s/it]
84%|████████▍ | 1316/1570 [1:43:12<16:35, 3.92s/it]
84%|████████▍ | 1317/1570 [1:43:16<16:25, 3.89s/it]
84%|████████▍ | 1318/1570 [1:43:20<16:31, 3.93s/it]
84%|████████▍ | 1319/1570 [1:43:24<16:14, 3.88s/it]
84%|████████▍ | 1320/1570 [1:43:27<15:44, 3.78s/it]
{'loss': '0.001191', 'grad_norm': '0.007226', 'learning_rate': '6.269e-06', 'epoch': '4.204'}
+
84%|████████▍ | 1320/1570 [1:43:27<15:44, 3.78s/it]
84%|████████▍ | 1321/1570 [1:43:30<14:28, 3.49s/it]
84%|████████▍ | 1322/1570 [1:43:33<13:39, 3.30s/it]
84%|████████▍ | 1323/1570 [1:43:36<13:02, 3.17s/it]
84%|████████▍ | 1324/1570 [1:43:38<12:38, 3.08s/it]
84%|████████▍ | 1325/1570 [1:43:41<12:21, 3.03s/it]
84%|████████▍ | 1326/1570 [1:43:44<12:09, 2.99s/it]
85%|████████▍ | 1327/1570 [1:43:47<11:56, 2.95s/it]
85%|████████▍ | 1328/1570 [1:43:50<11:47, 2.92s/it]
85%|████████▍ | 1329/1570 [1:43:53<11:45, 2.93s/it]
85%|████████▍ | 1330/1570 [1:43:56<11:47, 2.95s/it]
85%|████████▍ | 1331/1570 [1:43:59<11:36, 2.92s/it]
85%|████████▍ | 1332/1570 [1:44:02<11:33, 2.92s/it]
85%|████████▍ | 1333/1570 [1:44:05<11:30, 2.91s/it]
85%|████████▍ | 1334/1570 [1:44:07<11:25, 2.90s/it]
85%|████████▌ | 1335/1570 [1:44:10<11:19, 2.89s/it]
85%|████████▌ | 1336/1570 [1:44:13<11:13, 2.88s/it]
85%|████████▌ | 1337/1570 [1:44:16<11:11, 2.88s/it]
85%|████████▌ | 1338/1570 [1:44:19<11:06, 2.87s/it]
85%|████████▌ | 1339/1570 [1:44:22<11:03, 2.87s/it]
85%|████████▌ | 1340/1570 [1:44:25<11:00, 2.87s/it]
{'loss': '0.001154', 'grad_norm': '0.01284', 'learning_rate': '5.342e-06', 'epoch': '4.268'}
+
85%|████████▌ | 1340/1570 [1:44:25<11:00, 2.87s/it]
85%|████████▌ | 1341/1570 [1:44:28<10:57, 2.87s/it]
85%|████████▌ | 1342/1570 [1:44:30<10:55, 2.87s/it]
86%|████████▌ | 1343/1570 [1:44:33<10:53, 2.88s/it]
86%|████████▌ | 1344/1570 [1:44:36<10:49, 2.87s/it]
86%|████████▌ | 1345/1570 [1:44:39<10:47, 2.88s/it]
86%|████████▌ | 1346/1570 [1:44:42<10:45, 2.88s/it]
86%|████████▌ | 1347/1570 [1:44:45<10:41, 2.88s/it]
86%|████████▌ | 1348/1570 [1:44:48<10:35, 2.86s/it]
86%|████████▌ | 1349/1570 [1:44:50<10:31, 2.86s/it]
86%|████████▌ | 1350/1570 [1:44:53<10:28, 2.86s/it]
86%|████████▌ | 1351/1570 [1:44:56<10:27, 2.87s/it]
86%|████████▌ | 1352/1570 [1:44:59<10:26, 2.87s/it]
86%|████████▌ | 1353/1570 [1:45:02<10:22, 2.87s/it]
86%|████████▌ | 1354/1570 [1:45:05<10:20, 2.87s/it]
86%|████████▋ | 1355/1570 [1:45:08<10:15, 2.86s/it]
86%|████████▋ | 1356/1570 [1:45:11<10:10, 2.85s/it]
86%|████████▋ | 1357/1570 [1:45:13<10:07, 2.85s/it]
86%|████████▋ | 1358/1570 [1:45:16<10:02, 2.84s/it]
87%|████████▋ | 1359/1570 [1:45:19<10:02, 2.85s/it]
87%|████████▋ | 1360/1570 [1:45:22<10:03, 2.87s/it]
{'loss': '0.004314', 'grad_norm': '0.6861', 'learning_rate': '4.486e-06', 'epoch': '4.332'}
+
87%|████████▋ | 1360/1570 [1:45:22<10:03, 2.87s/it]
87%|████████▋ | 1361/1570 [1:45:25<09:58, 2.86s/it]
87%|████████▋ | 1362/1570 [1:45:28<09:54, 2.86s/it]
87%|████████▋ | 1363/1570 [1:45:31<09:49, 2.85s/it]
87%|████████▋ | 1364/1570 [1:45:33<09:48, 2.86s/it]
87%|████████▋ | 1365/1570 [1:45:36<09:46, 2.86s/it]
87%|████████▋ | 1366/1570 [1:45:39<09:43, 2.86s/it]
87%|████████▋ | 1367/1570 [1:45:42<09:40, 2.86s/it]
87%|████████▋ | 1368/1570 [1:45:45<09:37, 2.86s/it]
87%|████████▋ | 1369/1570 [1:45:48<09:32, 2.85s/it]
87%|████████▋ | 1370/1570 [1:45:51<09:30, 2.85s/it]
87%|████████▋ | 1371/1570 [1:45:53<09:28, 2.85s/it]
87%|████████▋ | 1372/1570 [1:45:56<09:26, 2.86s/it]
87%|████████▋ | 1373/1570 [1:45:59<09:21, 2.85s/it]
88%|████████▊ | 1374/1570 [1:46:02<09:18, 2.85s/it]
88%|████████▊ | 1375/1570 [1:46:05<09:16, 2.85s/it]
88%|████████▊ | 1376/1570 [1:46:08<09:11, 2.85s/it]
88%|████████▊ | 1377/1570 [1:46:10<09:03, 2.82s/it]
88%|████████▊ | 1378/1570 [1:46:13<09:04, 2.83s/it]
88%|████████▊ | 1379/1570 [1:46:16<09:02, 2.84s/it]
88%|████████▊ | 1380/1570 [1:46:19<08:59, 2.84s/it]
{'loss': '0.001426', 'grad_norm': '0.02545', 'learning_rate': '3.704e-06', 'epoch': '4.396'}
+
88%|████████▊ | 1380/1570 [1:46:19<08:59, 2.84s/it]
88%|████████▊ | 1381/1570 [1:46:22<09:23, 2.98s/it]
88%|████████▊ | 1382/1570 [1:46:25<09:11, 2.94s/it]
88%|████████▊ | 1383/1570 [1:46:28<09:05, 2.92s/it]
88%|████████▊ | 1384/1570 [1:46:31<08:57, 2.89s/it]
88%|████████▊ | 1385/1570 [1:46:34<08:53, 2.88s/it]
88%|████████▊ | 1386/1570 [1:46:36<08:47, 2.87s/it]
88%|████████▊ | 1387/1570 [1:46:39<08:46, 2.87s/it]
88%|████████▊ | 1388/1570 [1:46:42<08:43, 2.88s/it]
88%|████████▊ | 1389/1570 [1:46:45<08:39, 2.87s/it]
89%|████████▊ | 1390/1570 [1:46:48<09:00, 3.00s/it]
89%|████████▊ | 1391/1570 [1:46:51<08:52, 2.97s/it]
89%|████████▊ | 1392/1570 [1:46:54<08:41, 2.93s/it]
89%|████████▊ | 1393/1570 [1:46:57<08:35, 2.91s/it]
89%|████████▉ | 1394/1570 [1:47:00<08:31, 2.91s/it]
89%|████████▉ | 1395/1570 [1:47:03<08:26, 2.90s/it]
89%|████████▉ | 1396/1570 [1:47:06<08:22, 2.89s/it]
89%|████████▉ | 1397/1570 [1:47:09<08:18, 2.88s/it]
89%|████████▉ | 1398/1570 [1:47:11<08:15, 2.88s/it]
89%|████████▉ | 1399/1570 [1:47:14<08:12, 2.88s/it]
89%|████████▉ | 1400/1570 [1:47:18<08:39, 3.06s/it]
{'loss': '0.001893', 'grad_norm': '0.0237', 'learning_rate': '2.996e-06', 'epoch': '4.46'}
+
89%|████████▉ | 1400/1570 [1:47:18<08:39, 3.06s/it]
89%|████████▉ | 1401/1570 [1:47:21<09:09, 3.25s/it]
89%|████████▉ | 1402/1570 [1:47:26<09:46, 3.49s/it]
89%|████████▉ | 1403/1570 [1:47:30<10:08, 3.65s/it]
89%|████████▉ | 1404/1570 [1:47:33<10:20, 3.74s/it]
89%|████████▉ | 1405/1570 [1:47:37<10:30, 3.82s/it]
90%|████████▉ | 1406/1570 [1:47:42<10:36, 3.88s/it]
90%|████████▉ | 1407/1570 [1:47:45<10:25, 3.84s/it]
90%|████████▉ | 1408/1570 [1:47:49<10:30, 3.89s/it]
90%|████████▉ | 1409/1570 [1:47:52<09:50, 3.67s/it]
90%|████████▉ | 1410/1570 [1:47:55<09:08, 3.43s/it]
90%|████████▉ | 1411/1570 [1:47:58<08:37, 3.25s/it]
90%|████████▉ | 1412/1570 [1:48:01<08:17, 3.15s/it]
90%|█████████ | 1413/1570 [1:48:04<08:01, 3.07s/it]
90%|█████████ | 1414/1570 [1:48:07<07:49, 3.01s/it]
90%|█████████ | 1415/1570 [1:48:10<07:41, 2.98s/it]
90%|█████████ | 1416/1570 [1:48:13<07:31, 2.93s/it]
90%|█████████ | 1417/1570 [1:48:15<07:24, 2.90s/it]
90%|█████████ | 1418/1570 [1:48:18<07:18, 2.88s/it]
90%|█████████ | 1419/1570 [1:48:21<07:14, 2.87s/it]
90%|█████████ | 1420/1570 [1:48:24<07:12, 2.88s/it]
{'loss': '0.001612', 'grad_norm': '0.04371', 'learning_rate': '2.363e-06', 'epoch': '4.523'}
+
90%|█████████ | 1420/1570 [1:48:24<07:12, 2.88s/it]
91%|█████████ | 1421/1570 [1:48:27<07:07, 2.87s/it]
91%|█████████ | 1422/1570 [1:48:30<07:02, 2.86s/it]
91%|█████████ | 1423/1570 [1:48:32<06:59, 2.85s/it]
91%|█████████ | 1424/1570 [1:48:35<06:58, 2.87s/it]
91%|█████████ | 1425/1570 [1:48:38<06:56, 2.87s/it]
91%|█████████ | 1426/1570 [1:48:41<06:53, 2.87s/it]
91%|█████████ | 1427/1570 [1:48:44<06:48, 2.86s/it]
91%|█████████ | 1428/1570 [1:48:47<06:45, 2.86s/it]
91%|█████████ | 1429/1570 [1:48:50<06:43, 2.86s/it]
91%|█████████ | 1430/1570 [1:48:52<06:39, 2.85s/it]
91%|█████████ | 1431/1570 [1:48:55<06:36, 2.85s/it]
91%|█████████ | 1432/1570 [1:48:58<06:35, 2.86s/it]
91%|█████████▏| 1433/1570 [1:49:01<06:33, 2.87s/it]
91%|█████████▏| 1434/1570 [1:49:04<06:33, 2.89s/it]
91%|█████████▏| 1435/1570 [1:49:07<06:29, 2.89s/it]
91%|█████████▏| 1436/1570 [1:49:10<06:24, 2.87s/it]
92%|█████████▏| 1437/1570 [1:49:13<06:22, 2.87s/it]
92%|█████████▏| 1438/1570 [1:49:16<06:20, 2.89s/it]
92%|█████████▏| 1439/1570 [1:49:18<06:17, 2.88s/it]
92%|█████████▏| 1440/1570 [1:49:21<06:14, 2.88s/it]
{'loss': '0.001082', 'grad_norm': '0.01302', 'learning_rate': '1.806e-06', 'epoch': '4.587'}
+
92%|█████████▏| 1440/1570 [1:49:21<06:14, 2.88s/it]
92%|█████████▏| 1441/1570 [1:49:24<06:12, 2.89s/it]
92%|█████████▏| 1442/1570 [1:49:27<06:06, 2.87s/it]
92%|█████████▏| 1443/1570 [1:49:30<06:04, 2.87s/it]
92%|█████████▏| 1444/1570 [1:49:33<06:01, 2.87s/it]
92%|█████████▏| 1445/1570 [1:49:36<05:57, 2.86s/it]
92%|█████████▏| 1446/1570 [1:49:39<05:56, 2.87s/it]
92%|█████████▏| 1447/1570 [1:49:41<05:52, 2.87s/it]
92%|█████████▏| 1448/1570 [1:49:44<05:49, 2.87s/it]
92%|█████████▏| 1449/1570 [1:49:47<05:46, 2.86s/it]
92%|█████████▏| 1450/1570 [1:49:50<05:44, 2.87s/it]
92%|█████████▏| 1451/1570 [1:49:53<05:46, 2.91s/it]
92%|█████████▏| 1452/1570 [1:49:56<05:50, 2.97s/it]
93%|█████████▎| 1453/1570 [1:49:59<05:52, 3.02s/it]
93%|█████████▎| 1454/1570 [1:50:02<05:51, 3.03s/it]
93%|█████████▎| 1455/1570 [1:50:05<05:49, 3.04s/it]
93%|█████████▎| 1456/1570 [1:50:08<05:40, 2.99s/it]
93%|█████████▎| 1457/1570 [1:50:11<05:33, 2.95s/it]
93%|█████████▎| 1458/1570 [1:50:14<05:30, 2.95s/it]
93%|█████████▎| 1459/1570 [1:50:17<05:25, 2.93s/it]
93%|█████████▎| 1460/1570 [1:50:20<05:20, 2.91s/it]
{'loss': '0.0008092', 'grad_norm': '0.03708', 'learning_rate': '1.327e-06', 'epoch': '4.651'}
+
93%|█████████▎| 1460/1570 [1:50:20<05:20, 2.91s/it]
93%|█████████▎| 1461/1570 [1:50:23<05:15, 2.90s/it]
93%|█████████▎| 1462/1570 [1:50:26<05:12, 2.89s/it]
93%|█████████▎| 1463/1570 [1:50:28<05:10, 2.90s/it]
93%|█████████▎| 1464/1570 [1:50:31<05:07, 2.90s/it]
93%|█████████▎| 1465/1570 [1:50:35<05:24, 3.09s/it]
93%|█████████▎| 1466/1570 [1:50:39<05:42, 3.30s/it]
93%|█████████▎| 1467/1570 [1:50:43<06:00, 3.50s/it]
94%|█████████▎| 1468/1570 [1:50:47<06:13, 3.66s/it]
94%|█████████▎| 1469/1570 [1:50:51<06:19, 3.76s/it]
94%|█████████▎| 1470/1570 [1:50:55<06:23, 3.84s/it]
94%|█████████▎| 1471/1570 [1:50:59<06:24, 3.89s/it]
94%|█████████▍| 1472/1570 [1:51:02<06:19, 3.87s/it]
94%|█████████▍| 1473/1570 [1:51:06<06:17, 3.89s/it]
94%|█████████▍| 1474/1570 [1:51:10<06:16, 3.92s/it]
94%|█████████▍| 1475/1570 [1:51:14<06:12, 3.92s/it]
94%|█████████▍| 1476/1570 [1:51:18<06:08, 3.92s/it]
94%|█████████▍| 1477/1570 [1:51:22<06:01, 3.89s/it]
94%|█████████▍| 1478/1570 [1:51:26<05:57, 3.88s/it]
94%|█████████▍| 1479/1570 [1:51:30<05:54, 3.90s/it]
94%|█████████▍| 1480/1570 [1:51:34<05:53, 3.92s/it]
{'loss': '0.0007033', 'grad_norm': '0.006077', 'learning_rate': '9.258e-07', 'epoch': '4.715'}
+
94%|█████████▍| 1480/1570 [1:51:34<05:53, 3.92s/it]
94%|█████████▍| 1481/1570 [1:51:38<05:51, 3.95s/it]
94%|█████████▍| 1482/1570 [1:51:42<05:46, 3.94s/it]
94%|█████████▍| 1483/1570 [1:51:46<05:43, 3.95s/it]
95%|█████████▍| 1484/1570 [1:51:50<05:38, 3.94s/it]
95%|█████████▍| 1485/1570 [1:51:54<05:35, 3.95s/it]
95%|█████████▍| 1486/1570 [1:51:57<05:16, 3.77s/it]
95%|█████████▍| 1487/1570 [1:52:01<05:17, 3.82s/it]
95%|█████████▍| 1488/1570 [1:52:05<05:18, 3.88s/it]
95%|█████████▍| 1489/1570 [1:52:09<05:15, 3.90s/it]
95%|█████████▍| 1490/1570 [1:52:13<05:10, 3.89s/it]
95%|█████████▍| 1491/1570 [1:52:17<05:09, 3.92s/it]
95%|█████████▌| 1492/1570 [1:52:21<05:07, 3.95s/it]
95%|█████████▌| 1493/1570 [1:52:25<05:04, 3.96s/it]
95%|█████████▌| 1494/1570 [1:52:29<05:01, 3.97s/it]
95%|█████████▌| 1495/1570 [1:52:33<04:56, 3.96s/it]
95%|█████████▌| 1496/1570 [1:52:37<04:54, 3.98s/it]
95%|█████████▌| 1497/1570 [1:52:41<04:50, 3.97s/it]
95%|█████████▌| 1498/1570 [1:52:44<04:42, 3.92s/it]
95%|█████████▌| 1499/1570 [1:52:48<04:29, 3.80s/it]
96%|█████████▌| 1500/1570 [1:52:52<04:20, 3.73s/it]
{'loss': '0.001508', 'grad_norm': '0.01427', 'learning_rate': '6.033e-07', 'epoch': '4.779'}
+
96%|█████████▌| 1500/1570 [1:52:52<04:20, 3.73s/it]
96%|█████████▌| 1501/1570 [1:52:55<04:14, 3.68s/it]
96%|█████████▌| 1502/1570 [1:52:59<04:08, 3.65s/it]
96%|█████████▌| 1503/1570 [1:53:02<04:00, 3.59s/it]
96%|█████████▌| 1504/1570 [1:53:06<03:56, 3.59s/it]
96%|█████████▌| 1505/1570 [1:53:09<03:53, 3.59s/it]
96%|█████████▌| 1506/1570 [1:53:13<03:49, 3.59s/it]
96%|█████████▌| 1507/1570 [1:53:17<03:46, 3.59s/it]
96%|█████████▌| 1508/1570 [1:53:20<03:41, 3.58s/it]
96%|█████████▌| 1509/1570 [1:53:24<03:37, 3.57s/it]
96%|█████████▌| 1510/1570 [1:53:27<03:34, 3.57s/it]
96%|█████████▌| 1511/1570 [1:53:31<03:30, 3.58s/it]
96%|█████████▋| 1512/1570 [1:53:34<03:27, 3.58s/it]
96%|█████████▋| 1513/1570 [1:53:38<03:24, 3.58s/it]
96%|█████████▋| 1514/1570 [1:53:42<03:20, 3.59s/it]
96%|█████████▋| 1515/1570 [1:53:45<03:11, 3.49s/it]
97%|█████████▋| 1516/1570 [1:53:48<03:04, 3.42s/it]
97%|█████████▋| 1517/1570 [1:53:51<02:58, 3.37s/it]
97%|█████████▋| 1518/1570 [1:53:55<02:53, 3.34s/it]
97%|█████████▋| 1519/1570 [1:53:58<02:48, 3.31s/it]
97%|█████████▋| 1520/1570 [1:54:01<02:44, 3.29s/it]
{'loss': '0.004572', 'grad_norm': '0.01251', 'learning_rate': '3.599e-07', 'epoch': '4.842'}
+
97%|█████████▋| 1520/1570 [1:54:01<02:44, 3.29s/it]
97%|█████████▋| 1521/1570 [1:54:04<02:40, 3.28s/it]
97%|█████████▋| 1522/1570 [1:54:08<02:36, 3.26s/it]
97%|█████████▋| 1523/1570 [1:54:10<02:28, 3.16s/it]
97%|█████████▋| 1524/1570 [1:54:13<02:21, 3.09s/it]
97%|█████████▋| 1525/1570 [1:54:17<02:19, 3.11s/it]
97%|█████████▋| 1526/1570 [1:54:20<02:16, 3.11s/it]
97%|█████████▋| 1527/1570 [1:54:23<02:16, 3.17s/it]
97%|█████████▋| 1528/1570 [1:54:26<02:14, 3.21s/it]
97%|█████████▋| 1529/1570 [1:54:29<02:11, 3.22s/it]
97%|█████████▋| 1530/1570 [1:54:33<02:08, 3.22s/it]
98%|█████████▊| 1531/1570 [1:54:36<02:05, 3.21s/it]
98%|█████████▊| 1532/1570 [1:54:39<01:57, 3.10s/it]
98%|█████████▊| 1533/1570 [1:54:42<01:51, 3.03s/it]
98%|█████████▊| 1534/1570 [1:54:44<01:46, 2.96s/it]
98%|█████████▊| 1535/1570 [1:54:47<01:43, 2.95s/it]
98%|█████████▊| 1536/1570 [1:54:50<01:39, 2.92s/it]
98%|█████████▊| 1537/1570 [1:54:53<01:35, 2.90s/it]
98%|█████████▊| 1538/1570 [1:54:56<01:32, 2.89s/it]
98%|█████████▊| 1539/1570 [1:54:59<01:29, 2.87s/it]
98%|█████████▊| 1540/1570 [1:55:02<01:25, 2.86s/it]
{'loss': '0.0009093', 'grad_norm': '0.1619', 'learning_rate': '1.961e-07', 'epoch': '4.906'}
+
98%|█████████▊| 1540/1570 [1:55:02<01:25, 2.86s/it]
98%|█████████▊| 1541/1570 [1:55:04<01:23, 2.87s/it]
98%|█████████▊| 1542/1570 [1:55:07<01:19, 2.85s/it]
98%|█████████▊| 1543/1570 [1:55:10<01:17, 2.87s/it]
98%|█████████▊| 1544/1570 [1:55:13<01:14, 2.86s/it]
98%|█████████▊| 1545/1570 [1:55:16<01:11, 2.86s/it]
98%|█████████▊| 1546/1570 [1:55:19<01:08, 2.85s/it]
99%|█████████▊| 1547/1570 [1:55:22<01:05, 2.86s/it]
99%|█████████▊| 1548/1570 [1:55:24<01:02, 2.86s/it]
99%|█████████▊| 1549/1570 [1:55:27<01:00, 2.87s/it]
99%|█████████▊| 1550/1570 [1:55:30<00:57, 2.87s/it]
99%|█████████▉| 1551/1570 [1:55:33<00:54, 2.87s/it]
99%|█████████▉| 1552/1570 [1:55:36<00:51, 2.87s/it]
99%|█████████▉| 1553/1570 [1:55:39<00:48, 2.87s/it]
99%|█████████▉| 1554/1570 [1:55:42<00:45, 2.86s/it]
99%|█████████▉| 1555/1570 [1:55:45<00:42, 2.86s/it]
99%|█████████▉| 1556/1570 [1:55:47<00:40, 2.86s/it]
99%|█████████▉| 1557/1570 [1:55:50<00:37, 2.87s/it]
99%|█████████▉| 1558/1570 [1:55:53<00:34, 2.86s/it]
99%|█████████▉| 1559/1570 [1:55:56<00:31, 2.85s/it]
99%|█████████▉| 1560/1570 [1:55:59<00:28, 2.85s/it]
{'loss': '0.001506', 'grad_norm': '0.02685', 'learning_rate': '1.121e-07', 'epoch': '4.97'}
+
99%|█████████▉| 1560/1570 [1:55:59<00:28, 2.85s/it]
99%|█████████▉| 1561/1570 [1:56:02<00:25, 2.86s/it]
99%|█████████▉| 1562/1570 [1:56:05<00:22, 2.85s/it]
100%|█████████▉| 1563/1570 [1:56:07<00:19, 2.85s/it]
100%|█████████▉| 1564/1570 [1:56:10<00:17, 2.85s/it]
100%|█████████▉| 1565/1570 [1:56:13<00:14, 2.85s/it]
100%|█████████▉| 1566/1570 [1:56:16<00:11, 2.85s/it]
100%|█████████▉| 1567/1570 [1:56:19<00:08, 2.85s/it]
100%|█████████▉| 1568/1570 [1:56:22<00:05, 2.85s/it]
100%|█████████▉| 1569/1570 [1:56:24<00:02, 2.84s/it]
100%|██████████| 1570/1570 [1:56:26<00:00, 2.33s/it][INFO|trainer.py:2668] 2026-08-22 16:55:24,506 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 16:55:24,507 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 16:55:24,507 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/sfkl/checkpoint-1570
+[INFO|configuration_utils.py:780] 2026-08-22 17:05:38,140 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 17:05:38,142 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 17:05:38,176 >> chat template saved in results/qwen3/sfkl/checkpoint-1570/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 17:05:38,176 >> tokenizer config file saved in results/qwen3/sfkl/checkpoint-1570/tokenizer_config.json
+[INFO|trainer.py:1823] 2026-08-22 17:05:38,972 >>
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
{'train_runtime': '7601', 'train_samples_per_second': '6.596', 'train_steps_per_second': '0.207', 'train_loss': '0.01212', 'epoch': '5'}
+
100%|██████████| 1570/1570 [2:06:40<00:00, 2.33s/it]
100%|██████████| 1570/1570 [2:06:40<00:00, 4.84s/it]
+[INFO|trainer.py:3815] 2026-08-22 17:05:39,449 >> Saving model checkpoint to results/qwen3/sfkl
+[transformers] The following generation flags are not valid and may be ignored: ['temperature', 'top_p']. Set `TRANSFORMERS_VERBOSITY=info` for more details.
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[INFO|configuration_utils.py:780] 2026-08-22 17:05:40,032 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 17:05:40,033 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 17:05:40,067 >> chat template saved in results/qwen3/sfkl/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 17:05:40,067 >> tokenizer config file saved in results/qwen3/sfkl/tokenizer_config.json
+***** train metrics *****
+ epoch = 5.0
+ total_flos = 54682091GF
+ train_loss = 0.0121
+ train_runtime = 2:06:40.54
+ train_samples_per_second = 6.596
+ train_steps_per_second = 0.207
+[INFO|trainer.py:2668] 2026-08-22 17:05:40,194 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 17:05:40,194 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 17:05:40,194 >> Batch size = 16
+[WARNING|logging.py:340] 2026-08-22 17:05:40,313 >> The following generation flags are not valid and may be ignored: ['temperature', 'top_p'].
+[INFO|logging.py:355] 2026-08-22 17:05:40,313 >> - `temperature`: `do_sample` is not set to `True`. However, `temperature` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.
+- `top_p`: `do_sample` is not set to `True`. However, `top_p` is set to `0.7` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.
+If you're using a pretrained model, note that some of these attributes may be set through the model's `generation_config.json` file.
+[WARNING|utils.py:1642] 2026-08-22 17:05:40,315 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
0%| | 0/107 [00:00> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
2%|▏ | 2/107 [00:05<04:53, 2.80s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:05:51,613 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
3%|▎ | 3/107 [00:11<06:51, 3.95s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:05:57,186 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
4%|▎ | 4/107 [00:16<07:50, 4.57s/it][WARNING|utils.py:1642] 2026-08-22 17:06:02,790 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
5%|▍ | 5/107 [00:22<08:22, 4.93s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:06:08,394 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
6%|▌ | 6/107 [00:27<08:15, 4.91s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:06:13,254 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 7/107 [00:32<08:30, 5.11s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:06:18,786 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 8/107 [00:38<08:35, 5.21s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:06:24,219 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
8%|▊ | 9/107 [00:44<08:54, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:06:30,229 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
9%|▉ | 10/107 [00:49<08:59, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:06:36,010 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
10%|█ | 11/107 [00:54<08:28, 5.30s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:06:40,725 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
11%|█ | 12/107 [01:00<08:38, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:06:46,545 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
12%|█▏ | 13/107 [01:06<08:37, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:06:52,151 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
13%|█▎ | 14/107 [01:11<08:28, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:06:57,523 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
14%|█▍ | 15/107 [01:17<08:26, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:07:03,144 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
15%|█▍ | 16/107 [01:22<08:25, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:07:08,817 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
16%|█▌ | 17/107 [01:28<08:31, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:07:14,774 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
17%|█▋ | 18/107 [01:34<08:27, 5.70s/it][WARNING|utils.py:1642] 2026-08-22 17:07:20,531 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
18%|█▊ | 19/107 [01:40<08:21, 5.69s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:07:26,203 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
19%|█▊ | 20/107 [01:45<08:14, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:07:31,854 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
20%|█▉ | 21/107 [01:50<07:52, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:07:36,895 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██ | 22/107 [01:56<07:55, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:07:42,734 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██▏ | 23/107 [02:02<07:51, 5.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:07:48,382 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
22%|██▏ | 24/107 [02:07<07:44, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:07:53,927 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
23%|██▎ | 25/107 [02:14<08:02, 5.88s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:08:00,488 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
24%|██▍ | 26/107 [02:20<08:00, 5.93s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:08:06,519 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
25%|██▌ | 27/107 [02:26<07:49, 5.86s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:08:12,236 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
26%|██▌ | 28/107 [02:32<07:45, 5.89s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:08:18,191 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
27%|██▋ | 29/107 [02:38<07:49, 6.03s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:08:24,530 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
28%|██▊ | 30/107 [02:44<07:33, 5.89s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:08:30,100 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
29%|██▉ | 31/107 [02:48<06:59, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:08:34,743 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
30%|██▉ | 32/107 [02:54<06:54, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:08:40,277 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
31%|███ | 33/107 [02:59<06:51, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:08:45,915 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
32%|███▏ | 34/107 [03:06<07:03, 5.80s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:08:52,281 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
33%|███▎ | 35/107 [03:11<06:52, 5.73s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:08:57,841 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
34%|███▎ | 36/107 [03:17<06:41, 5.65s/it][WARNING|utils.py:1642] 2026-08-22 17:09:03,319 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
35%|███▍ | 37/107 [03:22<06:18, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:09:08,173 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▌ | 38/107 [03:28<06:40, 5.81s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:09:14,912 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▋ | 39/107 [03:33<06:19, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:09:19,969 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
37%|███▋ | 40/107 [03:39<06:10, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:09:25,362 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
38%|███▊ | 41/107 [03:45<06:08, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:09:31,082 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
39%|███▉ | 42/107 [03:51<06:15, 5.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:09:37,307 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
40%|████ | 43/107 [03:57<06:14, 5.84s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:09:43,308 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
41%|████ | 44/107 [04:02<06:02, 5.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:09:48,853 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
42%|████▏ | 45/107 [04:08<05:46, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:09:54,054 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
43%|████▎ | 46/107 [04:13<05:38, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:09:59,518 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
44%|████▍ | 47/107 [04:19<05:39, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:10:05,446 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
45%|████▍ | 48/107 [04:25<05:33, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:10:11,083 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
46%|████▌ | 49/107 [04:30<05:22, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:10:16,402 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
47%|████▋ | 50/107 [04:35<05:14, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:10:21,856 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
48%|████▊ | 51/107 [04:41<05:13, 5.60s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:10:27,639 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
49%|████▊ | 52/107 [04:46<05:01, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:10:32,858 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|████▉ | 53/107 [04:52<05:03, 5.62s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:10:38,795 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|█████ | 54/107 [04:58<04:53, 5.54s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:10:44,132 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
51%|█████▏ | 55/107 [05:03<04:51, 5.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:10:49,916 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
52%|█████▏ | 56/107 [05:09<04:52, 5.73s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:10:55,910 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
53%|█████▎ | 57/107 [05:15<04:47, 5.76s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:11:01,743 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
54%|█████▍ | 58/107 [05:21<04:45, 5.83s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:11:07,727 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
55%|█████▌ | 59/107 [05:27<04:37, 5.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:11:13,394 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
56%|█████▌ | 60/107 [05:33<04:29, 5.74s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:11:19,034 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
57%|█████▋ | 61/107 [05:38<04:14, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:11:24,065 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
58%|█████▊ | 62/107 [05:43<04:07, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:11:29,540 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
59%|█████▉ | 63/107 [05:49<04:03, 5.54s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:11:35,161 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
60%|█████▉ | 64/107 [05:54<03:56, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:11:40,532 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
61%|██████ | 65/107 [05:59<03:48, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:11:45,834 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
62%|██████▏ | 66/107 [06:05<03:42, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:11:51,270 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
63%|██████▎ | 67/107 [06:10<03:37, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:11:56,713 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▎ | 68/107 [06:15<03:25, 5.27s/it][WARNING|utils.py:1642] 2026-08-22 17:12:01,598 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▍ | 69/107 [06:20<03:19, 5.25s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:12:06,801 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
65%|██████▌ | 70/107 [06:26<03:23, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:12:12,859 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
66%|██████▋ | 71/107 [06:32<03:21, 5.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:12:18,725 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
67%|██████▋ | 72/107 [06:38<03:14, 5.54s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:12:24,129 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
68%|██████▊ | 73/107 [06:43<03:09, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:12:29,768 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
69%|██████▉ | 74/107 [06:49<03:03, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:12:35,287 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
70%|███████ | 75/107 [06:54<02:56, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:12:40,678 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
71%|███████ | 76/107 [07:00<02:51, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:12:46,237 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
72%|███████▏ | 77/107 [07:05<02:45, 5.53s/it][WARNING|utils.py:1642] 2026-08-22 17:12:51,771 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
73%|███████▎ | 78/107 [07:11<02:41, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:12:57,474 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
74%|███████▍ | 79/107 [07:16<02:35, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:13:02,973 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
75%|███████▍ | 80/107 [07:22<02:32, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:13:08,842 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
76%|███████▌ | 81/107 [07:28<02:25, 5.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:13:14,362 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
77%|███████▋ | 82/107 [07:33<02:18, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:13:19,766 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
78%|███████▊ | 83/107 [07:39<02:11, 5.47s/it][WARNING|utils.py:1642] 2026-08-22 17:13:25,040 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▊ | 84/107 [07:44<02:04, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:13:30,301 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▉ | 85/107 [07:49<01:55, 5.26s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:13:35,208 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
80%|████████ | 86/107 [07:54<01:53, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:13:40,887 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
81%|████████▏ | 87/107 [07:59<01:43, 5.17s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:13:45,553 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
82%|████████▏ | 88/107 [08:05<01:40, 5.26s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:13:51,045 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
83%|████████▎ | 89/107 [08:11<01:41, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:13:57,603 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
84%|████████▍ | 90/107 [08:17<01:35, 5.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:14:03,122 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
85%|████████▌ | 91/107 [08:22<01:28, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:14:08,361 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
86%|████████▌ | 92/107 [08:27<01:21, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:14:13,630 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
87%|████████▋ | 93/107 [08:33<01:16, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:14:19,185 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
88%|████████▊ | 94/107 [08:38<01:12, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:14:25,013 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
89%|████████▉ | 95/107 [08:44<01:06, 5.56s/it][WARNING|utils.py:1642] 2026-08-22 17:14:30,533 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
90%|████████▉ | 96/107 [08:49<01:00, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:14:35,922 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
91%|█████████ | 97/107 [08:55<00:55, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:14:41,621 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
92%|█████████▏| 98/107 [09:01<00:50, 5.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:14:47,325 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 99/107 [09:06<00:44, 5.62s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:14:52,991 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 100/107 [09:12<00:38, 5.54s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:14:58,330 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
94%|█████████▍| 101/107 [09:18<00:33, 5.63s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:15:04,158 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
95%|█████████▌| 102/107 [09:23<00:28, 5.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:15:09,726 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
96%|█████████▋| 103/107 [09:29<00:22, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:15:15,031 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
97%|█████████▋| 104/107 [09:34<00:16, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:15:20,599 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
98%|█████████▊| 105/107 [09:40<00:11, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:15:26,301 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
99%|█████████▉| 106/107 [09:45<00:05, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 17:15:31,949 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
100%|██████████| 107/107 [09:51<00:00, 5.64s/it]
100%|██████████| 107/107 [09:52<00:00, 5.54s/it]
+***** eval metrics *****
+ epoch = 5.0
+ eval_generator_count = 1712.0
+ eval_generator_exact_match = 96.729
+ eval_generator_rouge1 = 99.9412
+ eval_generator_rouge2 = 99.8856
+ eval_generator_rougeL = 99.7768
+ eval_loss = 0.0032
+ eval_runtime = 0:09:58.58
+ eval_samples_per_second = 5.703
+ eval_selector_accuracy = 98.715
+ eval_selector_count = 1712.0
+ eval_steps_per_second = 0.179
+[INFO|modelcard.py:263] 2026-08-22 17:15:38,777 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
diff --git a/qwen3/run_all_logs/20260822_031516/sft.log b/qwen3/run_all_logs/20260822_031516/sft.log
new file mode 100644
index 0000000000000000000000000000000000000000..fcf4d57bf092d71fd70ae88dfa871fb3785df6af
--- /dev/null
+++ b/qwen3/run_all_logs/20260822_031516/sft.log
@@ -0,0 +1,1772 @@
+W0822 08:12:29.731000 1354901 torch/distributed/run.py:803]
+W0822 08:12:29.731000 1354901 torch/distributed/run.py:803] *****************************************
+W0822 08:12:29.731000 1354901 torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+W0822 08:12:29.731000 1354901 torch/distributed/run.py:803] *****************************************
+[INFO|2026-08-22 08:12:36] llamafactory.hparams.parser:651 >> Process rank: 1, world size: 2, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16
+[INFO|2026-08-22 08:12:36] llamafactory.hparams.parser:144 >> Set `ddp_find_unused_parameters` to False in DDP training since LoRA is enabled.
+[INFO|2026-08-22 08:12:36] llamafactory.hparams.parser:651 >> Process rank: 0, world size: 2, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16
+Starting sft with LlamaFactory and DeepSpeed=True.
+[INFO|configuration_utils.py:780] 2026-08-22 08:12:37,790 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 08:12:37,795 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 08:12:40,889 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 08:12:40,892 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 08:12:41,130 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 08:12:41,131 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 08:12:42] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_train.jsonl...
+[INFO|2026-08-22 08:12:43] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_eval.jsonl...
+
Running tokenizer on dataset: 0%| | 0/10027 [00:00system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of art movements linked to paintings in the genre art style that ended before the year 1600?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "ArtMovement",
+ "properties": {
+ "end_year": "INTEGER",
+ "start_year": "INTEGER"
+ }
+ },
+ {
+ "label": "Genre",
+ "properties": {}
+ },
+ {
+ "label": "Painting",
+ "properties": {
+ "country_of_origin": "STRING",
+ "creation_year": "INTEGER"
+ }
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "ArtMovement",
+ "type": "associatedWith"
+ },
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "Genre",
+ "type": "hasGenre"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+
Running tokenizer on dataset: 0%| | 0/3414 [00:00system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of weapons used in either the 2021 Baghdad suicide bombings or the Botroseya Church bombing?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "TerroristAttack",
+ "properties": {
+ "date": "DATE",
+ "locations": "LIST",
+ "number_of_deaths": "INTEGER",
+ "number_of_injuries": "INTEGER"
+ }
+ },
+ {
+ "label": "Weapon",
+ "properties": {}
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "TerroristAttack",
+ "target": "Weapon",
+ "type": "employs"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+[INFO|configuration_utils.py:780] 2026-08-22 08:12:59,086 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 08:12:59,089 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 08:12:59] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training.
+[INFO|modeling_utils.py:769] 2026-08-22 08:12:59,836 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/model.safetensors
+[INFO|modeling_utils.py:839] 2026-08-22 08:12:59,836 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 08:12:59,837 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": false
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 08:13:00,721 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95
+}
+
+[INFO|2026-08-22 08:13:00] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled.
+[INFO|2026-08-22 08:13:00] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-22 08:13:00] llamafactory.model.adapter:144 >> Upcasting trainable params to float32.
+[INFO|2026-08-22 08:13:00] llamafactory.model.adapter:144 >> Fine-tuning method: LoRA
+[INFO|2026-08-22 08:13:00] llamafactory.model.model_utils.misc:144 >> Found linear modules: o_proj,k_proj,up_proj,gate_proj,q_proj,down_proj,v_proj
+[INFO|2026-08-22 08:13:01] llamafactory.model.loader:144 >> trainable params: 20,185,088 || all params: 616,235,008 || trainable%: 3.2756
+[WARNING|trainer_utils.py:1238] 2026-08-22 08:13:01,112 >> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}.
+Gradient accumulation steps mismatch: GradientAccumulationPlugin has 1, DeepSpeed config has 8. Using DeepSpeed's value.
+
Loading weights: 0%| | 0/311 [00:00> ***** Running training *****
+[INFO|trainer.py:1470] 2026-08-22 08:13:05,252 >> Num examples = 10,027
+[INFO|trainer.py:1471] 2026-08-22 08:13:05,252 >> Num Epochs = 5
+[INFO|trainer.py:1472] 2026-08-22 08:13:05,252 >> Num update steps per epoch = 314
+[INFO|trainer.py:1473] 2026-08-22 08:13:05,252 >> Instantaneous batch size per device = 2
+[INFO|trainer.py:1476] 2026-08-22 08:13:05,252 >> Total train batch size (w. parallel, distributed & accumulation) = 32
+[INFO|trainer.py:1477] 2026-08-22 08:13:05,252 >> Gradient Accumulation steps = 8
+[INFO|trainer.py:1478] 2026-08-22 08:13:05,252 >> Total optimization steps = 1,570
+[INFO|trainer.py:1479] 2026-08-22 08:13:05,256 >> Number of trainable parameters = 20,185,088
+
0%| | 0/1570 [00:00>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 08:24:35,370 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 08:24:35,370 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/sft/checkpoint-314
+[INFO|configuration_utils.py:780] 2026-08-22 08:34:20,997 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 08:34:20,998 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 08:34:21,032 >> chat template saved in results/qwen3/sft/checkpoint-314/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 08:34:21,032 >> tokenizer config file saved in results/qwen3/sft/checkpoint-314/tokenizer_config.json
+
20%|██ | 315/1570 [21:18<61:59:51, 177.84s/it]
20%|██ | 316/1570 [21:20<43:35:25, 125.14s/it]
20%|██ | 317/1570 [21:23<30:42:52, 88.25s/it]
20%|██ | 318/1570 [21:25<21:42:58, 62.44s/it]
20%|██ | 319/1570 [21:27<15:24:56, 44.36s/it]
20%|██ | 320/1570 [21:29<11:00:28, 31.70s/it]
{'loss': '0.009933', 'grad_norm': '0.4359', 'learning_rate': '9.016e-05', 'epoch': '1.019'}
+
20%|██ | 320/1570 [21:29<11:00:28, 31.70s/it]
20%|██ | 321/1570 [21:31<7:55:53, 22.86s/it]
21%|██ | 322/1570 [21:34<5:46:26, 16.66s/it]
21%|██ | 323/1570 [21:36<4:15:51, 12.31s/it]
21%|██ | 324/1570 [21:38<3:12:26, 9.27s/it]
21%|██ | 325/1570 [21:40<2:28:00, 7.13s/it]
21%|██ | 326/1570 [21:42<1:57:05, 5.65s/it]
21%|██ | 327/1570 [21:44<1:35:17, 4.60s/it]
21%|██ | 328/1570 [21:47<1:20:13, 3.88s/it]
21%|██ | 329/1570 [21:49<1:09:43, 3.37s/it]
21%|██ | 330/1570 [21:51<1:02:25, 3.02s/it]
21%|██ | 331/1570 [21:53<57:16, 2.77s/it]
21%|██ | 332/1570 [21:55<53:59, 2.62s/it]
21%|██ | 333/1570 [21:58<51:23, 2.49s/it]
21%|██▏ | 334/1570 [22:00<49:19, 2.39s/it]
21%|██▏ | 335/1570 [22:02<48:05, 2.34s/it]
21%|██▏ | 336/1570 [22:04<46:53, 2.28s/it]
21%|██▏ | 337/1570 [22:06<46:17, 2.25s/it]
22%|██▏ | 338/1570 [22:09<45:42, 2.23s/it]
22%|██▏ | 339/1570 [22:11<46:00, 2.24s/it]
22%|██▏ | 340/1570 [22:13<48:05, 2.35s/it]
{'loss': '0.00509', 'grad_norm': '0.1324', 'learning_rate': '8.894e-05', 'epoch': '1.083'}
+
22%|██▏ | 340/1570 [22:13<48:05, 2.35s/it]
22%|██▏ | 341/1570 [22:16<49:52, 2.44s/it]
22%|██▏ | 342/1570 [22:19<51:04, 2.50s/it]
22%|██▏ | 343/1570 [22:21<52:09, 2.55s/it]
22%|██▏ | 344/1570 [22:24<53:14, 2.61s/it]
22%|██▏ | 345/1570 [22:27<53:33, 2.62s/it]
22%|██▏ | 346/1570 [22:29<53:51, 2.64s/it]
22%|██▏ | 347/1570 [22:32<54:00, 2.65s/it]
22%|██▏ | 348/1570 [22:35<54:19, 2.67s/it]
22%|██▏ | 349/1570 [22:37<54:06, 2.66s/it]
22%|██▏ | 350/1570 [22:40<53:40, 2.64s/it]
22%|██▏ | 351/1570 [22:43<54:06, 2.66s/it]
22%|██▏ | 352/1570 [22:45<53:57, 2.66s/it]
22%|██▏ | 353/1570 [22:48<53:43, 2.65s/it]
23%|██▎ | 354/1570 [22:51<54:01, 2.67s/it]
23%|██▎ | 355/1570 [22:53<53:38, 2.65s/it]
23%|██▎ | 356/1570 [22:56<53:56, 2.67s/it]
23%|██▎ | 357/1570 [22:59<53:48, 2.66s/it]
23%|██▎ | 358/1570 [23:01<53:22, 2.64s/it]
23%|██▎ | 359/1570 [23:04<53:40, 2.66s/it]
23%|██▎ | 360/1570 [23:07<53:17, 2.64s/it]
{'loss': '0.005343', 'grad_norm': '0.1341', 'learning_rate': '8.766e-05', 'epoch': '1.147'}
+
23%|██▎ | 360/1570 [23:07<53:17, 2.64s/it]
23%|██▎ | 361/1570 [23:09<53:45, 2.67s/it]
23%|██▎ | 362/1570 [23:12<53:42, 2.67s/it]
23%|██▎ | 363/1570 [23:14<50:42, 2.52s/it]
23%|██▎ | 364/1570 [23:16<48:35, 2.42s/it]
23%|██▎ | 365/1570 [23:18<46:59, 2.34s/it]
23%|██▎ | 366/1570 [23:21<45:48, 2.28s/it]
23%|██▎ | 367/1570 [23:23<44:51, 2.24s/it]
23%|██▎ | 368/1570 [23:25<44:15, 2.21s/it]
24%|██▎ | 369/1570 [23:27<44:12, 2.21s/it]
24%|██▎ | 370/1570 [23:29<43:50, 2.19s/it]
24%|██▎ | 371/1570 [23:31<43:39, 2.19s/it]
24%|██▎ | 372/1570 [23:34<43:20, 2.17s/it]
24%|██▍ | 373/1570 [23:36<46:29, 2.33s/it]
24%|██▍ | 374/1570 [23:39<48:54, 2.45s/it]
24%|██▍ | 375/1570 [23:42<50:27, 2.53s/it]
24%|██▍ | 376/1570 [23:44<51:26, 2.59s/it]
24%|██▍ | 377/1570 [23:47<51:44, 2.60s/it]
24%|██▍ | 378/1570 [23:50<52:07, 2.62s/it]
24%|██▍ | 379/1570 [23:53<52:42, 2.66s/it]
24%|██▍ | 380/1570 [23:55<53:04, 2.68s/it]
{'loss': '0.004769', 'grad_norm': '0.2253', 'learning_rate': '8.631e-05', 'epoch': '1.211'}
+
24%|██▍ | 380/1570 [23:55<53:04, 2.68s/it]
24%|██▍ | 381/1570 [23:58<53:09, 2.68s/it]
24%|██▍ | 382/1570 [24:01<53:20, 2.69s/it]
24%|██▍ | 383/1570 [24:03<50:26, 2.55s/it]
24%|██▍ | 384/1570 [24:05<48:10, 2.44s/it]
25%|██▍ | 385/1570 [24:07<46:33, 2.36s/it]
25%|██▍ | 386/1570 [24:09<45:39, 2.31s/it]
25%|██▍ | 387/1570 [24:12<44:54, 2.28s/it]
25%|██▍ | 388/1570 [24:14<44:14, 2.25s/it]
25%|██▍ | 389/1570 [24:16<44:17, 2.25s/it]
25%|██▍ | 390/1570 [24:18<43:53, 2.23s/it]
25%|██▍ | 391/1570 [24:20<43:27, 2.21s/it]
25%|██▍ | 392/1570 [24:23<46:29, 2.37s/it]
25%|██▌ | 393/1570 [24:26<48:06, 2.45s/it]
25%|██▌ | 394/1570 [24:28<49:36, 2.53s/it]
25%|██▌ | 395/1570 [24:31<50:39, 2.59s/it]
25%|██▌ | 396/1570 [24:34<51:21, 2.62s/it]
25%|██▌ | 397/1570 [24:37<51:40, 2.64s/it]
25%|██▌ | 398/1570 [24:39<52:01, 2.66s/it]
25%|██▌ | 399/1570 [24:42<52:07, 2.67s/it]
25%|██▌ | 400/1570 [24:45<52:03, 2.67s/it]
{'loss': '0.006693', 'grad_norm': '0.2751', 'learning_rate': '8.491e-05', 'epoch': '1.274'}
+
25%|██▌ | 400/1570 [24:45<52:03, 2.67s/it]
26%|██▌ | 401/1570 [24:47<49:03, 2.52s/it]
26%|██▌ | 402/1570 [24:49<46:58, 2.41s/it]
26%|██▌ | 403/1570 [24:51<45:35, 2.34s/it]
26%|██▌ | 404/1570 [24:53<44:28, 2.29s/it]
26%|██▌ | 405/1570 [24:56<43:43, 2.25s/it]
26%|██▌ | 406/1570 [24:58<43:19, 2.23s/it]
26%|██▌ | 407/1570 [25:00<42:49, 2.21s/it]
26%|██▌ | 408/1570 [25:02<42:32, 2.20s/it]
26%|██▌ | 409/1570 [25:04<42:11, 2.18s/it]
26%|██▌ | 410/1570 [25:06<41:50, 2.16s/it]
26%|██▌ | 411/1570 [25:09<42:17, 2.19s/it]
26%|██▌ | 412/1570 [25:11<42:31, 2.20s/it]
26%|██▋ | 413/1570 [25:13<42:33, 2.21s/it]
26%|██▋ | 414/1570 [25:15<42:14, 2.19s/it]
26%|██▋ | 415/1570 [25:17<42:12, 2.19s/it]
26%|██▋ | 416/1570 [25:19<41:54, 2.18s/it]
27%|██▋ | 417/1570 [25:22<41:35, 2.16s/it]
27%|██▋ | 418/1570 [25:24<41:35, 2.17s/it]
27%|██▋ | 419/1570 [25:27<44:52, 2.34s/it]
27%|██▋ | 420/1570 [25:29<47:02, 2.45s/it]
{'loss': '0.0103', 'grad_norm': '0.1621', 'learning_rate': '8.345e-05', 'epoch': '1.338'}
+
27%|██▋ | 420/1570 [25:29<47:02, 2.45s/it]
27%|██▋ | 421/1570 [25:32<48:34, 2.54s/it]
27%|██▋ | 422/1570 [25:35<49:46, 2.60s/it]
27%|██▋ | 423/1570 [25:37<50:21, 2.63s/it]
27%|██▋ | 424/1570 [25:40<50:48, 2.66s/it]
27%|██▋ | 425/1570 [25:43<51:06, 2.68s/it]
27%|██▋ | 426/1570 [25:46<51:10, 2.68s/it]
27%|██▋ | 427/1570 [25:48<51:25, 2.70s/it]
27%|██▋ | 428/1570 [25:51<51:33, 2.71s/it]
27%|██▋ | 429/1570 [25:54<51:23, 2.70s/it]
27%|██▋ | 430/1570 [25:56<51:34, 2.71s/it]
27%|██▋ | 431/1570 [25:59<51:02, 2.69s/it]
28%|██▊ | 432/1570 [26:02<51:08, 2.70s/it]
28%|██▊ | 433/1570 [26:05<51:21, 2.71s/it]
28%|██▊ | 434/1570 [26:07<51:34, 2.72s/it]
28%|██▊ | 435/1570 [26:10<51:35, 2.73s/it]
28%|██▊ | 436/1570 [26:13<51:32, 2.73s/it]
28%|██▊ | 437/1570 [26:16<51:38, 2.73s/it]
28%|██▊ | 438/1570 [26:18<51:42, 2.74s/it]
28%|██▊ | 439/1570 [26:21<51:46, 2.75s/it]
28%|██▊ | 440/1570 [26:24<51:41, 2.74s/it]
{'loss': '0.005286', 'grad_norm': '0.1539', 'learning_rate': '8.194e-05', 'epoch': '1.402'}
+
28%|██▊ | 440/1570 [26:24<51:41, 2.74s/it]
28%|██▊ | 441/1570 [26:26<51:13, 2.72s/it]
28%|██▊ | 442/1570 [26:29<51:01, 2.71s/it]
28%|██▊ | 443/1570 [26:32<50:41, 2.70s/it]
28%|██▊ | 444/1570 [26:35<50:47, 2.71s/it]
28%|██▊ | 445/1570 [26:37<50:46, 2.71s/it]
28%|██▊ | 446/1570 [26:40<50:54, 2.72s/it]
28%|██▊ | 447/1570 [26:43<51:00, 2.73s/it]
29%|██▊ | 448/1570 [26:45<50:58, 2.73s/it]
29%|██▊ | 449/1570 [26:48<50:27, 2.70s/it]
29%|██▊ | 450/1570 [26:51<50:25, 2.70s/it]
29%|██▊ | 451/1570 [26:53<47:25, 2.54s/it]
29%|██▉ | 452/1570 [26:55<45:10, 2.42s/it]
29%|██▉ | 453/1570 [26:57<43:34, 2.34s/it]
29%|██▉ | 454/1570 [26:59<42:25, 2.28s/it]
29%|██▉ | 455/1570 [27:02<41:36, 2.24s/it]
29%|██▉ | 456/1570 [27:04<41:02, 2.21s/it]
29%|██▉ | 457/1570 [27:06<40:44, 2.20s/it]
29%|██▉ | 458/1570 [27:08<40:25, 2.18s/it]
29%|██▉ | 459/1570 [27:10<40:10, 2.17s/it]
29%|██▉ | 460/1570 [27:12<40:12, 2.17s/it]
{'loss': '0.006016', 'grad_norm': '0.9736', 'learning_rate': '8.037e-05', 'epoch': '1.466'}
+
29%|██▉ | 460/1570 [27:12<40:12, 2.17s/it]
29%|██▉ | 461/1570 [27:15<40:08, 2.17s/it]
29%|██▉ | 462/1570 [27:17<40:03, 2.17s/it]
29%|██▉ | 463/1570 [27:19<39:55, 2.16s/it]
30%|██▉ | 464/1570 [27:21<39:49, 2.16s/it]
30%|██▉ | 465/1570 [27:23<39:51, 2.16s/it]
30%|██▉ | 466/1570 [27:25<39:52, 2.17s/it]
30%|██▉ | 467/1570 [27:27<39:49, 2.17s/it]
30%|██▉ | 468/1570 [27:30<39:48, 2.17s/it]
30%|██▉ | 469/1570 [27:32<39:46, 2.17s/it]
30%|██▉ | 470/1570 [27:34<39:39, 2.16s/it]
30%|███ | 471/1570 [27:36<39:32, 2.16s/it]
30%|███ | 472/1570 [27:38<39:36, 2.16s/it]
30%|███ | 473/1570 [27:40<39:33, 2.16s/it]
30%|███ | 474/1570 [27:43<39:31, 2.16s/it]
30%|███ | 475/1570 [27:45<39:29, 2.16s/it]
30%|███ | 476/1570 [27:47<39:25, 2.16s/it]
30%|███ | 477/1570 [27:49<39:24, 2.16s/it]
30%|███ | 478/1570 [27:51<39:23, 2.16s/it]
31%|███ | 479/1570 [27:53<39:21, 2.16s/it]
31%|███ | 480/1570 [27:56<39:20, 2.17s/it]
{'loss': '0.003304', 'grad_norm': '0.05532', 'learning_rate': '7.876e-05', 'epoch': '1.53'}
+
31%|███ | 480/1570 [27:56<39:20, 2.17s/it]
31%|███ | 481/1570 [27:58<39:21, 2.17s/it]
31%|███ | 482/1570 [28:00<39:19, 2.17s/it]
31%|███ | 483/1570 [28:02<39:16, 2.17s/it]
31%|███ | 484/1570 [28:04<39:10, 2.16s/it]
31%|███ | 485/1570 [28:06<39:10, 2.17s/it]
31%|███ | 486/1570 [28:09<39:01, 2.16s/it]
31%|███ | 487/1570 [28:11<39:04, 2.16s/it]
31%|███ | 488/1570 [28:13<39:02, 2.17s/it]
31%|███ | 489/1570 [28:15<39:01, 2.17s/it]
31%|███ | 490/1570 [28:17<38:55, 2.16s/it]
31%|███▏ | 491/1570 [28:19<38:53, 2.16s/it]
31%|███▏ | 492/1570 [28:22<38:46, 2.16s/it]
31%|███▏ | 493/1570 [28:24<38:46, 2.16s/it]
31%|███▏ | 494/1570 [28:26<38:47, 2.16s/it]
32%|███▏ | 495/1570 [28:28<38:48, 2.17s/it]
32%|███▏ | 496/1570 [28:30<38:44, 2.16s/it]
32%|███▏ | 497/1570 [28:32<38:45, 2.17s/it]
32%|███▏ | 498/1570 [28:35<38:39, 2.16s/it]
32%|███▏ | 499/1570 [28:37<38:39, 2.17s/it]
32%|███▏ | 500/1570 [28:39<38:36, 2.16s/it]
{'loss': '0.004242', 'grad_norm': '0.08567', 'learning_rate': '7.71e-05', 'epoch': '1.594'}
+
32%|███▏ | 500/1570 [28:39<38:36, 2.16s/it]
32%|███▏ | 501/1570 [28:41<38:31, 2.16s/it]
32%|███▏ | 502/1570 [28:43<38:29, 2.16s/it]
32%|███▏ | 503/1570 [28:45<38:26, 2.16s/it]
32%|███▏ | 504/1570 [28:48<38:24, 2.16s/it]
32%|███▏ | 505/1570 [28:50<38:20, 2.16s/it]
32%|███▏ | 506/1570 [28:52<38:18, 2.16s/it]
32%|███▏ | 507/1570 [28:54<38:17, 2.16s/it]
32%|███▏ | 508/1570 [28:56<38:14, 2.16s/it]
32%|███▏ | 509/1570 [28:58<38:19, 2.17s/it]
32%|███▏ | 510/1570 [29:01<38:15, 2.17s/it]
33%|███▎ | 511/1570 [29:03<38:10, 2.16s/it]
33%|███▎ | 512/1570 [29:05<38:11, 2.17s/it]
33%|███▎ | 513/1570 [29:07<38:06, 2.16s/it]
33%|███▎ | 514/1570 [29:09<38:06, 2.17s/it]
33%|███▎ | 515/1570 [29:11<38:02, 2.16s/it]
33%|███▎ | 516/1570 [29:14<38:04, 2.17s/it]
33%|███▎ | 517/1570 [29:16<37:58, 2.16s/it]
33%|███▎ | 518/1570 [29:18<37:58, 2.17s/it]
33%|███▎ | 519/1570 [29:20<37:52, 2.16s/it]
33%|███▎ | 520/1570 [29:22<37:51, 2.16s/it]
{'loss': '0.01045', 'grad_norm': '0.07485', 'learning_rate': '7.54e-05', 'epoch': '1.657'}
+
33%|███▎ | 520/1570 [29:22<37:51, 2.16s/it]
33%|███▎ | 521/1570 [29:24<37:53, 2.17s/it]
33%|███▎ | 522/1570 [29:27<37:51, 2.17s/it]
33%|███▎ | 523/1570 [29:29<37:44, 2.16s/it]
33%|███▎ | 524/1570 [29:31<37:44, 2.17s/it]
33%|███▎ | 525/1570 [29:33<37:39, 2.16s/it]
34%|███▎ | 526/1570 [29:35<37:37, 2.16s/it]
34%|███▎ | 527/1570 [29:37<37:31, 2.16s/it]
34%|███▎ | 528/1570 [29:39<37:34, 2.16s/it]
34%|███▎ | 529/1570 [29:42<37:30, 2.16s/it]
34%|███▍ | 530/1570 [29:44<37:29, 2.16s/it]
34%|███▍ | 531/1570 [29:46<37:25, 2.16s/it]
34%|███▍ | 532/1570 [29:48<37:21, 2.16s/it]
34%|███▍ | 533/1570 [29:50<37:11, 2.15s/it]
34%|███▍ | 534/1570 [29:52<37:05, 2.15s/it]
34%|███▍ | 535/1570 [29:55<37:04, 2.15s/it]
34%|███▍ | 536/1570 [29:57<37:01, 2.15s/it]
34%|███▍ | 537/1570 [29:59<37:00, 2.15s/it]
34%|███▍ | 538/1570 [30:01<36:56, 2.15s/it]
34%|███▍ | 539/1570 [30:03<36:54, 2.15s/it]
34%|███▍ | 540/1570 [30:05<36:53, 2.15s/it]
{'loss': '0.012', 'grad_norm': '0.3843', 'learning_rate': '7.366e-05', 'epoch': '1.721'}
+
34%|███▍ | 540/1570 [30:05<36:53, 2.15s/it]
34%|███▍ | 541/1570 [30:07<36:49, 2.15s/it]
35%|███▍ | 542/1570 [30:10<36:46, 2.15s/it]
35%|███▍ | 543/1570 [30:12<36:45, 2.15s/it]
35%|███▍ | 544/1570 [30:14<36:41, 2.15s/it]
35%|███▍ | 545/1570 [30:16<36:36, 2.14s/it]
35%|███▍ | 546/1570 [30:18<36:35, 2.14s/it]
35%|███▍ | 547/1570 [30:20<36:31, 2.14s/it]
35%|███▍ | 548/1570 [30:22<36:31, 2.14s/it]
35%|███▍ | 549/1570 [30:25<36:32, 2.15s/it]
35%|███▌ | 550/1570 [30:27<36:29, 2.15s/it]
35%|███▌ | 551/1570 [30:29<36:31, 2.15s/it]
35%|███▌ | 552/1570 [30:31<36:24, 2.15s/it]
35%|███▌ | 553/1570 [30:33<36:25, 2.15s/it]
35%|███▌ | 554/1570 [30:35<36:22, 2.15s/it]
35%|███▌ | 555/1570 [30:37<36:22, 2.15s/it]
35%|███▌ | 556/1570 [30:40<36:18, 2.15s/it]
35%|███▌ | 557/1570 [30:42<36:12, 2.14s/it]
36%|███▌ | 558/1570 [30:44<36:13, 2.15s/it]
36%|███▌ | 559/1570 [30:46<36:09, 2.15s/it]
36%|███▌ | 560/1570 [30:48<36:07, 2.15s/it]
{'loss': '0.004458', 'grad_norm': '0.2757', 'learning_rate': '7.188e-05', 'epoch': '1.785'}
+
36%|███▌ | 560/1570 [30:48<36:07, 2.15s/it]
36%|███▌ | 561/1570 [30:50<36:05, 2.15s/it]
36%|███▌ | 562/1570 [30:53<36:05, 2.15s/it]
36%|███▌ | 563/1570 [30:55<36:04, 2.15s/it]
36%|███▌ | 564/1570 [30:57<36:04, 2.15s/it]
36%|███▌ | 565/1570 [30:59<36:01, 2.15s/it]
36%|███▌ | 566/1570 [31:01<35:57, 2.15s/it]
36%|███▌ | 567/1570 [31:03<35:58, 2.15s/it]
36%|███▌ | 568/1570 [31:05<35:55, 2.15s/it]
36%|███▌ | 569/1570 [31:08<35:52, 2.15s/it]
36%|███▋ | 570/1570 [31:10<35:47, 2.15s/it]
36%|███▋ | 571/1570 [31:12<35:43, 2.15s/it]
36%|███▋ | 572/1570 [31:14<35:45, 2.15s/it]
36%|███▋ | 573/1570 [31:16<35:44, 2.15s/it]
37%|███▋ | 574/1570 [31:18<35:42, 2.15s/it]
37%|███▋ | 575/1570 [31:20<35:37, 2.15s/it]
37%|███▋ | 576/1570 [31:23<35:36, 2.15s/it]
37%|███▋ | 577/1570 [31:25<35:36, 2.15s/it]
37%|███▋ | 578/1570 [31:27<35:35, 2.15s/it]
37%|███▋ | 579/1570 [31:29<35:31, 2.15s/it]
37%|███▋ | 580/1570 [31:31<35:32, 2.15s/it]
{'loss': '0.01123', 'grad_norm': '1.083', 'learning_rate': '7.006e-05', 'epoch': '1.849'}
+
37%|███▋ | 580/1570 [31:31<35:32, 2.15s/it]
37%|███▋ | 581/1570 [31:33<35:30, 2.15s/it]
37%|███▋ | 582/1570 [31:36<35:24, 2.15s/it]
37%|███▋ | 583/1570 [31:38<35:21, 2.15s/it]
37%|███▋ | 584/1570 [31:40<35:16, 2.15s/it]
37%|███▋ | 585/1570 [31:42<35:17, 2.15s/it]
37%|███▋ | 586/1570 [31:44<35:15, 2.15s/it]
37%|███▋ | 587/1570 [31:46<35:09, 2.15s/it]
37%|███▋ | 588/1570 [31:48<35:15, 2.15s/it]
38%|███▊ | 589/1570 [31:51<35:11, 2.15s/it]
38%|███▊ | 590/1570 [31:53<35:11, 2.15s/it]
38%|███▊ | 591/1570 [31:55<35:07, 2.15s/it]
38%|███▊ | 592/1570 [31:57<35:00, 2.15s/it]
38%|███▊ | 593/1570 [31:59<34:57, 2.15s/it]
38%|███▊ | 594/1570 [32:01<34:54, 2.15s/it]
38%|███▊ | 595/1570 [32:03<34:51, 2.14s/it]
38%|███▊ | 596/1570 [32:06<34:48, 2.14s/it]
38%|███▊ | 597/1570 [32:08<34:48, 2.15s/it]
38%|███▊ | 598/1570 [32:10<34:44, 2.14s/it]
38%|███▊ | 599/1570 [32:12<34:36, 2.14s/it]
38%|███▊ | 600/1570 [32:14<34:37, 2.14s/it]
{'loss': '0.006709', 'grad_norm': '0.8778', 'learning_rate': '6.821e-05', 'epoch': '1.913'}
+
38%|███▊ | 600/1570 [32:14<34:37, 2.14s/it]
38%|███▊ | 601/1570 [32:16<34:34, 2.14s/it]
38%|███▊ | 602/1570 [32:18<34:33, 2.14s/it]
38%|███▊ | 603/1570 [32:21<34:29, 2.14s/it]
38%|███▊ | 604/1570 [32:23<34:25, 2.14s/it]
39%|███▊ | 605/1570 [32:25<34:24, 2.14s/it]
39%|███▊ | 606/1570 [32:27<34:21, 2.14s/it]
39%|███▊ | 607/1570 [32:29<34:20, 2.14s/it]
39%|███▊ | 608/1570 [32:31<34:13, 2.13s/it]
39%|███▉ | 609/1570 [32:33<34:12, 2.14s/it]
39%|███▉ | 610/1570 [32:36<34:13, 2.14s/it]
39%|███▉ | 611/1570 [32:38<34:10, 2.14s/it]
39%|███▉ | 612/1570 [32:40<34:12, 2.14s/it]
39%|███▉ | 613/1570 [32:42<34:07, 2.14s/it]
39%|███▉ | 614/1570 [32:44<34:06, 2.14s/it]
39%|███▉ | 615/1570 [32:46<34:05, 2.14s/it]
39%|███▉ | 616/1570 [32:48<34:01, 2.14s/it]
39%|███▉ | 617/1570 [32:51<33:59, 2.14s/it]
39%|███▉ | 618/1570 [32:53<33:56, 2.14s/it]
39%|███▉ | 619/1570 [32:55<33:54, 2.14s/it]
39%|███▉ | 620/1570 [32:57<33:48, 2.14s/it]
{'loss': '0.004196', 'grad_norm': '0.1681', 'learning_rate': '6.634e-05', 'epoch': '1.976'}
+
39%|███▉ | 620/1570 [32:57<33:48, 2.14s/it]
40%|███▉ | 621/1570 [32:59<33:47, 2.14s/it]
40%|███▉ | 622/1570 [33:01<33:49, 2.14s/it]
40%|███▉ | 623/1570 [33:03<33:52, 2.15s/it]
40%|███▉ | 624/1570 [33:06<33:49, 2.15s/it]
40%|███▉ | 625/1570 [33:08<33:43, 2.14s/it]
40%|███▉ | 626/1570 [33:10<33:47, 2.15s/it]
40%|███▉ | 627/1570 [33:12<33:45, 2.15s/it]
40%|████ | 628/1570 [33:13<27:53, 1.78s/it][INFO|trainer.py:2668] 2026-08-22 08:46:18,636 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 08:46:18,636 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 08:46:18,636 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/sft/checkpoint-628
+[INFO|configuration_utils.py:780] 2026-08-22 08:56:01,671 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 08:56:01,672 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 08:56:01,708 >> chat template saved in results/qwen3/sft/checkpoint-628/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 08:56:01,708 >> tokenizer config file saved in results/qwen3/sft/checkpoint-628/tokenizer_config.json
+
40%|████ | 629/1570 [42:59<46:16:33, 177.04s/it]
40%|████ | 630/1570 [43:01<32:31:36, 124.57s/it]
40%|████ | 631/1570 [43:03<22:54:48, 87.85s/it]
40%|████ | 632/1570 [43:05<16:11:27, 62.14s/it]
40%|████ | 633/1570 [43:07<11:29:20, 44.14s/it]
40%|████ | 634/1570 [43:10<8:12:05, 31.54s/it]
40%|████ | 635/1570 [43:12<5:54:07, 22.72s/it]
41%|████ | 636/1570 [43:14<4:17:32, 16.54s/it]
41%|████ | 637/1570 [43:16<3:10:08, 12.23s/it]
41%|████ | 638/1570 [43:18<2:22:53, 9.20s/it]
41%|████ | 639/1570 [43:20<1:49:53, 7.08s/it]
41%|████ | 640/1570 [43:22<1:26:42, 5.59s/it]
{'loss': '0.00328', 'grad_norm': '0.2708', 'learning_rate': '6.444e-05', 'epoch': '2.038'}
+
41%|████ | 640/1570 [43:22<1:26:42, 5.59s/it]
41%|████ | 641/1570 [43:25<1:10:35, 4.56s/it]
41%|████ | 642/1570 [43:27<59:16, 3.83s/it]
41%|████ | 643/1570 [43:29<51:20, 3.32s/it]
41%|████ | 644/1570 [43:31<45:52, 2.97s/it]
41%|████ | 645/1570 [43:33<41:56, 2.72s/it]
41%|████ | 646/1570 [43:35<39:13, 2.55s/it]
41%|████ | 647/1570 [43:37<37:20, 2.43s/it]
41%|████▏ | 648/1570 [43:40<36:01, 2.34s/it]
41%|████▏ | 649/1570 [43:42<35:01, 2.28s/it]
41%|████▏ | 650/1570 [43:44<34:20, 2.24s/it]
41%|████▏ | 651/1570 [43:46<33:50, 2.21s/it]
42%|████▏ | 652/1570 [43:48<33:32, 2.19s/it]
42%|████▏ | 653/1570 [43:50<33:14, 2.17s/it]
42%|████▏ | 654/1570 [43:52<33:08, 2.17s/it]
42%|████▏ | 655/1570 [43:55<32:59, 2.16s/it]
42%|████▏ | 656/1570 [43:57<32:50, 2.16s/it]
42%|████▏ | 657/1570 [43:59<32:43, 2.15s/it]
42%|████▏ | 658/1570 [44:01<32:40, 2.15s/it]
42%|████▏ | 659/1570 [44:03<32:37, 2.15s/it]
42%|████▏ | 660/1570 [44:05<32:34, 2.15s/it]
{'loss': '0.002625', 'grad_norm': '0.1907', 'learning_rate': '6.251e-05', 'epoch': '2.102'}
+
42%|████▏ | 660/1570 [44:05<32:34, 2.15s/it]
42%|████▏ | 661/1570 [44:07<32:29, 2.14s/it]
42%|████▏ | 662/1570 [44:10<32:31, 2.15s/it]
42%|████▏ | 663/1570 [44:12<32:28, 2.15s/it]
42%|████▏ | 664/1570 [44:14<32:24, 2.15s/it]
42%|████▏ | 665/1570 [44:16<32:22, 2.15s/it]
42%|████▏ | 666/1570 [44:18<32:17, 2.14s/it]
42%|████▏ | 667/1570 [44:20<32:14, 2.14s/it]
43%|████▎ | 668/1570 [44:22<32:14, 2.14s/it]
43%|████▎ | 669/1570 [44:25<32:12, 2.14s/it]
43%|████▎ | 670/1570 [44:27<32:10, 2.15s/it]
43%|████▎ | 671/1570 [44:29<32:07, 2.14s/it]
43%|████▎ | 672/1570 [44:31<32:05, 2.14s/it]
43%|████▎ | 673/1570 [44:33<32:03, 2.14s/it]
43%|████▎ | 674/1570 [44:35<31:58, 2.14s/it]
43%|████▎ | 675/1570 [44:37<31:53, 2.14s/it]
43%|████▎ | 676/1570 [44:40<31:53, 2.14s/it]
43%|████▎ | 677/1570 [44:42<31:53, 2.14s/it]
43%|████▎ | 678/1570 [44:44<31:57, 2.15s/it]
43%|████▎ | 679/1570 [44:46<31:52, 2.15s/it]
43%|████▎ | 680/1570 [44:48<31:53, 2.15s/it]
{'loss': '0.001672', 'grad_norm': '0.0759', 'learning_rate': '6.057e-05', 'epoch': '2.166'}
+
43%|████▎ | 680/1570 [44:48<31:53, 2.15s/it]
43%|████▎ | 681/1570 [44:50<31:47, 2.15s/it]
43%|████▎ | 682/1570 [44:52<31:42, 2.14s/it]
44%|████▎ | 683/1570 [44:55<31:43, 2.15s/it]
44%|████▎ | 684/1570 [44:57<31:40, 2.15s/it]
44%|████▎ | 685/1570 [44:59<31:37, 2.14s/it]
44%|████▎ | 686/1570 [45:01<31:38, 2.15s/it]
44%|████▍ | 687/1570 [45:03<31:37, 2.15s/it]
44%|████▍ | 688/1570 [45:05<31:35, 2.15s/it]
44%|████▍ | 689/1570 [45:08<31:29, 2.15s/it]
44%|████▍ | 690/1570 [45:10<31:28, 2.15s/it]
44%|████▍ | 691/1570 [45:12<31:26, 2.15s/it]
44%|████▍ | 692/1570 [45:14<31:23, 2.14s/it]
44%|████▍ | 693/1570 [45:16<31:20, 2.14s/it]
44%|████▍ | 694/1570 [45:18<31:20, 2.15s/it]
44%|████▍ | 695/1570 [45:20<31:16, 2.14s/it]
44%|████▍ | 696/1570 [45:23<31:14, 2.14s/it]
44%|████▍ | 697/1570 [45:25<31:15, 2.15s/it]
44%|████▍ | 698/1570 [45:27<31:12, 2.15s/it]
45%|████▍ | 699/1570 [45:29<31:10, 2.15s/it]
45%|████▍ | 700/1570 [45:31<31:09, 2.15s/it]
{'loss': '0.001929', 'grad_norm': '0.8344', 'learning_rate': '5.86e-05', 'epoch': '2.23'}
+
45%|████▍ | 700/1570 [45:31<31:09, 2.15s/it]
45%|████▍ | 701/1570 [45:33<31:05, 2.15s/it]
45%|████▍ | 702/1570 [45:35<31:01, 2.14s/it]
45%|████▍ | 703/1570 [45:38<30:58, 2.14s/it]
45%|████▍ | 704/1570 [45:40<30:52, 2.14s/it]
45%|████▍ | 705/1570 [45:42<30:49, 2.14s/it]
45%|████▍ | 706/1570 [45:44<30:47, 2.14s/it]
45%|████▌ | 707/1570 [45:46<30:44, 2.14s/it]
45%|████▌ | 708/1570 [45:48<30:43, 2.14s/it]
45%|████▌ | 709/1570 [45:50<30:39, 2.14s/it]
45%|████▌ | 710/1570 [45:53<30:41, 2.14s/it]
45%|████▌ | 711/1570 [45:55<30:39, 2.14s/it]
45%|████▌ | 712/1570 [45:57<30:38, 2.14s/it]
45%|████▌ | 713/1570 [45:59<30:35, 2.14s/it]
45%|████▌ | 714/1570 [46:01<30:34, 2.14s/it]
46%|████▌ | 715/1570 [46:03<30:33, 2.14s/it]
46%|████▌ | 716/1570 [46:05<30:31, 2.15s/it]
46%|████▌ | 717/1570 [46:08<30:30, 2.15s/it]
46%|████▌ | 718/1570 [46:10<30:23, 2.14s/it]
46%|████▌ | 719/1570 [46:12<30:19, 2.14s/it]
46%|████▌ | 720/1570 [46:14<30:18, 2.14s/it]
{'loss': '0.002199', 'grad_norm': '0.08497', 'learning_rate': '5.663e-05', 'epoch': '2.294'}
+
46%|████▌ | 720/1570 [46:14<30:18, 2.14s/it]
46%|████▌ | 721/1570 [46:16<30:18, 2.14s/it]
46%|████▌ | 722/1570 [46:18<30:18, 2.14s/it]
46%|████▌ | 723/1570 [46:20<30:16, 2.14s/it]
46%|████▌ | 724/1570 [46:23<30:10, 2.14s/it]
46%|████▌ | 725/1570 [46:25<30:06, 2.14s/it]
46%|████▌ | 726/1570 [46:27<30:06, 2.14s/it]
46%|████▋ | 727/1570 [46:29<30:04, 2.14s/it]
46%|████▋ | 728/1570 [46:31<30:00, 2.14s/it]
46%|████▋ | 729/1570 [46:33<29:59, 2.14s/it]
46%|████▋ | 730/1570 [46:35<30:00, 2.14s/it]
47%|████▋ | 731/1570 [46:37<29:56, 2.14s/it]
47%|████▋ | 732/1570 [46:40<29:55, 2.14s/it]
47%|████▋ | 733/1570 [46:42<29:53, 2.14s/it]
47%|████▋ | 734/1570 [46:44<29:52, 2.14s/it]
47%|████▋ | 735/1570 [46:46<29:50, 2.14s/it]
47%|████▋ | 736/1570 [46:48<29:47, 2.14s/it]
47%|████▋ | 737/1570 [46:50<29:48, 2.15s/it]
47%|████▋ | 738/1570 [46:53<29:45, 2.15s/it]
47%|████▋ | 739/1570 [46:55<29:38, 2.14s/it]
47%|████▋ | 740/1570 [46:57<29:28, 2.13s/it]
{'loss': '0.005482', 'grad_norm': '0.1171', 'learning_rate': '5.464e-05', 'epoch': '2.357'}
+
47%|████▋ | 740/1570 [46:57<29:28, 2.13s/it]
47%|████▋ | 741/1570 [46:59<29:25, 2.13s/it]
47%|████▋ | 742/1570 [47:01<29:21, 2.13s/it]
47%|████▋ | 743/1570 [47:03<29:17, 2.12s/it]
47%|████▋ | 744/1570 [47:05<29:13, 2.12s/it]
47%|████▋ | 745/1570 [47:07<29:13, 2.13s/it]
48%|████▊ | 746/1570 [47:10<29:16, 2.13s/it]
48%|████▊ | 747/1570 [47:12<29:14, 2.13s/it]
48%|████▊ | 748/1570 [47:14<29:16, 2.14s/it]
48%|████▊ | 749/1570 [47:16<29:14, 2.14s/it]
48%|████▊ | 750/1570 [47:18<29:10, 2.13s/it]
48%|████▊ | 751/1570 [47:20<29:03, 2.13s/it]
48%|████▊ | 752/1570 [47:22<29:03, 2.13s/it]
48%|████▊ | 753/1570 [47:24<29:01, 2.13s/it]
48%|████▊ | 754/1570 [47:27<28:59, 2.13s/it]
48%|████▊ | 755/1570 [47:29<28:57, 2.13s/it]
48%|████▊ | 756/1570 [47:31<28:51, 2.13s/it]
48%|████▊ | 757/1570 [47:33<28:51, 2.13s/it]
48%|████▊ | 758/1570 [47:35<28:46, 2.13s/it]
48%|████▊ | 759/1570 [47:37<28:43, 2.13s/it]
48%|████▊ | 760/1570 [47:39<28:41, 2.12s/it]
{'loss': '0.00296', 'grad_norm': '0.02737', 'learning_rate': '5.265e-05', 'epoch': '2.421'}
+
48%|████▊ | 760/1570 [47:39<28:41, 2.12s/it]
48%|████▊ | 761/1570 [47:41<28:36, 2.12s/it]
49%|████▊ | 762/1570 [47:44<28:39, 2.13s/it]
49%|████▊ | 763/1570 [47:46<28:37, 2.13s/it]
49%|████▊ | 764/1570 [47:48<28:36, 2.13s/it]
49%|████▊ | 765/1570 [47:50<28:33, 2.13s/it]
49%|████▉ | 766/1570 [47:52<28:30, 2.13s/it]
49%|████▉ | 767/1570 [47:54<28:28, 2.13s/it]
49%|████▉ | 768/1570 [47:56<28:27, 2.13s/it]
49%|████▉ | 769/1570 [47:58<28:26, 2.13s/it]
49%|████▉ | 770/1570 [48:01<28:26, 2.13s/it]
49%|████▉ | 771/1570 [48:03<28:24, 2.13s/it]
49%|████▉ | 772/1570 [48:05<28:24, 2.14s/it]
49%|████▉ | 773/1570 [48:07<28:22, 2.14s/it]
49%|████▉ | 774/1570 [48:09<28:22, 2.14s/it]
49%|████▉ | 775/1570 [48:11<28:18, 2.14s/it]
49%|████▉ | 776/1570 [48:13<28:20, 2.14s/it]
49%|████▉ | 777/1570 [48:16<28:20, 2.14s/it]
50%|████▉ | 778/1570 [48:18<28:16, 2.14s/it]
50%|████▉ | 779/1570 [48:20<28:16, 2.14s/it]
50%|████▉ | 780/1570 [48:22<28:16, 2.15s/it]
{'loss': '0.001288', 'grad_norm': '0.03141', 'learning_rate': '5.065e-05', 'epoch': '2.485'}
+
50%|████▉ | 780/1570 [48:22<28:16, 2.15s/it]
50%|████▉ | 781/1570 [48:24<28:19, 2.15s/it]
50%|████▉ | 782/1570 [48:26<28:12, 2.15s/it]
50%|████▉ | 783/1570 [48:29<28:10, 2.15s/it]
50%|████▉ | 784/1570 [48:31<28:08, 2.15s/it]
50%|█████ | 785/1570 [48:33<28:04, 2.15s/it]
50%|█████ | 786/1570 [48:35<28:03, 2.15s/it]
50%|█████ | 787/1570 [48:37<28:00, 2.15s/it]
50%|█████ | 788/1570 [48:39<27:59, 2.15s/it]
50%|█████ | 789/1570 [48:41<27:59, 2.15s/it]
50%|█████ | 790/1570 [48:44<27:56, 2.15s/it]
50%|█████ | 791/1570 [48:46<27:50, 2.14s/it]
50%|█████ | 792/1570 [48:48<27:47, 2.14s/it]
51%|█████ | 793/1570 [48:50<27:47, 2.15s/it]
51%|█████ | 794/1570 [48:52<27:48, 2.15s/it]
51%|█████ | 795/1570 [48:54<27:43, 2.15s/it]
51%|█████ | 796/1570 [48:56<27:42, 2.15s/it]
51%|█████ | 797/1570 [48:59<27:40, 2.15s/it]
51%|█████ | 798/1570 [49:01<27:38, 2.15s/it]
51%|█████ | 799/1570 [49:03<27:32, 2.14s/it]
51%|█████ | 800/1570 [49:05<27:29, 2.14s/it]
{'loss': '0.004979', 'grad_norm': '0.1769', 'learning_rate': '4.865e-05', 'epoch': '2.549'}
+
51%|█████ | 800/1570 [49:05<27:29, 2.14s/it]
51%|█████ | 801/1570 [49:07<27:24, 2.14s/it]
51%|█████ | 802/1570 [49:09<27:21, 2.14s/it]
51%|█████ | 803/1570 [49:11<27:19, 2.14s/it]
51%|█████ | 804/1570 [49:14<27:21, 2.14s/it]
51%|█████▏ | 805/1570 [49:16<27:18, 2.14s/it]
51%|█████▏ | 806/1570 [49:18<27:18, 2.14s/it]
51%|█████▏ | 807/1570 [49:20<27:12, 2.14s/it]
51%|█████▏ | 808/1570 [49:22<27:06, 2.13s/it]
52%|█████▏ | 809/1570 [49:24<27:04, 2.14s/it]
52%|█████▏ | 810/1570 [49:26<27:06, 2.14s/it]
52%|█████▏ | 811/1570 [49:29<27:03, 2.14s/it]
52%|█████▏ | 812/1570 [49:31<27:03, 2.14s/it]
52%|█████▏ | 813/1570 [49:33<26:59, 2.14s/it]
52%|█████▏ | 814/1570 [49:35<26:57, 2.14s/it]
52%|█████▏ | 815/1570 [49:37<26:56, 2.14s/it]
52%|█████▏ | 816/1570 [49:39<26:53, 2.14s/it]
52%|█████▏ | 817/1570 [49:41<26:54, 2.14s/it]
52%|█████▏ | 818/1570 [49:44<26:49, 2.14s/it]
52%|█████▏ | 819/1570 [49:46<26:49, 2.14s/it]
52%|█████▏ | 820/1570 [49:48<26:46, 2.14s/it]
{'loss': '0.002898', 'grad_norm': '0.5066', 'learning_rate': '4.665e-05', 'epoch': '2.613'}
+
52%|█████▏ | 820/1570 [49:48<26:46, 2.14s/it]
52%|█████▏ | 821/1570 [49:50<26:46, 2.15s/it]
52%|█████▏ | 822/1570 [49:52<26:46, 2.15s/it]
52%|█████▏ | 823/1570 [49:54<26:44, 2.15s/it]
52%|█████▏ | 824/1570 [49:56<26:45, 2.15s/it]
53%|█████▎ | 825/1570 [49:59<26:42, 2.15s/it]
53%|█████▎ | 826/1570 [50:01<26:36, 2.15s/it]
53%|█████▎ | 827/1570 [50:03<26:32, 2.14s/it]
53%|█████▎ | 828/1570 [50:05<26:32, 2.15s/it]
53%|█████▎ | 829/1570 [50:07<26:27, 2.14s/it]
53%|█████▎ | 830/1570 [50:09<26:24, 2.14s/it]
53%|█████▎ | 831/1570 [50:11<26:22, 2.14s/it]
53%|█████▎ | 832/1570 [50:14<26:23, 2.15s/it]
53%|█████▎ | 833/1570 [50:16<26:17, 2.14s/it]
53%|█████▎ | 834/1570 [50:18<26:14, 2.14s/it]
53%|█████▎ | 835/1570 [50:20<26:11, 2.14s/it]
53%|█████▎ | 836/1570 [50:22<26:10, 2.14s/it]
53%|█████▎ | 837/1570 [50:24<26:07, 2.14s/it]
53%|█████▎ | 838/1570 [50:26<26:01, 2.13s/it]
53%|█████▎ | 839/1570 [50:28<25:57, 2.13s/it]
54%|█████▎ | 840/1570 [50:31<26:00, 2.14s/it]
{'loss': '0.002082', 'grad_norm': '0.1322', 'learning_rate': '4.466e-05', 'epoch': '2.677'}
+
54%|█████▎ | 840/1570 [50:31<26:00, 2.14s/it]
54%|█████▎ | 841/1570 [50:33<25:57, 2.14s/it]
54%|█████▎ | 842/1570 [50:35<25:58, 2.14s/it]
54%|█████▎ | 843/1570 [50:37<25:57, 2.14s/it]
54%|█████▍ | 844/1570 [50:39<25:51, 2.14s/it]
54%|█████▍ | 845/1570 [50:41<25:49, 2.14s/it]
54%|█████▍ | 846/1570 [50:43<25:47, 2.14s/it]
54%|█████▍ | 847/1570 [50:46<25:47, 2.14s/it]
54%|█████▍ | 848/1570 [50:48<25:44, 2.14s/it]
54%|█████▍ | 849/1570 [50:50<25:44, 2.14s/it]
54%|█████▍ | 850/1570 [50:52<25:39, 2.14s/it]
54%|█████▍ | 851/1570 [50:54<25:35, 2.14s/it]
54%|█████▍ | 852/1570 [50:56<25:31, 2.13s/it]
54%|█████▍ | 853/1570 [50:58<25:32, 2.14s/it]
54%|█████▍ | 854/1570 [51:01<25:31, 2.14s/it]
54%|█████▍ | 855/1570 [51:03<25:28, 2.14s/it]
55%|█████▍ | 856/1570 [51:05<25:27, 2.14s/it]
55%|█████▍ | 857/1570 [51:07<25:24, 2.14s/it]
55%|█████▍ | 858/1570 [51:09<25:24, 2.14s/it]
55%|█████▍ | 859/1570 [51:11<25:23, 2.14s/it]
55%|█████▍ | 860/1570 [51:13<25:23, 2.15s/it]
{'loss': '0.004749', 'grad_norm': '0.08058', 'learning_rate': '4.268e-05', 'epoch': '2.74'}
+
55%|█████▍ | 860/1570 [51:13<25:23, 2.15s/it]
55%|█████▍ | 861/1570 [51:16<25:22, 2.15s/it]
55%|█████▍ | 862/1570 [51:18<25:21, 2.15s/it]
55%|█████▍ | 863/1570 [51:20<25:19, 2.15s/it]
55%|█████▌ | 864/1570 [51:22<25:15, 2.15s/it]
55%|█████▌ | 865/1570 [51:24<25:15, 2.15s/it]
55%|█████▌ | 866/1570 [51:26<25:15, 2.15s/it]
55%|█████▌ | 867/1570 [51:29<25:12, 2.15s/it]
55%|█████▌ | 868/1570 [51:31<25:11, 2.15s/it]
55%|█████▌ | 869/1570 [51:33<25:07, 2.15s/it]
55%|█████▌ | 870/1570 [51:35<25:06, 2.15s/it]
55%|█████▌ | 871/1570 [51:37<25:02, 2.15s/it]
56%|█████▌ | 872/1570 [51:39<25:01, 2.15s/it]
56%|█████▌ | 873/1570 [51:41<24:59, 2.15s/it]
56%|█████▌ | 874/1570 [51:44<24:58, 2.15s/it]
56%|█████▌ | 875/1570 [51:46<24:56, 2.15s/it]
56%|█████▌ | 876/1570 [51:48<24:54, 2.15s/it]
56%|█████▌ | 877/1570 [51:50<24:50, 2.15s/it]
56%|█████▌ | 878/1570 [51:52<24:48, 2.15s/it]
56%|█████▌ | 879/1570 [51:54<24:46, 2.15s/it]
56%|█████▌ | 880/1570 [51:56<24:40, 2.15s/it]
{'loss': '0.00141', 'grad_norm': '0.2788', 'learning_rate': '4.071e-05', 'epoch': '2.804'}
+
56%|█████▌ | 880/1570 [51:56<24:40, 2.15s/it]
56%|█████▌ | 881/1570 [51:59<24:29, 2.13s/it]
56%|█████▌ | 882/1570 [52:01<24:25, 2.13s/it]
56%|█████▌ | 883/1570 [52:03<24:19, 2.12s/it]
56%|█████▋ | 884/1570 [52:05<24:19, 2.13s/it]
56%|█████▋ | 885/1570 [52:07<24:16, 2.13s/it]
56%|█████▋ | 886/1570 [52:09<24:15, 2.13s/it]
56%|█████▋ | 887/1570 [52:11<24:15, 2.13s/it]
57%|█████▋ | 888/1570 [52:13<24:15, 2.13s/it]
57%|█████▋ | 889/1570 [52:16<24:13, 2.13s/it]
57%|█████▋ | 890/1570 [52:18<24:10, 2.13s/it]
57%|█████▋ | 891/1570 [52:20<24:08, 2.13s/it]
57%|█████▋ | 892/1570 [52:22<24:07, 2.14s/it]
57%|█████▋ | 893/1570 [52:24<24:07, 2.14s/it]
57%|█████▋ | 894/1570 [52:26<24:06, 2.14s/it]
57%|█████▋ | 895/1570 [52:28<24:03, 2.14s/it]
57%|█████▋ | 896/1570 [52:31<23:58, 2.13s/it]
57%|█████▋ | 897/1570 [52:33<23:54, 2.13s/it]
57%|█████▋ | 898/1570 [52:35<23:54, 2.14s/it]
57%|█████▋ | 899/1570 [52:37<23:52, 2.13s/it]
57%|█████▋ | 900/1570 [52:39<23:48, 2.13s/it]
{'loss': '0.001781', 'grad_norm': '0.6502', 'learning_rate': '3.875e-05', 'epoch': '2.868'}
+
57%|█████▋ | 900/1570 [52:39<23:48, 2.13s/it]
57%|█████▋ | 901/1570 [52:41<23:48, 2.13s/it]
57%|█████▋ | 902/1570 [52:43<23:48, 2.14s/it]
58%|█████▊ | 903/1570 [52:46<23:46, 2.14s/it]
58%|█████▊ | 904/1570 [52:48<23:46, 2.14s/it]
58%|█████▊ | 905/1570 [52:50<23:44, 2.14s/it]
58%|█████▊ | 906/1570 [52:52<23:40, 2.14s/it]
58%|█████▊ | 907/1570 [52:54<23:38, 2.14s/it]
58%|█████▊ | 908/1570 [52:56<23:35, 2.14s/it]
58%|█████▊ | 909/1570 [52:58<23:34, 2.14s/it]
58%|█████▊ | 910/1570 [53:00<23:33, 2.14s/it]
58%|█████▊ | 911/1570 [53:03<23:31, 2.14s/it]
58%|█████▊ | 912/1570 [53:05<23:27, 2.14s/it]
58%|█████▊ | 913/1570 [53:07<23:25, 2.14s/it]
58%|█████▊ | 914/1570 [53:09<23:22, 2.14s/it]
58%|█████▊ | 915/1570 [53:11<23:19, 2.14s/it]
58%|█████▊ | 916/1570 [53:13<23:22, 2.15s/it]
58%|█████▊ | 917/1570 [53:15<23:19, 2.14s/it]
58%|█████▊ | 918/1570 [53:18<23:19, 2.15s/it]
59%|█████▊ | 919/1570 [53:20<23:16, 2.14s/it]
59%|█████▊ | 920/1570 [53:22<23:13, 2.14s/it]
{'loss': '0.002541', 'grad_norm': '0.007601', 'learning_rate': '3.682e-05', 'epoch': '2.932'}
+
59%|█████▊ | 920/1570 [53:22<23:13, 2.14s/it]
59%|█████▊ | 921/1570 [53:24<23:10, 2.14s/it]
59%|█████▊ | 922/1570 [53:26<23:08, 2.14s/it]
59%|█████▉ | 923/1570 [53:28<23:07, 2.14s/it]
59%|█████▉ | 924/1570 [53:30<23:04, 2.14s/it]
59%|█████▉ | 925/1570 [53:33<23:02, 2.14s/it]
59%|█████▉ | 926/1570 [53:35<23:00, 2.14s/it]
59%|█████▉ | 927/1570 [53:37<22:58, 2.14s/it]
59%|█████▉ | 928/1570 [53:39<22:57, 2.15s/it]
59%|█████▉ | 929/1570 [53:41<22:54, 2.14s/it]
59%|█████▉ | 930/1570 [53:43<22:51, 2.14s/it]
59%|█████▉ | 931/1570 [53:46<22:49, 2.14s/it]
59%|█████▉ | 932/1570 [53:48<22:47, 2.14s/it]
59%|█████▉ | 933/1570 [53:50<22:45, 2.14s/it]
59%|█████▉ | 934/1570 [53:52<22:43, 2.14s/it]
60%|█████▉ | 935/1570 [53:54<22:41, 2.14s/it]
60%|█████▉ | 936/1570 [53:56<22:37, 2.14s/it]
60%|█████▉ | 937/1570 [53:58<22:36, 2.14s/it]
60%|█████▉ | 938/1570 [54:00<22:33, 2.14s/it]
60%|█████▉ | 939/1570 [54:03<22:32, 2.14s/it]
60%|█████▉ | 940/1570 [54:05<22:32, 2.15s/it]
{'loss': '0.001643', 'grad_norm': '0.1264', 'learning_rate': '3.49e-05', 'epoch': '2.996'}
+
60%|█████▉ | 940/1570 [54:05<22:32, 2.15s/it]
60%|█████▉ | 941/1570 [54:07<22:29, 2.15s/it]
60%|██████ | 942/1570 [54:08<18:36, 1.78s/it][INFO|trainer.py:2668] 2026-08-22 09:07:13,631 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 09:07:13,632 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 09:07:13,632 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/sft/checkpoint-942
+[INFO|configuration_utils.py:780] 2026-08-22 09:16:58,589 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 09:16:58,592 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 09:16:58,627 >> chat template saved in results/qwen3/sft/checkpoint-942/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 09:16:58,627 >> tokenizer config file saved in results/qwen3/sft/checkpoint-942/tokenizer_config.json
+
60%|██████ | 943/1570 [1:03:56<30:56:05, 177.62s/it]
60%|██████ | 944/1570 [1:03:58<21:43:59, 124.98s/it]
60%|██████ | 945/1570 [1:04:00<15:18:01, 88.13s/it]
60%|██████ | 946/1570 [1:04:02<10:48:17, 62.34s/it]
60%|██████ | 947/1570 [1:04:04<7:39:48, 44.28s/it]
60%|██████ | 948/1570 [1:04:07<5:28:01, 31.64s/it]
60%|██████ | 949/1570 [1:04:09<3:55:57, 22.80s/it]
61%|██████ | 950/1570 [1:04:11<2:51:36, 16.61s/it]
61%|██████ | 951/1570 [1:04:13<2:06:37, 12.27s/it]
61%|██████ | 952/1570 [1:04:15<1:35:09, 9.24s/it]
61%|██████ | 953/1570 [1:04:17<1:13:06, 7.11s/it]
61%|██████ | 954/1570 [1:04:19<57:41, 5.62s/it]
61%|██████ | 955/1570 [1:04:22<46:52, 4.57s/it]
61%|██████ | 956/1570 [1:04:24<39:21, 3.85s/it]
61%|██████ | 957/1570 [1:04:26<34:05, 3.34s/it]
61%|██████ | 958/1570 [1:04:28<30:27, 2.99s/it]
61%|██████ | 959/1570 [1:04:30<27:51, 2.74s/it]
61%|██████ | 960/1570 [1:04:32<25:58, 2.56s/it]
{'loss': '0.001967', 'grad_norm': '0.06882', 'learning_rate': '3.301e-05', 'epoch': '3.057'}
+
61%|██████ | 960/1570 [1:04:32<25:58, 2.56s/it]
61%|██████ | 961/1570 [1:04:34<24:41, 2.43s/it]
61%|██████▏ | 962/1570 [1:04:37<23:47, 2.35s/it]
61%|██████▏ | 963/1570 [1:04:39<23:08, 2.29s/it]
61%|██████▏ | 964/1570 [1:04:41<22:41, 2.25s/it]
61%|██████▏ | 965/1570 [1:04:43<22:21, 2.22s/it]
62%|██████▏ | 966/1570 [1:04:45<22:04, 2.19s/it]
62%|██████▏ | 967/1570 [1:04:47<21:55, 2.18s/it]
62%|██████▏ | 968/1570 [1:04:50<21:49, 2.18s/it]
62%|██████▏ | 969/1570 [1:04:52<21:41, 2.17s/it]
62%|██████▏ | 970/1570 [1:04:54<21:38, 2.16s/it]
62%|██████▏ | 971/1570 [1:04:56<21:33, 2.16s/it]
62%|██████▏ | 972/1570 [1:04:58<21:34, 2.16s/it]
62%|██████▏ | 973/1570 [1:05:00<21:29, 2.16s/it]
62%|██████▏ | 974/1570 [1:05:02<21:26, 2.16s/it]
62%|██████▏ | 975/1570 [1:05:05<21:24, 2.16s/it]
62%|██████▏ | 976/1570 [1:05:07<21:22, 2.16s/it]
62%|██████▏ | 977/1570 [1:05:09<21:19, 2.16s/it]
62%|██████▏ | 978/1570 [1:05:11<21:16, 2.16s/it]
62%|██████▏ | 979/1570 [1:05:13<21:14, 2.16s/it]
62%|██████▏ | 980/1570 [1:05:15<21:10, 2.15s/it]
{'loss': '0.001483', 'grad_norm': '0.004794', 'learning_rate': '3.114e-05', 'epoch': '3.121'}
+
62%|██████▏ | 980/1570 [1:05:15<21:10, 2.15s/it]
62%|██████▏ | 981/1570 [1:05:18<21:08, 2.15s/it]
63%|██████▎ | 982/1570 [1:05:20<21:07, 2.16s/it]
63%|██████▎ | 983/1570 [1:05:22<21:04, 2.15s/it]
63%|██████▎ | 984/1570 [1:05:24<21:01, 2.15s/it]
63%|██████▎ | 985/1570 [1:05:26<20:57, 2.15s/it]
63%|██████▎ | 986/1570 [1:05:28<20:56, 2.15s/it]
63%|██████▎ | 987/1570 [1:05:30<20:54, 2.15s/it]
63%|██████▎ | 988/1570 [1:05:33<20:50, 2.15s/it]
63%|██████▎ | 989/1570 [1:05:35<20:51, 2.15s/it]
63%|██████▎ | 990/1570 [1:05:37<20:50, 2.16s/it]
63%|██████▎ | 991/1570 [1:05:39<20:48, 2.16s/it]
63%|██████▎ | 992/1570 [1:05:41<20:45, 2.15s/it]
63%|██████▎ | 993/1570 [1:05:43<20:45, 2.16s/it]
63%|██████▎ | 994/1570 [1:05:46<20:39, 2.15s/it]
63%|██████▎ | 995/1570 [1:05:48<20:35, 2.15s/it]
63%|██████▎ | 996/1570 [1:05:50<20:35, 2.15s/it]
64%|██████▎ | 997/1570 [1:05:52<20:32, 2.15s/it]
64%|██████▎ | 998/1570 [1:05:54<20:28, 2.15s/it]
64%|██████▎ | 999/1570 [1:05:56<20:28, 2.15s/it]
64%|██████▎ | 1000/1570 [1:05:58<20:27, 2.15s/it]
{'loss': '0.0005886', 'grad_norm': '0.005231', 'learning_rate': '2.931e-05', 'epoch': '3.185'}
+
64%|██████▎ | 1000/1570 [1:05:58<20:27, 2.15s/it]
64%|██████▍ | 1001/1570 [1:06:01<20:25, 2.15s/it]
64%|██████▍ | 1002/1570 [1:06:03<20:23, 2.15s/it]
64%|██████▍ | 1003/1570 [1:06:05<20:21, 2.15s/it]
64%|██████▍ | 1004/1570 [1:06:07<20:17, 2.15s/it]
64%|██████▍ | 1005/1570 [1:06:09<20:17, 2.16s/it]
64%|██████▍ | 1006/1570 [1:06:11<20:14, 2.15s/it]
64%|██████▍ | 1007/1570 [1:06:14<20:10, 2.15s/it]
64%|██████▍ | 1008/1570 [1:06:16<20:08, 2.15s/it]
64%|██████▍ | 1009/1570 [1:06:18<20:07, 2.15s/it]
64%|██████▍ | 1010/1570 [1:06:20<20:06, 2.15s/it]
64%|██████▍ | 1011/1570 [1:06:22<20:01, 2.15s/it]
64%|██████▍ | 1012/1570 [1:06:24<20:02, 2.15s/it]
65%|██████▍ | 1013/1570 [1:06:26<20:00, 2.15s/it]
65%|██████▍ | 1014/1570 [1:06:29<19:56, 2.15s/it]
65%|██████▍ | 1015/1570 [1:06:31<19:54, 2.15s/it]
65%|██████▍ | 1016/1570 [1:06:33<19:51, 2.15s/it]
65%|██████▍ | 1017/1570 [1:06:35<19:48, 2.15s/it]
65%|██████▍ | 1018/1570 [1:06:37<19:48, 2.15s/it]
65%|██████▍ | 1019/1570 [1:06:39<19:44, 2.15s/it]
65%|██████▍ | 1020/1570 [1:06:42<19:44, 2.15s/it]
{'loss': '0.0016', 'grad_norm': '0.08027', 'learning_rate': '2.751e-05', 'epoch': '3.249'}
+
65%|██████▍ | 1020/1570 [1:06:42<19:44, 2.15s/it]
65%|██████▌ | 1021/1570 [1:06:44<19:43, 2.16s/it]
65%|██████▌ | 1022/1570 [1:06:46<19:40, 2.15s/it]
65%|██████▌ | 1023/1570 [1:06:48<19:39, 2.16s/it]
65%|██████▌ | 1024/1570 [1:06:50<19:35, 2.15s/it]
65%|██████▌ | 1025/1570 [1:06:52<19:34, 2.15s/it]
65%|██████▌ | 1026/1570 [1:06:54<19:29, 2.15s/it]
65%|██████▌ | 1027/1570 [1:06:57<19:20, 2.14s/it]
65%|██████▌ | 1028/1570 [1:06:59<19:14, 2.13s/it]
66%|██████▌ | 1029/1570 [1:07:01<19:10, 2.13s/it]
66%|██████▌ | 1030/1570 [1:07:03<19:08, 2.13s/it]
66%|██████▌ | 1031/1570 [1:07:05<19:04, 2.12s/it]
66%|██████▌ | 1032/1570 [1:07:07<19:01, 2.12s/it]
66%|██████▌ | 1033/1570 [1:07:09<19:00, 2.12s/it]
66%|██████▌ | 1034/1570 [1:07:11<18:56, 2.12s/it]
66%|██████▌ | 1035/1570 [1:07:14<18:58, 2.13s/it]
66%|██████▌ | 1036/1570 [1:07:16<18:59, 2.13s/it]
66%|██████▌ | 1037/1570 [1:07:18<18:54, 2.13s/it]
66%|██████▌ | 1038/1570 [1:07:20<18:50, 2.12s/it]
66%|██████▌ | 1039/1570 [1:07:22<18:46, 2.12s/it]
66%|██████▌ | 1040/1570 [1:07:24<18:44, 2.12s/it]
{'loss': '0.0007604', 'grad_norm': '0.04866', 'learning_rate': '2.574e-05', 'epoch': '3.313'}
+
66%|██████▌ | 1040/1570 [1:07:24<18:44, 2.12s/it]
66%|██████▋ | 1041/1570 [1:07:26<18:45, 2.13s/it]
66%|██████▋ | 1042/1570 [1:07:28<18:45, 2.13s/it]
66%|██████▋ | 1043/1570 [1:07:31<18:40, 2.13s/it]
66%|██████▋ | 1044/1570 [1:07:33<18:40, 2.13s/it]
67%|██████▋ | 1045/1570 [1:07:35<18:38, 2.13s/it]
67%|██████▋ | 1046/1570 [1:07:37<18:37, 2.13s/it]
67%|██████▋ | 1047/1570 [1:07:39<18:36, 2.13s/it]
67%|██████▋ | 1048/1570 [1:07:41<18:34, 2.13s/it]
67%|██████▋ | 1049/1570 [1:07:43<18:30, 2.13s/it]
67%|██████▋ | 1050/1570 [1:07:45<18:29, 2.13s/it]
67%|██████▋ | 1051/1570 [1:07:48<18:26, 2.13s/it]
67%|██████▋ | 1052/1570 [1:07:50<18:24, 2.13s/it]
67%|██████▋ | 1053/1570 [1:07:52<18:22, 2.13s/it]
67%|██████▋ | 1054/1570 [1:07:54<18:19, 2.13s/it]
67%|██████▋ | 1055/1570 [1:07:56<18:17, 2.13s/it]
67%|██████▋ | 1056/1570 [1:07:58<18:16, 2.13s/it]
67%|██████▋ | 1057/1570 [1:08:00<18:13, 2.13s/it]
67%|██████▋ | 1058/1570 [1:08:03<18:09, 2.13s/it]
67%|██████▋ | 1059/1570 [1:08:05<18:07, 2.13s/it]
68%|██████▊ | 1060/1570 [1:08:07<18:07, 2.13s/it]
{'loss': '0.001686', 'grad_norm': '0.002228', 'learning_rate': '2.402e-05', 'epoch': '3.377'}
+
68%|██████▊ | 1060/1570 [1:08:07<18:07, 2.13s/it]
68%|██████▊ | 1061/1570 [1:08:09<18:06, 2.13s/it]
68%|██████▊ | 1062/1570 [1:08:11<18:04, 2.13s/it]
68%|██████▊ | 1063/1570 [1:08:13<18:04, 2.14s/it]
68%|██████▊ | 1064/1570 [1:08:15<18:00, 2.13s/it]
68%|██████▊ | 1065/1570 [1:08:17<17:57, 2.13s/it]
68%|██████▊ | 1066/1570 [1:08:20<17:54, 2.13s/it]
68%|██████▊ | 1067/1570 [1:08:22<17:52, 2.13s/it]
68%|██████▊ | 1068/1570 [1:08:24<17:51, 2.13s/it]
68%|██████▊ | 1069/1570 [1:08:26<17:48, 2.13s/it]
68%|██████▊ | 1070/1570 [1:08:28<17:45, 2.13s/it]
68%|██████▊ | 1071/1570 [1:08:30<17:44, 2.13s/it]
68%|██████▊ | 1072/1570 [1:08:32<17:43, 2.14s/it]
68%|██████▊ | 1073/1570 [1:08:35<17:41, 2.14s/it]
68%|██████▊ | 1074/1570 [1:08:37<17:39, 2.14s/it]
68%|██████▊ | 1075/1570 [1:08:39<17:37, 2.14s/it]
69%|██████▊ | 1076/1570 [1:08:41<17:35, 2.14s/it]
69%|██████▊ | 1077/1570 [1:08:43<17:33, 2.14s/it]
69%|██████▊ | 1078/1570 [1:08:45<17:31, 2.14s/it]
69%|██████▊ | 1079/1570 [1:08:47<17:30, 2.14s/it]
69%|██████▉ | 1080/1570 [1:08:49<17:27, 2.14s/it]
{'loss': '0.0008015', 'grad_norm': '0.1205', 'learning_rate': '2.233e-05', 'epoch': '3.44'}
+
69%|██████▉ | 1080/1570 [1:08:49<17:27, 2.14s/it]
69%|██████▉ | 1081/1570 [1:08:52<17:24, 2.14s/it]
69%|██████▉ | 1082/1570 [1:08:54<17:22, 2.14s/it]
69%|██████▉ | 1083/1570 [1:08:56<17:19, 2.13s/it]
69%|██████▉ | 1084/1570 [1:08:58<17:17, 2.13s/it]
69%|██████▉ | 1085/1570 [1:09:00<17:15, 2.13s/it]
69%|██████▉ | 1086/1570 [1:09:02<17:13, 2.14s/it]
69%|██████▉ | 1087/1570 [1:09:04<17:12, 2.14s/it]
69%|██████▉ | 1088/1570 [1:09:07<17:10, 2.14s/it]
69%|██████▉ | 1089/1570 [1:09:09<17:07, 2.14s/it]
69%|██████▉ | 1090/1570 [1:09:11<17:07, 2.14s/it]
69%|██████▉ | 1091/1570 [1:09:13<17:03, 2.14s/it]
70%|██████▉ | 1092/1570 [1:09:15<16:59, 2.13s/it]
70%|██████▉ | 1093/1570 [1:09:17<16:57, 2.13s/it]
70%|██████▉ | 1094/1570 [1:09:19<16:54, 2.13s/it]
70%|██████▉ | 1095/1570 [1:09:22<16:54, 2.14s/it]
70%|██████▉ | 1096/1570 [1:09:24<16:53, 2.14s/it]
70%|██████▉ | 1097/1570 [1:09:26<16:52, 2.14s/it]
70%|██████▉ | 1098/1570 [1:09:28<16:49, 2.14s/it]
70%|███████ | 1099/1570 [1:09:30<16:46, 2.14s/it]
70%|███████ | 1100/1570 [1:09:32<16:43, 2.13s/it]
{'loss': '0.001147', 'grad_norm': '0.004857', 'learning_rate': '2.069e-05', 'epoch': '3.504'}
+
70%|███████ | 1100/1570 [1:09:32<16:43, 2.13s/it]
70%|███████ | 1101/1570 [1:09:34<16:40, 2.13s/it]
70%|███████ | 1102/1570 [1:09:36<16:38, 2.13s/it]
70%|███████ | 1103/1570 [1:09:39<16:36, 2.13s/it]
70%|███████ | 1104/1570 [1:09:41<16:33, 2.13s/it]
70%|███████ | 1105/1570 [1:09:43<16:29, 2.13s/it]
70%|███████ | 1106/1570 [1:09:45<16:27, 2.13s/it]
71%|███████ | 1107/1570 [1:09:47<16:26, 2.13s/it]
71%|███████ | 1108/1570 [1:09:49<16:26, 2.13s/it]
71%|███████ | 1109/1570 [1:09:51<16:24, 2.14s/it]
71%|███████ | 1110/1570 [1:09:54<16:22, 2.14s/it]
71%|███████ | 1111/1570 [1:09:56<16:20, 2.14s/it]
71%|███████ | 1112/1570 [1:09:58<16:21, 2.14s/it]
71%|███████ | 1113/1570 [1:10:00<16:18, 2.14s/it]
71%|███████ | 1114/1570 [1:10:02<16:18, 2.15s/it]
71%|███████ | 1115/1570 [1:10:04<16:14, 2.14s/it]
71%|███████ | 1116/1570 [1:10:06<16:09, 2.14s/it]
71%|███████ | 1117/1570 [1:10:09<16:08, 2.14s/it]
71%|███████ | 1118/1570 [1:10:11<16:08, 2.14s/it]
71%|███████▏ | 1119/1570 [1:10:13<16:05, 2.14s/it]
71%|███████▏ | 1120/1570 [1:10:15<16:05, 2.15s/it]
{'loss': '0.00131', 'grad_norm': '0.03997', 'learning_rate': '1.91e-05', 'epoch': '3.568'}
+
71%|███████▏ | 1120/1570 [1:10:15<16:05, 2.15s/it]
71%|███████▏ | 1121/1570 [1:10:17<16:02, 2.14s/it]
71%|███████▏ | 1122/1570 [1:10:19<15:59, 2.14s/it]
72%|███████▏ | 1123/1570 [1:10:21<15:56, 2.14s/it]
72%|███████▏ | 1124/1570 [1:10:24<15:54, 2.14s/it]
72%|███████▏ | 1125/1570 [1:10:26<15:53, 2.14s/it]
72%|███████▏ | 1126/1570 [1:10:28<15:49, 2.14s/it]
72%|███████▏ | 1127/1570 [1:10:30<15:47, 2.14s/it]
72%|███████▏ | 1128/1570 [1:10:32<15:43, 2.13s/it]
72%|███████▏ | 1129/1570 [1:10:34<15:42, 2.14s/it]
72%|███████▏ | 1130/1570 [1:10:36<15:40, 2.14s/it]
72%|███████▏ | 1131/1570 [1:10:38<15:39, 2.14s/it]
72%|███████▏ | 1132/1570 [1:10:41<15:37, 2.14s/it]
72%|███████▏ | 1133/1570 [1:10:43<15:33, 2.14s/it]
72%|███████▏ | 1134/1570 [1:10:45<15:31, 2.14s/it]
72%|███████▏ | 1135/1570 [1:10:47<15:29, 2.14s/it]
72%|███████▏ | 1136/1570 [1:10:49<15:28, 2.14s/it]
72%|███████▏ | 1137/1570 [1:10:51<15:27, 2.14s/it]
72%|███████▏ | 1138/1570 [1:10:53<15:24, 2.14s/it]
73%|███████▎ | 1139/1570 [1:10:56<15:25, 2.15s/it]
73%|███████▎ | 1140/1570 [1:10:58<15:22, 2.15s/it]
{'loss': '0.0008654', 'grad_norm': '0.0442', 'learning_rate': '1.755e-05', 'epoch': '3.632'}
+
73%|███████▎ | 1140/1570 [1:10:58<15:22, 2.15s/it]
73%|███████▎ | 1141/1570 [1:11:00<15:21, 2.15s/it]
73%|███████▎ | 1142/1570 [1:11:02<15:18, 2.15s/it]
73%|███████▎ | 1143/1570 [1:11:04<15:16, 2.15s/it]
73%|███████▎ | 1144/1570 [1:11:06<15:13, 2.14s/it]
73%|███████▎ | 1145/1570 [1:11:09<15:13, 2.15s/it]
73%|███████▎ | 1146/1570 [1:11:11<15:10, 2.15s/it]
73%|███████▎ | 1147/1570 [1:11:13<15:06, 2.14s/it]
73%|███████▎ | 1148/1570 [1:11:15<15:03, 2.14s/it]
73%|███████▎ | 1149/1570 [1:11:17<15:01, 2.14s/it]
73%|███████▎ | 1150/1570 [1:11:19<14:57, 2.14s/it]
73%|███████▎ | 1151/1570 [1:11:21<14:55, 2.14s/it]
73%|███████▎ | 1152/1570 [1:11:23<14:52, 2.14s/it]
73%|███████▎ | 1153/1570 [1:11:26<14:51, 2.14s/it]
74%|███████▎ | 1154/1570 [1:11:28<14:51, 2.14s/it]
74%|███████▎ | 1155/1570 [1:11:30<14:49, 2.14s/it]
74%|███████▎ | 1156/1570 [1:11:32<14:46, 2.14s/it]
74%|███████▎ | 1157/1570 [1:11:34<14:44, 2.14s/it]
74%|███████▍ | 1158/1570 [1:11:36<14:41, 2.14s/it]
74%|███████▍ | 1159/1570 [1:11:38<14:40, 2.14s/it]
74%|███████▍ | 1160/1570 [1:11:41<14:38, 2.14s/it]
{'loss': '0.004081', 'grad_norm': '0.01121', 'learning_rate': '1.606e-05', 'epoch': '3.696'}
+
74%|███████▍ | 1160/1570 [1:11:41<14:38, 2.14s/it]
74%|███████▍ | 1161/1570 [1:11:43<14:37, 2.15s/it]
74%|███████▍ | 1162/1570 [1:11:45<14:35, 2.15s/it]
74%|███████▍ | 1163/1570 [1:11:47<14:31, 2.14s/it]
74%|███████▍ | 1164/1570 [1:11:49<14:29, 2.14s/it]
74%|███████▍ | 1165/1570 [1:11:51<14:26, 2.14s/it]
74%|███████▍ | 1166/1570 [1:11:53<14:23, 2.14s/it]
74%|███████▍ | 1167/1570 [1:11:56<14:22, 2.14s/it]
74%|███████▍ | 1168/1570 [1:11:58<14:17, 2.13s/it]
74%|███████▍ | 1169/1570 [1:12:00<14:13, 2.13s/it]
75%|███████▍ | 1170/1570 [1:12:02<14:10, 2.13s/it]
75%|███████▍ | 1171/1570 [1:12:04<14:07, 2.12s/it]
75%|███████▍ | 1172/1570 [1:12:06<14:03, 2.12s/it]
75%|███████▍ | 1173/1570 [1:12:08<14:03, 2.12s/it]
75%|███████▍ | 1174/1570 [1:12:10<14:01, 2.12s/it]
75%|███████▍ | 1175/1570 [1:12:13<13:58, 2.12s/it]
75%|███████▍ | 1176/1570 [1:12:15<13:57, 2.13s/it]
75%|███████▍ | 1177/1570 [1:12:17<13:54, 2.12s/it]
75%|███████▌ | 1178/1570 [1:12:19<13:53, 2.13s/it]
75%|███████▌ | 1179/1570 [1:12:21<13:50, 2.13s/it]
75%|███████▌ | 1180/1570 [1:12:23<13:48, 2.12s/it]
{'loss': '0.001724', 'grad_norm': '0.01562', 'learning_rate': '1.462e-05', 'epoch': '3.759'}
+
75%|███████▌ | 1180/1570 [1:12:23<13:48, 2.12s/it]
75%|███████▌ | 1181/1570 [1:12:25<13:47, 2.13s/it]
75%|███████▌ | 1182/1570 [1:12:27<13:45, 2.13s/it]
75%|███████▌ | 1183/1570 [1:12:30<13:44, 2.13s/it]
75%|███████▌ | 1184/1570 [1:12:32<13:44, 2.14s/it]
75%|███████▌ | 1185/1570 [1:12:34<13:43, 2.14s/it]
76%|███████▌ | 1186/1570 [1:12:36<13:40, 2.14s/it]
76%|███████▌ | 1187/1570 [1:12:38<13:36, 2.13s/it]
76%|███████▌ | 1188/1570 [1:12:40<13:35, 2.13s/it]
76%|███████▌ | 1189/1570 [1:12:42<13:32, 2.13s/it]
76%|███████▌ | 1190/1570 [1:12:45<13:31, 2.14s/it]
76%|███████▌ | 1191/1570 [1:12:47<13:31, 2.14s/it]
76%|███████▌ | 1192/1570 [1:12:49<13:25, 2.13s/it]
76%|███████▌ | 1193/1570 [1:12:51<13:21, 2.13s/it]
76%|███████▌ | 1194/1570 [1:12:53<13:21, 2.13s/it]
76%|███████▌ | 1195/1570 [1:12:55<13:19, 2.13s/it]
76%|███████▌ | 1196/1570 [1:12:57<13:17, 2.13s/it]
76%|███████▌ | 1197/1570 [1:12:59<13:15, 2.13s/it]
76%|███████▋ | 1198/1570 [1:13:02<13:13, 2.13s/it]
76%|███████▋ | 1199/1570 [1:13:04<13:12, 2.14s/it]
76%|███████▋ | 1200/1570 [1:13:06<13:10, 2.14s/it]
{'loss': '0.001184', 'grad_norm': '0.02863', 'learning_rate': '1.324e-05', 'epoch': '3.823'}
+
76%|███████▋ | 1200/1570 [1:13:06<13:10, 2.14s/it]
76%|███████▋ | 1201/1570 [1:13:08<13:07, 2.13s/it]
77%|███████▋ | 1202/1570 [1:13:10<13:05, 2.13s/it]
77%|███████▋ | 1203/1570 [1:13:12<13:02, 2.13s/it]
77%|███████▋ | 1204/1570 [1:13:14<13:01, 2.14s/it]
77%|███████▋ | 1205/1570 [1:13:17<13:01, 2.14s/it]
77%|███████▋ | 1206/1570 [1:13:19<12:59, 2.14s/it]
77%|███████▋ | 1207/1570 [1:13:21<12:55, 2.14s/it]
77%|███████▋ | 1208/1570 [1:13:23<12:51, 2.13s/it]
77%|███████▋ | 1209/1570 [1:13:25<12:50, 2.13s/it]
77%|███████▋ | 1210/1570 [1:13:27<12:47, 2.13s/it]
77%|███████▋ | 1211/1570 [1:13:29<12:47, 2.14s/it]
77%|███████▋ | 1212/1570 [1:13:31<12:43, 2.13s/it]
77%|███████▋ | 1213/1570 [1:13:34<12:43, 2.14s/it]
77%|███████▋ | 1214/1570 [1:13:36<12:41, 2.14s/it]
77%|███████▋ | 1215/1570 [1:13:38<12:39, 2.14s/it]
77%|███████▋ | 1216/1570 [1:13:40<12:36, 2.14s/it]
78%|███████▊ | 1217/1570 [1:13:42<12:34, 2.14s/it]
78%|███████▊ | 1218/1570 [1:13:44<12:32, 2.14s/it]
78%|███████▊ | 1219/1570 [1:13:46<12:31, 2.14s/it]
78%|███████▊ | 1220/1570 [1:13:49<12:29, 2.14s/it]
{'loss': '0.001461', 'grad_norm': '0.1982', 'learning_rate': '1.192e-05', 'epoch': '3.887'}
+
78%|███████▊ | 1220/1570 [1:13:49<12:29, 2.14s/it]
78%|███████▊ | 1221/1570 [1:13:51<12:26, 2.14s/it]
78%|███████▊ | 1222/1570 [1:13:53<12:26, 2.14s/it]
78%|███████▊ | 1223/1570 [1:13:55<12:23, 2.14s/it]
78%|███████▊ | 1224/1570 [1:13:57<12:21, 2.14s/it]
78%|███████▊ | 1225/1570 [1:13:59<12:20, 2.15s/it]
78%|███████▊ | 1226/1570 [1:14:01<12:18, 2.15s/it]
78%|███████▊ | 1227/1570 [1:14:04<12:14, 2.14s/it]
78%|███████▊ | 1228/1570 [1:14:06<12:11, 2.14s/it]
78%|███████▊ | 1229/1570 [1:14:08<12:11, 2.14s/it]
78%|███████▊ | 1230/1570 [1:14:10<12:06, 2.14s/it]
78%|███████▊ | 1231/1570 [1:14:12<12:02, 2.13s/it]
78%|███████▊ | 1232/1570 [1:14:14<12:01, 2.13s/it]
79%|███████▊ | 1233/1570 [1:14:16<11:59, 2.14s/it]
79%|███████▊ | 1234/1570 [1:14:19<11:58, 2.14s/it]
79%|███████▊ | 1235/1570 [1:14:21<11:57, 2.14s/it]
79%|███████▊ | 1236/1570 [1:14:23<11:54, 2.14s/it]
79%|███████▉ | 1237/1570 [1:14:25<11:53, 2.14s/it]
79%|███████▉ | 1238/1570 [1:14:27<11:51, 2.14s/it]
79%|███████▉ | 1239/1570 [1:14:29<11:48, 2.14s/it]
79%|███████▉ | 1240/1570 [1:14:31<11:46, 2.14s/it]
{'loss': '0.0006166', 'grad_norm': '0.02603', 'learning_rate': '1.066e-05', 'epoch': '3.951'}
+
79%|███████▉ | 1240/1570 [1:14:31<11:46, 2.14s/it]
79%|███████▉ | 1241/1570 [1:14:34<11:45, 2.15s/it]
79%|███████▉ | 1242/1570 [1:14:36<11:41, 2.14s/it]
79%|███████▉ | 1243/1570 [1:14:38<11:38, 2.14s/it]
79%|███████▉ | 1244/1570 [1:14:40<11:36, 2.14s/it]
79%|███████▉ | 1245/1570 [1:14:42<11:31, 2.13s/it]
79%|███████▉ | 1246/1570 [1:14:44<11:28, 2.13s/it]
79%|███████▉ | 1247/1570 [1:14:46<11:25, 2.12s/it]
79%|███████▉ | 1248/1570 [1:14:48<11:24, 2.13s/it]
80%|███████▉ | 1249/1570 [1:14:51<11:22, 2.13s/it]
80%|███████▉ | 1250/1570 [1:14:53<11:20, 2.13s/it]
80%|███████▉ | 1251/1570 [1:14:55<11:17, 2.12s/it]
80%|███████▉ | 1252/1570 [1:14:57<11:17, 2.13s/it]
80%|███████▉ | 1253/1570 [1:14:59<11:16, 2.13s/it]
80%|███████▉ | 1254/1570 [1:15:01<11:14, 2.13s/it]
80%|███████▉ | 1255/1570 [1:15:03<11:13, 2.14s/it]
80%|████████ | 1256/1570 [1:15:04<09:15, 1.77s/it][INFO|trainer.py:2668] 2026-08-22 09:28:10,082 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 09:28:10,082 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 09:28:10,082 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/sft/checkpoint-1256
+[INFO|configuration_utils.py:780] 2026-08-22 09:37:56,308 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 09:37:56,310 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 09:37:56,344 >> chat template saved in results/qwen3/sft/checkpoint-1256/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 09:37:56,345 >> tokenizer config file saved in results/qwen3/sft/checkpoint-1256/tokenizer_config.json
+
80%|████████ | 1257/1570 [1:24:53<15:28:29, 177.99s/it]
80%|████████ | 1258/1570 [1:24:56<10:51:13, 125.23s/it]
80%|████████ | 1259/1570 [1:24:58<7:37:44, 88.31s/it]
80%|████████ | 1260/1570 [1:25:00<5:22:42, 62.46s/it]
{'loss': '0.001044', 'grad_norm': '0.06869', 'learning_rate': '9.464e-06', 'epoch': '4.013'}
+
80%|████████ | 1260/1570 [1:25:00<5:22:42, 62.46s/it]
80%|████████ | 1261/1570 [1:25:02<3:48:30, 44.37s/it]
80%|████████ | 1262/1570 [1:25:04<2:42:44, 31.70s/it]
80%|████████ | 1263/1570 [1:25:06<1:56:51, 22.84s/it]
81%|████████ | 1264/1570 [1:25:09<1:24:48, 16.63s/it]
81%|████████ | 1265/1570 [1:25:11<1:02:27, 12.29s/it]
81%|████████ | 1266/1570 [1:25:13<46:49, 9.24s/it]
81%|████████ | 1267/1570 [1:25:15<35:55, 7.12s/it]
81%|████████ | 1268/1570 [1:25:17<28:19, 5.63s/it]
81%|████████ | 1269/1570 [1:25:19<22:59, 4.58s/it]
81%|████████ | 1270/1570 [1:25:21<19:15, 3.85s/it]
81%|████████ | 1271/1570 [1:25:24<16:39, 3.34s/it]
81%|████████ | 1272/1570 [1:25:26<14:49, 2.99s/it]
81%|████████ | 1273/1570 [1:25:28<13:31, 2.73s/it]
81%|████████ | 1274/1570 [1:25:30<12:37, 2.56s/it]
81%|████████ | 1275/1570 [1:25:32<11:57, 2.43s/it]
81%|████████▏ | 1276/1570 [1:25:34<11:30, 2.35s/it]
81%|████████▏ | 1277/1570 [1:25:36<11:09, 2.29s/it]
81%|████████▏ | 1278/1570 [1:25:39<10:55, 2.24s/it]
81%|████████▏ | 1279/1570 [1:25:41<10:44, 2.21s/it]
82%|████████▏ | 1280/1570 [1:25:43<10:35, 2.19s/it]
{'loss': '0.0004742', 'grad_norm': '0.1313', 'learning_rate': '8.332e-06', 'epoch': '4.077'}
+
82%|████████▏ | 1280/1570 [1:25:43<10:35, 2.19s/it]
82%|████████▏ | 1281/1570 [1:25:45<10:30, 2.18s/it]
82%|████████▏ | 1282/1570 [1:25:47<10:25, 2.17s/it]
82%|████████▏ | 1283/1570 [1:25:49<10:21, 2.17s/it]
82%|████████▏ | 1284/1570 [1:25:51<10:16, 2.16s/it]
82%|████████▏ | 1285/1570 [1:25:54<10:13, 2.15s/it]
82%|████████▏ | 1286/1570 [1:25:56<10:10, 2.15s/it]
82%|████████▏ | 1287/1570 [1:25:58<10:08, 2.15s/it]
82%|████████▏ | 1288/1570 [1:26:00<10:07, 2.16s/it]
82%|████████▏ | 1289/1570 [1:26:02<10:05, 2.15s/it]
82%|████████▏ | 1290/1570 [1:26:04<10:02, 2.15s/it]
82%|████████▏ | 1291/1570 [1:26:07<10:00, 2.15s/it]
82%|████████▏ | 1292/1570 [1:26:09<09:57, 2.15s/it]
82%|████████▏ | 1293/1570 [1:26:11<09:54, 2.15s/it]
82%|████████▏ | 1294/1570 [1:26:13<09:51, 2.14s/it]
82%|████████▏ | 1295/1570 [1:26:15<09:50, 2.15s/it]
83%|████████▎ | 1296/1570 [1:26:17<09:48, 2.15s/it]
83%|████████▎ | 1297/1570 [1:26:19<09:45, 2.14s/it]
83%|████████▎ | 1298/1570 [1:26:22<09:44, 2.15s/it]
83%|████████▎ | 1299/1570 [1:26:24<09:40, 2.14s/it]
83%|████████▎ | 1300/1570 [1:26:26<09:40, 2.15s/it]
{'loss': '0.0009649', 'grad_norm': '0.001422', 'learning_rate': '7.266e-06', 'epoch': '4.14'}
+
83%|████████▎ | 1300/1570 [1:26:26<09:40, 2.15s/it]
83%|████████▎ | 1301/1570 [1:26:28<09:37, 2.15s/it]
83%|████████▎ | 1302/1570 [1:26:30<09:35, 2.15s/it]
83%|████████▎ | 1303/1570 [1:26:32<09:34, 2.15s/it]
83%|████████▎ | 1304/1570 [1:26:34<09:30, 2.14s/it]
83%|████████▎ | 1305/1570 [1:26:37<09:29, 2.15s/it]
83%|████████▎ | 1306/1570 [1:26:39<09:27, 2.15s/it]
83%|████████▎ | 1307/1570 [1:26:41<09:24, 2.15s/it]
83%|████████▎ | 1308/1570 [1:26:43<09:22, 2.15s/it]
83%|████████▎ | 1309/1570 [1:26:45<09:19, 2.15s/it]
83%|████████▎ | 1310/1570 [1:26:47<09:18, 2.15s/it]
84%|████████▎ | 1311/1570 [1:26:49<09:16, 2.15s/it]
84%|████████▎ | 1312/1570 [1:26:52<09:14, 2.15s/it]
84%|████████▎ | 1313/1570 [1:26:54<09:11, 2.15s/it]
84%|████████▎ | 1314/1570 [1:26:56<09:08, 2.14s/it]
84%|████████▍ | 1315/1570 [1:26:58<09:04, 2.14s/it]
84%|████████▍ | 1316/1570 [1:27:00<09:01, 2.13s/it]
84%|████████▍ | 1317/1570 [1:27:02<08:59, 2.13s/it]
84%|████████▍ | 1318/1570 [1:27:04<08:56, 2.13s/it]
84%|████████▍ | 1319/1570 [1:27:07<08:53, 2.13s/it]
84%|████████▍ | 1320/1570 [1:27:09<08:50, 2.12s/it]
{'loss': '0.000656', 'grad_norm': '0.01662', 'learning_rate': '6.269e-06', 'epoch': '4.204'}
+
84%|████████▍ | 1320/1570 [1:27:09<08:50, 2.12s/it]
84%|████████▍ | 1321/1570 [1:27:11<08:49, 2.13s/it]
84%|████████▍ | 1322/1570 [1:27:13<08:46, 2.12s/it]
84%|████████▍ | 1323/1570 [1:27:15<08:43, 2.12s/it]
84%|████████▍ | 1324/1570 [1:27:17<08:42, 2.13s/it]
84%|████████▍ | 1325/1570 [1:27:19<08:41, 2.13s/it]
84%|████████▍ | 1326/1570 [1:27:21<08:38, 2.13s/it]
85%|████████▍ | 1327/1570 [1:27:24<08:37, 2.13s/it]
85%|████████▍ | 1328/1570 [1:27:26<08:35, 2.13s/it]
85%|████████▍ | 1329/1570 [1:27:28<08:33, 2.13s/it]
85%|████████▍ | 1330/1570 [1:27:30<08:31, 2.13s/it]
85%|████████▍ | 1331/1570 [1:27:32<08:29, 2.13s/it]
85%|████████▍ | 1332/1570 [1:27:34<08:27, 2.13s/it]
85%|████████▍ | 1333/1570 [1:27:36<08:25, 2.13s/it]
85%|████████▍ | 1334/1570 [1:27:38<08:23, 2.13s/it]
85%|████████▌ | 1335/1570 [1:27:41<08:22, 2.14s/it]
85%|████████▌ | 1336/1570 [1:27:43<08:19, 2.13s/it]
85%|████████▌ | 1337/1570 [1:27:45<08:17, 2.13s/it]
85%|████████▌ | 1338/1570 [1:27:47<08:16, 2.14s/it]
85%|████████▌ | 1339/1570 [1:27:49<08:15, 2.14s/it]
85%|████████▌ | 1340/1570 [1:27:51<08:11, 2.14s/it]
{'loss': '0.001283', 'grad_norm': '0.05787', 'learning_rate': '5.342e-06', 'epoch': '4.268'}
+
85%|████████▌ | 1340/1570 [1:27:51<08:11, 2.14s/it]
85%|████████▌ | 1341/1570 [1:27:53<08:09, 2.14s/it]
85%|████████▌ | 1342/1570 [1:27:56<08:06, 2.14s/it]
86%|████████▌ | 1343/1570 [1:27:58<08:04, 2.14s/it]
86%|████████▌ | 1344/1570 [1:28:00<08:03, 2.14s/it]
86%|████████▌ | 1345/1570 [1:28:02<08:00, 2.14s/it]
86%|████████▌ | 1346/1570 [1:28:04<07:58, 2.14s/it]
86%|████████▌ | 1347/1570 [1:28:06<07:55, 2.13s/it]
86%|████████▌ | 1348/1570 [1:28:08<07:54, 2.14s/it]
86%|████████▌ | 1349/1570 [1:28:11<07:53, 2.14s/it]
86%|████████▌ | 1350/1570 [1:28:13<07:50, 2.14s/it]
86%|████████▌ | 1351/1570 [1:28:15<07:48, 2.14s/it]
86%|████████▌ | 1352/1570 [1:28:17<07:46, 2.14s/it]
86%|████████▌ | 1353/1570 [1:28:19<07:45, 2.14s/it]
86%|████████▌ | 1354/1570 [1:28:21<07:42, 2.14s/it]
86%|████████▋ | 1355/1570 [1:28:23<07:39, 2.14s/it]
86%|████████▋ | 1356/1570 [1:28:26<07:38, 2.14s/it]
86%|████████▋ | 1357/1570 [1:28:28<07:36, 2.14s/it]
86%|████████▋ | 1358/1570 [1:28:30<07:34, 2.14s/it]
87%|████████▋ | 1359/1570 [1:28:32<07:31, 2.14s/it]
87%|████████▋ | 1360/1570 [1:28:34<07:28, 2.14s/it]
{'loss': '0.0005498', 'grad_norm': '0.005152', 'learning_rate': '4.486e-06', 'epoch': '4.332'}
+
87%|████████▋ | 1360/1570 [1:28:34<07:28, 2.14s/it]
87%|████████▋ | 1361/1570 [1:28:36<07:27, 2.14s/it]
87%|████████▋ | 1362/1570 [1:28:38<07:25, 2.14s/it]
87%|████████▋ | 1363/1570 [1:28:41<07:24, 2.15s/it]
87%|████████▋ | 1364/1570 [1:28:43<07:21, 2.14s/it]
87%|████████▋ | 1365/1570 [1:28:45<07:19, 2.14s/it]
87%|████████▋ | 1366/1570 [1:28:47<07:16, 2.14s/it]
87%|████████▋ | 1367/1570 [1:28:49<07:13, 2.13s/it]
87%|████████▋ | 1368/1570 [1:28:51<07:11, 2.14s/it]
87%|████████▋ | 1369/1570 [1:28:53<07:10, 2.14s/it]
87%|████████▋ | 1370/1570 [1:28:56<07:34, 2.27s/it]
87%|████████▋ | 1371/1570 [1:28:58<07:23, 2.23s/it]
87%|████████▋ | 1372/1570 [1:29:00<07:16, 2.20s/it]
87%|████████▋ | 1373/1570 [1:29:02<07:10, 2.19s/it]
88%|████████▊ | 1374/1570 [1:29:04<07:06, 2.17s/it]
88%|████████▊ | 1375/1570 [1:29:07<07:01, 2.16s/it]
88%|████████▊ | 1376/1570 [1:29:09<06:58, 2.16s/it]
88%|████████▊ | 1377/1570 [1:29:11<06:56, 2.16s/it]
88%|████████▊ | 1378/1570 [1:29:13<06:53, 2.15s/it]
88%|████████▊ | 1379/1570 [1:29:16<07:14, 2.27s/it]
88%|████████▊ | 1380/1570 [1:29:18<07:04, 2.23s/it]
{'loss': '0.0009828', 'grad_norm': '0.01513', 'learning_rate': '3.704e-06', 'epoch': '4.396'}
+
88%|████████▊ | 1380/1570 [1:29:18<07:04, 2.23s/it]
88%|████████▊ | 1381/1570 [1:29:20<06:57, 2.21s/it]
88%|████████▊ | 1382/1570 [1:29:22<06:51, 2.19s/it]
88%|████████▊ | 1383/1570 [1:29:24<06:45, 2.17s/it]
88%|████████▊ | 1384/1570 [1:29:26<06:42, 2.16s/it]
88%|████████▊ | 1385/1570 [1:29:28<06:38, 2.15s/it]
88%|████████▊ | 1386/1570 [1:29:31<06:34, 2.15s/it]
88%|████████▊ | 1387/1570 [1:29:33<06:33, 2.15s/it]
88%|████████▊ | 1388/1570 [1:29:35<06:30, 2.15s/it]
88%|████████▊ | 1389/1570 [1:29:37<06:28, 2.15s/it]
89%|████████▊ | 1390/1570 [1:29:39<06:26, 2.15s/it]
89%|████████▊ | 1391/1570 [1:29:41<06:23, 2.14s/it]
89%|████████▊ | 1392/1570 [1:29:43<06:20, 2.14s/it]
89%|████████▊ | 1393/1570 [1:29:46<06:18, 2.14s/it]
89%|████████▉ | 1394/1570 [1:29:48<06:17, 2.14s/it]
89%|████████▉ | 1395/1570 [1:29:50<06:15, 2.14s/it]
89%|████████▉ | 1396/1570 [1:29:52<06:13, 2.14s/it]
89%|████████▉ | 1397/1570 [1:29:54<06:11, 2.15s/it]
89%|████████▉ | 1398/1570 [1:29:56<06:09, 2.15s/it]
89%|████████▉ | 1399/1570 [1:29:58<06:07, 2.15s/it]
89%|████████▉ | 1400/1570 [1:30:01<06:05, 2.15s/it]
{'loss': '0.0009567', 'grad_norm': '0.04744', 'learning_rate': '2.996e-06', 'epoch': '4.46'}
+
89%|████████▉ | 1400/1570 [1:30:01<06:05, 2.15s/it]
89%|████████▉ | 1401/1570 [1:30:03<06:03, 2.15s/it]
89%|████████▉ | 1402/1570 [1:30:05<06:00, 2.15s/it]
89%|████████▉ | 1403/1570 [1:30:07<05:58, 2.15s/it]
89%|████████▉ | 1404/1570 [1:30:09<05:56, 2.15s/it]
89%|████████▉ | 1405/1570 [1:30:11<05:54, 2.15s/it]
90%|████████▉ | 1406/1570 [1:30:13<05:51, 2.14s/it]
90%|████████▉ | 1407/1570 [1:30:16<05:48, 2.14s/it]
90%|████████▉ | 1408/1570 [1:30:18<05:47, 2.14s/it]
90%|████████▉ | 1409/1570 [1:30:20<05:45, 2.14s/it]
90%|████████▉ | 1410/1570 [1:30:22<05:43, 2.15s/it]
90%|████████▉ | 1411/1570 [1:30:24<05:41, 2.15s/it]
90%|████████▉ | 1412/1570 [1:30:26<05:38, 2.14s/it]
90%|█████████ | 1413/1570 [1:30:28<05:36, 2.14s/it]
90%|█████████ | 1414/1570 [1:30:31<05:33, 2.14s/it]
90%|█████████ | 1415/1570 [1:30:33<05:31, 2.14s/it]
90%|█████████ | 1416/1570 [1:30:35<05:29, 2.14s/it]
90%|█████████ | 1417/1570 [1:30:37<05:27, 2.14s/it]
90%|█████████ | 1418/1570 [1:30:39<05:25, 2.14s/it]
90%|█████████ | 1419/1570 [1:30:41<05:23, 2.14s/it]
90%|█████████ | 1420/1570 [1:30:43<05:20, 2.14s/it]
{'loss': '0.0007865', 'grad_norm': '0.07915', 'learning_rate': '2.363e-06', 'epoch': '4.523'}
+
90%|█████████ | 1420/1570 [1:30:43<05:20, 2.14s/it]
91%|█████████ | 1421/1570 [1:30:46<05:19, 2.15s/it]
91%|█████████ | 1422/1570 [1:30:48<05:17, 2.15s/it]
91%|█████████ | 1423/1570 [1:30:50<05:14, 2.14s/it]
91%|█████████ | 1424/1570 [1:30:52<05:12, 2.14s/it]
91%|█████████ | 1425/1570 [1:30:54<05:10, 2.14s/it]
91%|█████████ | 1426/1570 [1:30:56<05:08, 2.14s/it]
91%|█████████ | 1427/1570 [1:30:58<05:06, 2.14s/it]
91%|█████████ | 1428/1570 [1:31:01<05:04, 2.15s/it]
91%|█████████ | 1429/1570 [1:31:03<05:03, 2.15s/it]
91%|█████████ | 1430/1570 [1:31:05<05:01, 2.15s/it]
91%|█████████ | 1431/1570 [1:31:07<04:58, 2.15s/it]
91%|█████████ | 1432/1570 [1:31:09<04:55, 2.14s/it]
91%|█████████▏| 1433/1570 [1:31:11<04:53, 2.14s/it]
91%|█████████▏| 1434/1570 [1:31:13<04:50, 2.14s/it]
91%|█████████▏| 1435/1570 [1:31:16<04:48, 2.14s/it]
91%|█████████▏| 1436/1570 [1:31:18<04:47, 2.15s/it]
92%|█████████▏| 1437/1570 [1:31:20<04:45, 2.15s/it]
92%|█████████▏| 1438/1570 [1:31:22<04:45, 2.16s/it]
92%|█████████▏| 1439/1570 [1:31:24<04:43, 2.16s/it]
92%|█████████▏| 1440/1570 [1:31:26<04:42, 2.17s/it]
{'loss': '0.000466', 'grad_norm': '0.0259', 'learning_rate': '1.806e-06', 'epoch': '4.587'}
+
92%|█████████▏| 1440/1570 [1:31:26<04:42, 2.17s/it]
92%|█████████▏| 1441/1570 [1:31:29<04:39, 2.16s/it]
92%|█████████▏| 1442/1570 [1:31:31<04:35, 2.15s/it]
92%|█████████▏| 1443/1570 [1:31:33<04:33, 2.15s/it]
92%|█████████▏| 1444/1570 [1:31:35<04:32, 2.17s/it]
92%|█████████▏| 1445/1570 [1:31:37<04:35, 2.21s/it]
92%|█████████▏| 1446/1570 [1:31:40<04:48, 2.33s/it]
92%|█████████▏| 1447/1570 [1:31:43<05:02, 2.46s/it]
92%|█████████▏| 1448/1570 [1:31:45<04:53, 2.41s/it]
92%|█████████▏| 1449/1570 [1:31:47<04:41, 2.33s/it]
92%|█████████▏| 1450/1570 [1:31:49<04:33, 2.28s/it]
92%|█████████▏| 1451/1570 [1:31:52<04:28, 2.26s/it]
92%|█████████▏| 1452/1570 [1:31:54<04:23, 2.24s/it]
93%|█████████▎| 1453/1570 [1:31:56<04:17, 2.20s/it]
93%|█████████▎| 1454/1570 [1:31:58<04:13, 2.18s/it]
93%|█████████▎| 1455/1570 [1:32:00<04:11, 2.18s/it]
93%|█████████▎| 1456/1570 [1:32:02<04:09, 2.19s/it]
93%|█████████▎| 1457/1570 [1:32:05<04:17, 2.28s/it]
93%|█████████▎| 1458/1570 [1:32:08<04:29, 2.40s/it]
93%|█████████▎| 1459/1570 [1:32:10<04:22, 2.36s/it]
93%|█████████▎| 1460/1570 [1:32:13<04:50, 2.64s/it]
{'loss': '0.0005176', 'grad_norm': '0.02892', 'learning_rate': '1.327e-06', 'epoch': '4.651'}
+
93%|█████████▎| 1460/1570 [1:32:13<04:50, 2.64s/it]
93%|█████████▎| 1461/1570 [1:32:17<05:13, 2.88s/it]
93%|█████████▎| 1462/1570 [1:32:20<05:27, 3.04s/it]
93%|█████████▎| 1463/1570 [1:32:23<05:23, 3.02s/it]
93%|█████████▎| 1464/1570 [1:32:26<05:16, 2.98s/it]
93%|█████████▎| 1465/1570 [1:32:29<05:12, 2.97s/it]
93%|█████████▎| 1466/1570 [1:32:32<05:06, 2.94s/it]
93%|█████████▎| 1467/1570 [1:32:35<05:01, 2.92s/it]
94%|█████████▎| 1468/1570 [1:32:38<04:59, 2.94s/it]
94%|█████████▎| 1469/1570 [1:32:41<05:07, 3.04s/it]
94%|█████████▎| 1470/1570 [1:32:43<04:38, 2.79s/it]
94%|█████████▎| 1471/1570 [1:32:45<04:17, 2.61s/it]
94%|█████████▍| 1472/1570 [1:32:47<04:04, 2.50s/it]
94%|█████████▍| 1473/1570 [1:32:50<03:53, 2.41s/it]
94%|█████████▍| 1474/1570 [1:32:52<03:47, 2.37s/it]
94%|█████████▍| 1475/1570 [1:32:54<03:41, 2.33s/it]
94%|█████████▍| 1476/1570 [1:32:56<03:36, 2.30s/it]
94%|█████████▍| 1477/1570 [1:32:59<03:31, 2.27s/it]
94%|█████████▍| 1478/1570 [1:33:01<03:27, 2.26s/it]
94%|█████████▍| 1479/1570 [1:33:03<03:25, 2.26s/it]
94%|█████████▍| 1480/1570 [1:33:05<03:21, 2.24s/it]
{'loss': '0.0003303', 'grad_norm': '0.01238', 'learning_rate': '9.258e-07', 'epoch': '4.715'}
+
94%|█████████▍| 1480/1570 [1:33:05<03:21, 2.24s/it]
94%|█████████▍| 1481/1570 [1:33:08<03:19, 2.24s/it]
94%|█████████▍| 1482/1570 [1:33:10<03:15, 2.22s/it]
94%|█████████▍| 1483/1570 [1:33:12<03:16, 2.26s/it]
95%|█████████▍| 1484/1570 [1:33:14<03:12, 2.24s/it]
95%|█████████▍| 1485/1570 [1:33:16<03:09, 2.23s/it]
95%|█████████▍| 1486/1570 [1:33:19<03:06, 2.22s/it]
95%|█████████▍| 1487/1570 [1:33:21<03:04, 2.23s/it]
95%|█████████▍| 1488/1570 [1:33:23<03:03, 2.23s/it]
95%|█████████▍| 1489/1570 [1:33:25<03:01, 2.24s/it]
95%|█████████▍| 1490/1570 [1:33:28<02:58, 2.23s/it]
95%|█████████▍| 1491/1570 [1:33:30<02:55, 2.22s/it]
95%|█████████▌| 1492/1570 [1:33:32<02:53, 2.22s/it]
95%|█████████▌| 1493/1570 [1:33:34<02:51, 2.22s/it]
95%|█████████▌| 1494/1570 [1:33:37<02:50, 2.24s/it]
95%|█████████▌| 1495/1570 [1:33:39<02:46, 2.23s/it]
95%|█████████▌| 1496/1570 [1:33:41<02:45, 2.23s/it]
95%|█████████▌| 1497/1570 [1:33:43<02:42, 2.23s/it]
95%|█████████▌| 1498/1570 [1:33:45<02:39, 2.21s/it]
95%|█████████▌| 1499/1570 [1:33:48<02:38, 2.23s/it]
96%|█████████▌| 1500/1570 [1:33:50<02:36, 2.24s/it]
{'loss': '0.00107', 'grad_norm': '0.04462', 'learning_rate': '6.033e-07', 'epoch': '4.779'}
+
96%|█████████▌| 1500/1570 [1:33:50<02:36, 2.24s/it]
96%|█████████▌| 1501/1570 [1:33:52<02:34, 2.25s/it]
96%|█████████▌| 1502/1570 [1:33:54<02:31, 2.23s/it]
96%|█████████▌| 1503/1570 [1:33:57<02:30, 2.25s/it]
96%|█████████▌| 1504/1570 [1:33:59<02:26, 2.23s/it]
96%|█████████▌| 1505/1570 [1:34:01<02:23, 2.21s/it]
96%|█████████▌| 1506/1570 [1:34:03<02:22, 2.22s/it]
96%|█████████▌| 1507/1570 [1:34:05<02:19, 2.22s/it]
96%|█████████▌| 1508/1570 [1:34:08<02:16, 2.20s/it]
96%|█████████▌| 1509/1570 [1:34:10<02:14, 2.21s/it]
96%|█████████▌| 1510/1570 [1:34:12<02:14, 2.23s/it]
96%|█████████▌| 1511/1570 [1:34:14<02:11, 2.23s/it]
96%|█████████▋| 1512/1570 [1:34:17<02:09, 2.23s/it]
96%|█████████▋| 1513/1570 [1:34:19<02:06, 2.22s/it]
96%|█████████▋| 1514/1570 [1:34:21<02:04, 2.23s/it]
96%|█████████▋| 1515/1570 [1:34:23<02:02, 2.22s/it]
97%|█████████▋| 1516/1570 [1:34:25<01:59, 2.22s/it]
97%|█████████▋| 1517/1570 [1:34:28<01:56, 2.21s/it]
97%|█████████▋| 1518/1570 [1:34:30<01:57, 2.26s/it]
97%|█████████▋| 1519/1570 [1:34:32<01:54, 2.24s/it]
97%|█████████▋| 1520/1570 [1:34:34<01:52, 2.25s/it]
{'loss': '0.003198', 'grad_norm': '0.009103', 'learning_rate': '3.599e-07', 'epoch': '4.842'}
+
97%|█████████▋| 1520/1570 [1:34:34<01:52, 2.25s/it]
97%|█████████▋| 1521/1570 [1:34:37<01:50, 2.26s/it]
97%|█████████▋| 1522/1570 [1:34:39<01:48, 2.26s/it]
97%|█████████▋| 1523/1570 [1:34:41<01:46, 2.26s/it]
97%|█████████▋| 1524/1570 [1:34:43<01:42, 2.23s/it]
97%|█████████▋| 1525/1570 [1:34:46<01:42, 2.27s/it]
97%|█████████▋| 1526/1570 [1:34:48<01:39, 2.26s/it]
97%|█████████▋| 1527/1570 [1:34:50<01:36, 2.26s/it]
97%|█████████▋| 1528/1570 [1:34:53<01:34, 2.26s/it]
97%|█████████▋| 1529/1570 [1:34:55<01:32, 2.25s/it]
97%|█████████▋| 1530/1570 [1:34:57<01:30, 2.26s/it]
98%|█████████▊| 1531/1570 [1:34:59<01:27, 2.25s/it]
98%|█████████▊| 1532/1570 [1:35:02<01:25, 2.26s/it]
98%|█████████▊| 1533/1570 [1:35:04<01:22, 2.24s/it]
98%|█████████▊| 1534/1570 [1:35:06<01:20, 2.23s/it]
98%|█████████▊| 1535/1570 [1:35:08<01:17, 2.23s/it]
98%|█████████▊| 1536/1570 [1:35:10<01:16, 2.25s/it]
98%|█████████▊| 1537/1570 [1:35:13<01:13, 2.23s/it]
98%|█████████▊| 1538/1570 [1:35:15<01:11, 2.23s/it]
98%|█████████▊| 1539/1570 [1:35:17<01:09, 2.23s/it]
98%|█████████▊| 1540/1570 [1:35:19<01:06, 2.22s/it]
{'loss': '0.0005366', 'grad_norm': '0.1368', 'learning_rate': '1.961e-07', 'epoch': '4.906'}
+
98%|█████████▊| 1540/1570 [1:35:19<01:06, 2.22s/it]
98%|█████████▊| 1541/1570 [1:35:22<01:05, 2.24s/it]
98%|█████████▊| 1542/1570 [1:35:24<01:03, 2.26s/it]
98%|█████████▊| 1543/1570 [1:35:26<01:01, 2.28s/it]
98%|█████████▊| 1544/1570 [1:35:28<00:58, 2.26s/it]
98%|█████████▊| 1545/1570 [1:35:31<00:56, 2.25s/it]
98%|█████████▊| 1546/1570 [1:35:33<00:53, 2.24s/it]
99%|█████████▊| 1547/1570 [1:35:35<00:51, 2.25s/it]
99%|█████████▊| 1548/1570 [1:35:37<00:49, 2.24s/it]
99%|█████████▊| 1549/1570 [1:35:40<00:46, 2.24s/it]
99%|█████████▊| 1550/1570 [1:35:42<00:44, 2.23s/it]
99%|█████████▉| 1551/1570 [1:35:44<00:42, 2.22s/it]
99%|█████████▉| 1552/1570 [1:35:46<00:39, 2.22s/it]
99%|█████████▉| 1553/1570 [1:35:48<00:37, 2.22s/it]
99%|█████████▉| 1554/1570 [1:35:51<00:35, 2.22s/it]
99%|█████████▉| 1555/1570 [1:35:53<00:33, 2.21s/it]
99%|█████████▉| 1556/1570 [1:35:55<00:31, 2.22s/it]
99%|█████████▉| 1557/1570 [1:35:57<00:28, 2.21s/it]
99%|█████████▉| 1558/1570 [1:36:00<00:26, 2.21s/it]
99%|█████████▉| 1559/1570 [1:36:02<00:24, 2.22s/it]
99%|█████████▉| 1560/1570 [1:36:04<00:22, 2.28s/it]
{'loss': '0.0007433', 'grad_norm': '0.05482', 'learning_rate': '1.121e-07', 'epoch': '4.97'}
+
99%|█████████▉| 1560/1570 [1:36:04<00:22, 2.28s/it]
99%|█████████▉| 1561/1570 [1:36:07<00:22, 2.48s/it]
99%|█████████▉| 1562/1570 [1:36:09<00:19, 2.43s/it]
100%|█████████▉| 1563/1570 [1:36:12<00:16, 2.36s/it]
100%|█████████▉| 1564/1570 [1:36:14<00:13, 2.30s/it]
100%|█████████▉| 1565/1570 [1:36:16<00:11, 2.33s/it]
100%|█████████▉| 1566/1570 [1:36:18<00:09, 2.31s/it]
100%|█████████▉| 1567/1570 [1:36:21<00:06, 2.27s/it]
100%|█████████▉| 1568/1570 [1:36:23<00:04, 2.36s/it]
100%|█████████▉| 1569/1570 [1:36:26<00:02, 2.43s/it]
100%|██████████| 1570/1570 [1:36:27<00:00, 2.03s/it][INFO|trainer.py:2668] 2026-08-22 09:49:32,682 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 09:49:32,682 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 09:49:32,683 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/sft/checkpoint-1570
+[INFO|configuration_utils.py:780] 2026-08-22 10:00:12,228 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 10:00:12,230 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 10:00:12,264 >> chat template saved in results/qwen3/sft/checkpoint-1570/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 10:00:12,264 >> tokenizer config file saved in results/qwen3/sft/checkpoint-1570/tokenizer_config.json
+[INFO|trainer.py:1823] 2026-08-22 10:00:12,868 >>
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
{'train_runtime': '6428', 'train_samples_per_second': '7.8', 'train_steps_per_second': '0.244', 'train_loss': '0.01663', 'epoch': '5'}
+
100%|██████████| 1570/1570 [1:47:07<00:00, 2.03s/it]
100%|██████████| 1570/1570 [1:47:07<00:00, 4.09s/it]
+[INFO|trainer.py:3815] 2026-08-22 10:00:13,242 >> Saving model checkpoint to results/qwen3/sft
+[INFO|configuration_utils.py:780] 2026-08-22 10:00:13,754 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 10:00:13,758 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 10:00:13,794 >> chat template saved in results/qwen3/sft/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 10:00:13,794 >> tokenizer config file saved in results/qwen3/sft/tokenizer_config.json
+[transformers] The following generation flags are not valid and may be ignored: ['temperature', 'top_p']. Set `TRANSFORMERS_VERBOSITY=info` for more details.
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+***** train metrics *****
+ epoch = 5.0
+ total_flos = 54682091GF
+ train_loss = 0.0166
+ train_runtime = 1:47:07.61
+ train_samples_per_second = 7.8
+ train_steps_per_second = 0.244
+[INFO|trainer.py:2668] 2026-08-22 10:00:13,909 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 10:00:13,909 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 10:00:13,909 >> Batch size = 16
+[WARNING|logging.py:340] 2026-08-22 10:00:14,023 >> The following generation flags are not valid and may be ignored: ['temperature', 'top_p'].
+[INFO|logging.py:355] 2026-08-22 10:00:14,024 >> - `temperature`: `do_sample` is not set to `True`. However, `temperature` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.
+- `top_p`: `do_sample` is not set to `True`. However, `top_p` is set to `0.7` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.
+If you're using a pretrained model, note that some of these attributes may be set through the model's `generation_config.json` file.
+[WARNING|utils.py:1642] 2026-08-22 10:00:14,026 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
0%| | 0/107 [00:00> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
2%|▏ | 2/107 [00:05<04:49, 2.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:00:25,211 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
3%|▎ | 3/107 [00:11<06:45, 3.90s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:00:30,718 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
4%|▎ | 4/107 [00:16<07:46, 4.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:00:36,313 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
5%|▍ | 5/107 [00:22<08:17, 4.88s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:00:41,831 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
6%|▌ | 6/107 [00:26<08:11, 4.86s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:00:46,670 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 7/107 [00:32<08:25, 5.05s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:00:52,124 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 8/107 [00:37<08:32, 5.18s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:00:57,572 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
8%|▊ | 9/107 [00:43<08:51, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:01:03,558 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
9%|▉ | 10/107 [00:49<08:54, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:01:09,253 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
10%|█ | 11/107 [00:54<08:24, 5.25s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:01:13,919 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
11%|█ | 12/107 [00:59<08:33, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:01:19,658 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
12%|█▏ | 13/107 [01:05<08:34, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:01:25,296 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
13%|█▎ | 14/107 [01:10<08:27, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:01:30,703 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
14%|█▍ | 15/107 [01:16<08:26, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:01:36,328 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
15%|█▍ | 16/107 [01:22<08:24, 5.54s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:01:41,949 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
16%|█▌ | 17/107 [01:28<08:27, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:01:47,827 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
17%|█▋ | 18/107 [01:33<08:22, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:01:53,495 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
18%|█▊ | 19/107 [01:39<08:10, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:01:58,892 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
19%|█▊ | 20/107 [01:44<08:03, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:02:04,399 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
20%|█▉ | 21/107 [01:49<07:41, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:02:09,342 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██ | 22/107 [01:55<07:48, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:02:15,170 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██▏ | 23/107 [02:01<07:44, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:02:20,765 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
22%|██▏ | 24/107 [02:06<07:38, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:02:26,252 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
23%|██▎ | 25/107 [02:13<07:56, 5.81s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:02:32,742 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
24%|██▍ | 26/107 [02:18<07:50, 5.81s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:02:38,559 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
25%|██▌ | 27/107 [02:24<07:39, 5.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:02:44,148 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
26%|██▌ | 28/107 [02:30<07:32, 5.73s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:02:49,829 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
27%|██▋ | 29/107 [02:36<07:34, 5.83s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:02:55,901 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
28%|██▊ | 30/107 [02:41<07:21, 5.73s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:03:01,407 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
29%|██▉ | 31/107 [02:46<06:51, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:03:06,068 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
30%|██▉ | 32/107 [02:51<06:47, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:03:11,569 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
31%|███ | 33/107 [02:57<06:53, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:03:17,489 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
32%|███▏ | 34/107 [03:04<07:06, 5.85s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:03:23,948 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
33%|███▎ | 35/107 [03:10<07:01, 5.86s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:03:29,839 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
34%|███▎ | 36/107 [03:15<06:49, 5.76s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:03:35,377 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
35%|███▍ | 37/107 [03:20<06:26, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:03:40,327 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▌ | 38/107 [03:26<06:30, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:03:46,316 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▋ | 39/107 [03:31<06:17, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:03:51,611 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
37%|███▋ | 40/107 [03:37<06:13, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:03:57,265 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
38%|███▊ | 41/107 [03:43<06:14, 5.67s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:04:03,150 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
39%|███▉ | 42/107 [03:49<06:21, 5.87s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:04:09,477 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
40%|████ | 43/107 [03:55<06:20, 5.94s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:04:15,587 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
41%|████ | 44/107 [04:01<06:06, 5.82s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:04:21,130 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
42%|████▏ | 45/107 [04:06<05:49, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:04:26,330 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
43%|████▎ | 46/107 [04:12<05:46, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:04:32,124 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
44%|████▍ | 47/107 [04:18<05:54, 5.91s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:04:38,561 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
45%|████▍ | 48/107 [04:24<05:48, 5.90s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:04:44,439 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
46%|████▌ | 49/107 [04:30<05:35, 5.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:04:49,937 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
47%|████▋ | 50/107 [04:36<05:31, 5.82s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:04:55,855 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
48%|████▊ | 51/107 [04:43<05:53, 6.31s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:05:03,303 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
49%|████▊ | 52/107 [04:48<05:30, 6.01s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:05:08,613 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|████▉ | 53/107 [04:54<05:23, 5.99s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:05:14,570 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|█████ | 54/107 [05:00<05:07, 5.80s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:05:19,913 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
51%|█████▏ | 55/107 [05:05<05:00, 5.79s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:05:25,673 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
52%|█████▏ | 56/107 [05:11<04:56, 5.82s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:05:31,560 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
53%|█████▎ | 57/107 [05:17<04:49, 5.79s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:05:37,284 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
54%|█████▍ | 58/107 [05:23<04:38, 5.69s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:05:42,748 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
55%|█████▌ | 59/107 [05:28<04:30, 5.63s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:05:48,219 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
56%|█████▌ | 60/107 [05:33<04:22, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:05:53,678 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
57%|█████▋ | 61/107 [05:38<04:06, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:05:58,559 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
58%|█████▊ | 62/107 [05:44<04:01, 5.36s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:06:03,908 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
59%|█████▉ | 63/107 [05:49<03:58, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:06:09,462 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
60%|█████▉ | 64/107 [05:55<03:52, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:06:14,805 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
61%|██████ | 65/107 [06:00<03:45, 5.36s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:06:20,092 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
62%|██████▏ | 66/107 [06:05<03:40, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:06:25,522 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
63%|██████▎ | 67/107 [06:11<03:36, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:06:31,002 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▎ | 68/107 [06:16<03:25, 5.26s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:06:35,907 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▍ | 69/107 [06:21<03:19, 5.24s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:06:41,099 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
65%|██████▌ | 70/107 [06:27<03:23, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:06:47,174 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
66%|██████▋ | 71/107 [06:33<03:21, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:06:53,011 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
67%|██████▋ | 72/107 [06:39<03:17, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:06:58,774 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
68%|██████▊ | 73/107 [06:45<03:15, 5.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:07:04,756 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
69%|██████▉ | 74/107 [06:50<03:10, 5.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:07:10,603 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
70%|███████ | 75/107 [06:56<03:01, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:07:15,992 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
71%|███████ | 76/107 [07:02<02:56, 5.70s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:07:21,773 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
72%|███████▏ | 77/107 [07:07<02:49, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:07:27,307 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
73%|███████▎ | 78/107 [07:13<02:46, 5.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:07:33,289 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
74%|███████▍ | 79/107 [07:19<02:39, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:07:38,812 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
75%|███████▍ | 80/107 [07:25<02:35, 5.76s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:07:44,745 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
76%|███████▌ | 81/107 [07:32<02:43, 6.27s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:07:52,224 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
77%|███████▋ | 82/107 [07:38<02:31, 6.05s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:07:57,767 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
78%|███████▊ | 83/107 [07:43<02:19, 5.82s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:08:03,055 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▊ | 84/107 [07:48<02:09, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:08:08,281 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▉ | 85/107 [07:53<01:56, 5.28s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:08:12,710 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
80%|████████ | 86/107 [07:58<01:53, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:08:18,373 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
81%|████████▏ | 87/107 [08:03<01:43, 5.17s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:08:23,018 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
82%|████████▏ | 88/107 [08:08<01:39, 5.25s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:08:28,445 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
83%|████████▎ | 89/107 [08:15<01:42, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:08:35,121 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
84%|████████▍ | 90/107 [08:20<01:35, 5.60s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:08:40,561 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
85%|████████▌ | 91/107 [08:25<01:27, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:08:45,697 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
86%|████████▌ | 92/107 [08:31<01:20, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:08:50,859 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
87%|████████▋ | 93/107 [08:36<01:16, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:08:56,468 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
88%|████████▊ | 94/107 [08:42<01:11, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:09:02,129 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
89%|████████▉ | 95/107 [08:47<01:05, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:09:07,540 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
90%|████████▉ | 96/107 [08:53<00:59, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:09:12,814 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
91%|█████████ | 97/107 [08:58<00:55, 5.54s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:09:18,633 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
92%|█████████▏| 98/107 [09:04<00:50, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:09:24,328 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 99/107 [09:10<00:45, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:09:30,236 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 100/107 [09:15<00:39, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:09:35,567 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
94%|█████████▍| 101/107 [09:21<00:34, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:09:41,491 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
95%|█████████▌| 102/107 [09:27<00:29, 5.82s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:09:47,636 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
96%|█████████▋| 103/107 [09:33<00:22, 5.73s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:09:53,157 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
97%|█████████▋| 104/107 [09:39<00:17, 5.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:09:58,962 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
98%|█████████▊| 105/107 [09:44<00:11, 5.73s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:10:04,653 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
99%|█████████▉| 106/107 [09:50<00:05, 5.69s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 10:10:10,304 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
100%|██████████| 107/107 [09:56<00:00, 5.81s/it]
100%|██████████| 107/107 [09:57<00:00, 5.59s/it]
+***** eval metrics *****
+ epoch = 5.0
+ eval_generator_count = 1712.0
+ eval_generator_exact_match = 96.9626
+ eval_generator_rouge1 = 99.8875
+ eval_generator_rouge2 = 99.8292
+ eval_generator_rougeL = 99.735
+ eval_loss = 0.0033
+ eval_runtime = 0:10:03.56
+ eval_samples_per_second = 5.656
+ eval_selector_accuracy = 99.2407
+ eval_selector_count = 1712.0
+ eval_steps_per_second = 0.177
+[INFO|modelcard.py:263] 2026-08-22 10:10:17,476 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
diff --git a/qwen3/run_all_logs/20260822_031516/srkl.log b/qwen3/run_all_logs/20260822_031516/srkl.log
new file mode 100644
index 0000000000000000000000000000000000000000..e1603745cfceb1f22e045bbae465ad60cbd9653f
--- /dev/null
+++ b/qwen3/run_all_logs/20260822_031516/srkl.log
@@ -0,0 +1,2100 @@
+W0822 17:15:42.806000 1398113 torch/distributed/run.py:803]
+W0822 17:15:42.806000 1398113 torch/distributed/run.py:803] *****************************************
+W0822 17:15:42.806000 1398113 torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+W0822 17:15:42.806000 1398113 torch/distributed/run.py:803] *****************************************
+[INFO|2026-08-22 17:15:52] llamafactory.hparams.parser:651 >> Process rank: 1, world size: 2, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16
+[INFO|2026-08-22 17:15:52] llamafactory.hparams.parser:144 >> Set `ddp_find_unused_parameters` to False in DDP training since LoRA is enabled.
+[INFO|2026-08-22 17:15:52] llamafactory.hparams.parser:651 >> Process rank: 0, world size: 2, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16
+Starting srkl with LlamaFactory and DeepSpeed=True.
+[INFO|configuration_utils.py:780] 2026-08-22 17:15:53,854 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 17:15:53,859 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 17:15:56,979 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 17:15:56,982 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 17:15:57,221 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 17:15:57,224 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 17:15:58] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_train.jsonl...
+[INFO|2026-08-22 17:15:59] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_eval.jsonl...
+training example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 1947, 19029, 10592, 311, 35592, 304, 279, 17328, 1947, 1707, 429, 9482, 1573, 279, 1042, 220, 16, 21, 15, 15, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 9286, 38050, 756, 414, 330, 13193, 788, 341, 286, 330, 408, 14645, 788, 330, 68689, 756, 286, 330, 2468, 14645, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 37525, 756, 414, 330, 13193, 788, 5613, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 18098, 287, 756, 414, 330, 13193, 788, 341, 286, 330, 11141, 3575, 34043, 788, 330, 16468, 756, 286, 330, 37375, 14645, 788, 330, 68689, 698, 414, 456, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 9286, 38050, 756, 414, 330, 1313, 788, 330, 49087, 2354, 698, 262, 1153, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 18098, 287, 756, 414, 330, 5657, 788, 330, 37525, 756, 414, 330, 1313, 788, 330, 4648, 37525, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of art movements linked to paintings in the genre art style that ended before the year 1600?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "ArtMovement",
+ "properties": {
+ "end_year": "INTEGER",
+ "start_year": "INTEGER"
+ }
+ },
+ {
+ "label": "Genre",
+ "properties": {}
+ },
+ {
+ "label": "Painting",
+ "properties": {
+ "country_of_origin": "STRING",
+ "creation_year": "INTEGER"
+ }
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "ArtMovement",
+ "type": "associatedWith"
+ },
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "Genre",
+ "type": "hasGenre"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+eval example:
+input_ids:
+[151644, 8948, 198, 2610, 525, 264, 17748, 28082, 11361, 28358, 382, 7771, 3383, 374, 311, 6923, 264, 2697, 17748, 28082, 3239, 429, 11253, 279, 1196, 594, 5810, 4128, 3405, 1667, 1172, 279, 3897, 4771, 10802, 382, 26008, 510, 12, 5443, 1172, 9201, 11, 11871, 11, 323, 5888, 429, 3000, 304, 279, 10802, 624, 12, 3155, 537, 17023, 894, 10802, 5424, 624, 12, 3411, 1172, 279, 5043, 4362, 311, 4226, 279, 3405, 624, 12, 1416, 42328, 525, 3204, 11, 990, 59086, 979, 8311, 624, 12, 1416, 279, 3405, 7460, 51629, 11, 28273, 11, 476, 32894, 11, 990, 279, 4396, 17748, 28082, 49098, 624, 12, 29279, 279, 3239, 374, 83187, 531, 2673, 2697, 382, 5097, 3561, 510, 515, 220, 330, 11130, 28082, 788, 330, 785, 4583, 17748, 28082, 3239, 698, 630, 5598, 1172, 279, 4718, 1633, 323, 4302, 770, 624, 67, 10111, 7274, 1007, 151645, 198, 151644, 872, 198, 52428, 510, 3838, 525, 279, 5036, 315, 9893, 1483, 304, 2987, 279, 220, 17, 15, 17, 16, 55218, 18144, 73379, 476, 279, 23007, 24847, 7755, 9257, 36331, 1939, 3540, 35839, 510, 515, 220, 330, 20008, 788, 2278, 262, 341, 414, 330, 1502, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 13193, 788, 341, 286, 330, 1028, 788, 330, 7097, 756, 286, 330, 31309, 788, 330, 22852, 756, 286, 330, 4082, 3575, 2259, 26286, 788, 330, 68689, 756, 286, 330, 4082, 3575, 1243, 89827, 788, 330, 68689, 698, 414, 456, 262, 1153, 262, 341, 414, 330, 1502, 788, 330, 28214, 756, 414, 330, 13193, 788, 5613, 262, 456, 220, 3211, 220, 330, 85824, 788, 2278, 262, 341, 414, 330, 13193, 788, 14573, 414, 330, 2427, 788, 330, 51, 841, 380, 28602, 756, 414, 330, 5657, 788, 330, 28214, 756, 414, 330, 1313, 788, 330, 30210, 1047, 698, 262, 456, 220, 5133, 630, 31115, 264, 17748, 28082, 3239, 429, 11253, 279, 3405, 1667, 1172, 279, 3897, 10802, 624, 5598, 1172, 279, 4718, 1633, 304, 279, 2567, 3561, 13, 151645, 198, 151644, 77091, 198, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+inputs:
+<|im_start|>system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of weapons used in either the 2021 Baghdad suicide bombings or the Botroseya Church bombing?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "TerroristAttack",
+ "properties": {
+ "date": "DATE",
+ "locations": "LIST",
+ "number_of_deaths": "INTEGER",
+ "number_of_injuries": "INTEGER"
+ }
+ },
+ {
+ "label": "Weapon",
+ "properties": {}
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "TerroristAttack",
+ "target": "Weapon",
+ "type": "employs"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+[INFO|configuration_utils.py:780] 2026-08-22 17:16:03,195 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 17:16:03,197 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 17:16:03] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training.
+[INFO|modeling_utils.py:769] 2026-08-22 17:16:03,915 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/model.safetensors
+[INFO|modeling_utils.py:839] 2026-08-22 17:16:03,915 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 17:16:03,917 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": false
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 17:16:04,613 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.6,
+ "top_k": 20,
+ "top_p": 0.95
+}
+
+[INFO|2026-08-22 17:16:04] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled.
+[INFO|2026-08-22 17:16:04] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-22 17:16:04] llamafactory.model.adapter:144 >> Upcasting trainable params to float32.
+[INFO|2026-08-22 17:16:04] llamafactory.model.adapter:144 >> Fine-tuning method: LoRA
+[INFO|2026-08-22 17:16:04] llamafactory.model.model_utils.misc:144 >> Found linear modules: v_proj,gate_proj,o_proj,down_proj,k_proj,up_proj,q_proj
+[INFO|2026-08-22 17:16:05] llamafactory.model.loader:144 >> trainable params: 20,185,088 || all params: 616,235,008 || trainable%: 3.2756
+[INFO|configuration_utils.py:780] 2026-08-22 17:16:05,971 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 17:16:05,974 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+
Loading weights: 0%| | 0/311 [00:00> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 17:16:09,108 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 17:16:09,345 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 17:16:09,348 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 17:16:11,696 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 17:16:11,699 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 17:16:11] llamafactory.model.model_utils.kv_cache:144 >> KV cache is enabled for faster generation.
+[INFO|modeling_utils.py:769] 2026-08-22 17:16:11,935 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model.safetensors.index.json
+[INFO|modeling_utils.py:839] 2026-08-22 17:16:11,936 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 17:16:11,937 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": true
+}
+
+
Loading weights: 0%| | 0/398 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 17:16:14,155 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.7,
+ "top_k": 20,
+ "top_p": 0.8
+}
+
+[INFO|2026-08-22 17:16:14] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+
Loading weights: 0%| | 0/398 [00:00> Merged 1 adapter(s).
+[INFO|2026-08-22 17:16:16] llamafactory.model.adapter:144 >> Loaded adapter(s): results/qwen3/teacher_lora
+[INFO|2026-08-22 17:16:16] llamafactory.model.loader:144 >> all params: 4,022,468,096
+[INFO|2026-08-22 17:16:16] llamafactory.train.trainer_utils:144 >> Created reference model from Qwen/Qwen3-4B-Instruct-2507
+[transformers] The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}.
+begin bf16_optimizer
+MA 8.71 GB Max_MA 9.15 GB CA 9.37 GB Max_CA 9 GB
+CPU Virtual Memory: used = 16.25 GB, percent = 3.6%
+end bf16_ optimizer
+MA 8.71 GB Max_MA 8.71 GB CA 9.37 GB Max_CA 9 GB
+CPU Virtual Memory: used = 16.1 GB, percent = 3.6%
+[WARNING|trainer_utils.py:1238] 2026-08-22 17:16:18,273 >> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}.
+Gradient accumulation steps mismatch: GradientAccumulationPlugin has 1, DeepSpeed config has 8. Using DeepSpeed's value.
+[INFO|trainer.py:1469] 2026-08-22 17:16:18,868 >> ***** Running training *****
+[INFO|trainer.py:1470] 2026-08-22 17:16:18,868 >> Num examples = 10,027
+[INFO|trainer.py:1471] 2026-08-22 17:16:18,868 >> Num Epochs = 5
+[INFO|trainer.py:1472] 2026-08-22 17:16:18,868 >> Num update steps per epoch = 314
+[INFO|trainer.py:1473] 2026-08-22 17:16:18,868 >> Instantaneous batch size per device = 2
+[INFO|trainer.py:1476] 2026-08-22 17:16:18,868 >> Total train batch size (w. parallel, distributed & accumulation) = 32
+[INFO|trainer.py:1477] 2026-08-22 17:16:18,868 >> Gradient Accumulation steps = 8
+[INFO|trainer.py:1478] 2026-08-22 17:16:18,868 >> Total optimization steps = 1,570
+[INFO|trainer.py:1479] 2026-08-22 17:16:18,872 >> Number of trainable parameters = 20,185,088
+
0%| | 0/1570 [00:00>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 17:32:40,661 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 17:32:40,661 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/srkl/checkpoint-314
+[INFO|configuration_utils.py:780] 2026-08-22 17:42:56,093 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 17:42:56,094 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 17:42:56,129 >> chat template saved in results/qwen3/srkl/checkpoint-314/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 17:42:56,130 >> tokenizer config file saved in results/qwen3/srkl/checkpoint-314/tokenizer_config.json
+
20%|██ | 315/1570 [26:41<65:21:57, 187.50s/it]
20%|██ | 316/1570 [26:44<46:01:40, 132.14s/it]
20%|██ | 317/1570 [26:47<32:30:01, 93.38s/it]
20%|██ | 318/1570 [26:50<23:02:46, 66.27s/it]
20%|██ | 319/1570 [26:53<16:25:37, 47.27s/it]
20%|██ | 320/1570 [26:55<11:47:42, 33.97s/it]
{'loss': '0.01099', 'grad_norm': '0.6877', 'learning_rate': '9.016e-05', 'epoch': '1.019'}
+
20%|██ | 320/1570 [26:55<11:47:42, 33.97s/it]
20%|██ | 321/1570 [26:58<8:33:29, 24.67s/it]
21%|██ | 322/1570 [27:01<6:17:05, 18.13s/it]
21%|██ | 323/1570 [27:04<4:42:00, 13.57s/it]
21%|██ | 324/1570 [27:07<3:35:28, 10.38s/it]
21%|██ | 325/1570 [27:10<2:48:56, 8.14s/it]
21%|██ | 326/1570 [27:13<2:15:57, 6.56s/it]
21%|██ | 327/1570 [27:16<1:53:08, 5.46s/it]
21%|██ | 328/1570 [27:19<1:37:17, 4.70s/it]
21%|██ | 329/1570 [27:22<1:26:15, 4.17s/it]
21%|██ | 330/1570 [27:25<1:18:47, 3.81s/it]
21%|██ | 331/1570 [27:28<1:13:17, 3.55s/it]
21%|██ | 332/1570 [27:31<1:09:15, 3.36s/it]
21%|██ | 333/1570 [27:33<1:06:36, 3.23s/it]
21%|██▏ | 334/1570 [27:36<1:04:54, 3.15s/it]
21%|██▏ | 335/1570 [27:39<1:03:29, 3.08s/it]
21%|██▏ | 336/1570 [27:42<1:02:23, 3.03s/it]
21%|██▏ | 337/1570 [27:45<1:01:29, 2.99s/it]
22%|██▏ | 338/1570 [27:48<1:00:47, 2.96s/it]
22%|██▏ | 339/1570 [27:51<1:00:26, 2.95s/it]
22%|██▏ | 340/1570 [27:54<1:00:37, 2.96s/it]
{'loss': '0.0052', 'grad_norm': '0.1419', 'learning_rate': '8.894e-05', 'epoch': '1.083'}
+
22%|██▏ | 340/1570 [27:54<1:00:37, 2.96s/it]
22%|██▏ | 341/1570 [27:57<1:00:30, 2.95s/it]
22%|██▏ | 342/1570 [28:00<1:00:31, 2.96s/it]
22%|██▏ | 343/1570 [28:03<1:00:12, 2.94s/it]
22%|██▏ | 344/1570 [28:06<59:50, 2.93s/it]
22%|██▏ | 345/1570 [28:09<59:43, 2.93s/it]
22%|██▏ | 346/1570 [28:12<59:29, 2.92s/it]
22%|██▏ | 347/1570 [28:14<59:37, 2.93s/it]
22%|██▏ | 348/1570 [28:17<59:30, 2.92s/it]
22%|██▏ | 349/1570 [28:20<59:25, 2.92s/it]
22%|██▏ | 350/1570 [28:23<59:13, 2.91s/it]
22%|██▏ | 351/1570 [28:26<59:04, 2.91s/it]
22%|██▏ | 352/1570 [28:29<58:58, 2.91s/it]
22%|██▏ | 353/1570 [28:32<58:57, 2.91s/it]
23%|██▎ | 354/1570 [28:35<58:59, 2.91s/it]
23%|██▎ | 355/1570 [28:38<59:01, 2.92s/it]
23%|██▎ | 356/1570 [28:41<58:55, 2.91s/it]
23%|██▎ | 357/1570 [28:44<58:44, 2.91s/it]
23%|██▎ | 358/1570 [28:46<58:43, 2.91s/it]
23%|██▎ | 359/1570 [28:49<58:40, 2.91s/it]
23%|██▎ | 360/1570 [28:52<58:24, 2.90s/it]
{'loss': '0.004499', 'grad_norm': '0.1092', 'learning_rate': '8.766e-05', 'epoch': '1.147'}
+
23%|██▎ | 360/1570 [28:52<58:24, 2.90s/it]
23%|██▎ | 361/1570 [28:55<58:30, 2.90s/it]
23%|██▎ | 362/1570 [28:58<58:31, 2.91s/it]
23%|██▎ | 363/1570 [29:01<58:25, 2.90s/it]
23%|██▎ | 364/1570 [29:04<58:32, 2.91s/it]
23%|██▎ | 365/1570 [29:07<58:22, 2.91s/it]
23%|██▎ | 366/1570 [29:10<58:23, 2.91s/it]
23%|██▎ | 367/1570 [29:13<58:12, 2.90s/it]
23%|██▎ | 368/1570 [29:15<58:12, 2.91s/it]
24%|██▎ | 369/1570 [29:18<58:23, 2.92s/it]
24%|██▎ | 370/1570 [29:21<58:20, 2.92s/it]
24%|██▎ | 371/1570 [29:24<58:15, 2.92s/it]
24%|██▎ | 372/1570 [29:27<58:10, 2.91s/it]
24%|██▍ | 373/1570 [29:30<58:04, 2.91s/it]
24%|██▍ | 374/1570 [29:33<57:53, 2.90s/it]
24%|██▍ | 375/1570 [29:36<58:10, 2.92s/it]
24%|██▍ | 376/1570 [29:39<58:32, 2.94s/it]
24%|██▍ | 377/1570 [29:42<58:26, 2.94s/it]
24%|██▍ | 378/1570 [29:45<58:19, 2.94s/it]
24%|██▍ | 379/1570 [29:48<58:25, 2.94s/it]
24%|██▍ | 380/1570 [29:51<58:07, 2.93s/it]
{'loss': '0.005003', 'grad_norm': '0.2007', 'learning_rate': '8.631e-05', 'epoch': '1.211'}
+
24%|██▍ | 380/1570 [29:51<58:07, 2.93s/it]
24%|██▍ | 381/1570 [29:54<58:06, 2.93s/it]
24%|██▍ | 382/1570 [29:57<58:04, 2.93s/it]
24%|██▍ | 383/1570 [29:59<58:09, 2.94s/it]
24%|██▍ | 384/1570 [30:02<58:13, 2.95s/it]
25%|██▍ | 385/1570 [30:05<57:51, 2.93s/it]
25%|██▍ | 386/1570 [30:08<57:34, 2.92s/it]
25%|██▍ | 387/1570 [30:11<57:40, 2.92s/it]
25%|██▍ | 388/1570 [30:14<57:29, 2.92s/it]
25%|██▍ | 389/1570 [30:17<57:27, 2.92s/it]
25%|██▍ | 390/1570 [30:20<57:21, 2.92s/it]
25%|██▍ | 391/1570 [30:23<57:22, 2.92s/it]
25%|██▍ | 392/1570 [30:26<57:25, 2.93s/it]
25%|██▌ | 393/1570 [30:29<57:14, 2.92s/it]
25%|██▌ | 394/1570 [30:32<57:22, 2.93s/it]
25%|██▌ | 395/1570 [30:35<57:19, 2.93s/it]
25%|██▌ | 396/1570 [30:37<57:19, 2.93s/it]
25%|██▌ | 397/1570 [30:40<57:12, 2.93s/it]
25%|██▌ | 398/1570 [30:43<56:58, 2.92s/it]
25%|██▌ | 399/1570 [30:46<56:45, 2.91s/it]
25%|██▌ | 400/1570 [30:49<56:44, 2.91s/it]
{'loss': '0.006419', 'grad_norm': '0.08986', 'learning_rate': '8.491e-05', 'epoch': '1.274'}
+
25%|██▌ | 400/1570 [30:49<56:44, 2.91s/it]
26%|██▌ | 401/1570 [30:52<56:52, 2.92s/it]
26%|██▌ | 402/1570 [30:55<56:24, 2.90s/it]
26%|██▌ | 403/1570 [30:58<56:46, 2.92s/it]
26%|██▌ | 404/1570 [31:01<56:45, 2.92s/it]
26%|██▌ | 405/1570 [31:04<56:56, 2.93s/it]
26%|██▌ | 406/1570 [31:07<57:13, 2.95s/it]
26%|██▌ | 407/1570 [31:10<57:24, 2.96s/it]
26%|██▌ | 408/1570 [31:13<56:56, 2.94s/it]
26%|██▌ | 409/1570 [31:16<57:04, 2.95s/it]
26%|██▌ | 410/1570 [31:18<56:50, 2.94s/it]
26%|██▌ | 411/1570 [31:21<56:40, 2.93s/it]
26%|██▌ | 412/1570 [31:24<56:32, 2.93s/it]
26%|██▋ | 413/1570 [31:27<56:34, 2.93s/it]
26%|██▋ | 414/1570 [31:30<56:44, 2.95s/it]
26%|██▋ | 415/1570 [31:33<56:14, 2.92s/it]
26%|██▋ | 416/1570 [31:36<56:30, 2.94s/it]
27%|██▋ | 417/1570 [31:39<56:17, 2.93s/it]
27%|██▋ | 418/1570 [31:42<56:18, 2.93s/it]
27%|██▋ | 419/1570 [31:45<56:06, 2.92s/it]
27%|██▋ | 420/1570 [31:48<56:02, 2.92s/it]
{'loss': '0.007294', 'grad_norm': '0.1942', 'learning_rate': '8.345e-05', 'epoch': '1.338'}
+
27%|██▋ | 420/1570 [31:48<56:02, 2.92s/it]
27%|██▋ | 421/1570 [31:51<56:24, 2.95s/it]
27%|██▋ | 422/1570 [31:54<56:00, 2.93s/it]
27%|██▋ | 423/1570 [31:57<56:05, 2.93s/it]
27%|██▋ | 424/1570 [32:00<55:59, 2.93s/it]
27%|██▋ | 425/1570 [32:02<56:01, 2.94s/it]
27%|██▋ | 426/1570 [32:05<55:48, 2.93s/it]
27%|██▋ | 427/1570 [32:08<55:40, 2.92s/it]
27%|██▋ | 428/1570 [32:11<55:33, 2.92s/it]
27%|██▋ | 429/1570 [32:14<55:42, 2.93s/it]
27%|██▋ | 430/1570 [32:17<55:45, 2.93s/it]
27%|██▋ | 431/1570 [32:20<55:50, 2.94s/it]
28%|██▊ | 432/1570 [32:23<55:39, 2.93s/it]
28%|██▊ | 433/1570 [32:26<55:26, 2.93s/it]
28%|██▊ | 434/1570 [32:29<55:14, 2.92s/it]
28%|██▊ | 435/1570 [32:32<55:17, 2.92s/it]
28%|██▊ | 436/1570 [32:35<55:18, 2.93s/it]
28%|██▊ | 437/1570 [32:38<55:21, 2.93s/it]
28%|██▊ | 438/1570 [32:41<55:24, 2.94s/it]
28%|██▊ | 439/1570 [32:43<55:13, 2.93s/it]
28%|██▊ | 440/1570 [32:46<55:24, 2.94s/it]
{'loss': '0.003503', 'grad_norm': '0.06895', 'learning_rate': '8.194e-05', 'epoch': '1.402'}
+
28%|██▊ | 440/1570 [32:46<55:24, 2.94s/it]
28%|██▊ | 441/1570 [32:49<55:26, 2.95s/it]
28%|██▊ | 442/1570 [32:52<55:29, 2.95s/it]
28%|██▊ | 443/1570 [32:55<55:15, 2.94s/it]
28%|██▊ | 444/1570 [32:58<55:14, 2.94s/it]
28%|██▊ | 445/1570 [33:01<55:11, 2.94s/it]
28%|██▊ | 446/1570 [33:04<55:26, 2.96s/it]
28%|██▊ | 447/1570 [33:07<55:23, 2.96s/it]
29%|██▊ | 448/1570 [33:10<54:56, 2.94s/it]
29%|██▊ | 449/1570 [33:13<55:18, 2.96s/it]
29%|██▊ | 450/1570 [33:16<55:06, 2.95s/it]
29%|██▊ | 451/1570 [33:19<55:01, 2.95s/it]
29%|██▉ | 452/1570 [33:22<54:59, 2.95s/it]
29%|██▉ | 453/1570 [33:25<54:53, 2.95s/it]
29%|██▉ | 454/1570 [33:28<54:37, 2.94s/it]
29%|██▉ | 455/1570 [33:31<54:34, 2.94s/it]
29%|██▉ | 456/1570 [33:34<54:34, 2.94s/it]
29%|██▉ | 457/1570 [33:36<54:11, 2.92s/it]
29%|██▉ | 458/1570 [33:39<54:06, 2.92s/it]
29%|██▉ | 459/1570 [33:42<53:55, 2.91s/it]
29%|██▉ | 460/1570 [33:45<53:36, 2.90s/it]
{'loss': '0.005547', 'grad_norm': '0.6073', 'learning_rate': '8.037e-05', 'epoch': '1.466'}
+
29%|██▉ | 460/1570 [33:45<53:36, 2.90s/it]
29%|██▉ | 461/1570 [33:48<53:32, 2.90s/it]
29%|██▉ | 462/1570 [33:51<53:44, 2.91s/it]
29%|██▉ | 463/1570 [33:54<53:54, 2.92s/it]
30%|██▉ | 464/1570 [33:57<53:57, 2.93s/it]
30%|██▉ | 465/1570 [34:00<53:42, 2.92s/it]
30%|██▉ | 466/1570 [34:03<53:20, 2.90s/it]
30%|██▉ | 467/1570 [34:06<53:28, 2.91s/it]
30%|██▉ | 468/1570 [34:08<53:28, 2.91s/it]
30%|██▉ | 469/1570 [34:11<53:25, 2.91s/it]
30%|██▉ | 470/1570 [34:14<53:15, 2.91s/it]
30%|███ | 471/1570 [34:17<53:20, 2.91s/it]
30%|███ | 472/1570 [34:20<53:19, 2.91s/it]
30%|███ | 473/1570 [34:23<53:21, 2.92s/it]
30%|███ | 474/1570 [34:26<52:57, 2.90s/it]
30%|███ | 475/1570 [34:29<53:09, 2.91s/it]
30%|███ | 476/1570 [34:32<53:18, 2.92s/it]
30%|███ | 477/1570 [34:35<53:22, 2.93s/it]
30%|███ | 478/1570 [34:38<53:07, 2.92s/it]
31%|███ | 479/1570 [34:41<53:15, 2.93s/it]
31%|███ | 480/1570 [34:43<53:07, 2.92s/it]
{'loss': '0.003696', 'grad_norm': '0.02789', 'learning_rate': '7.876e-05', 'epoch': '1.53'}
+
31%|███ | 480/1570 [34:43<53:07, 2.92s/it]
31%|███ | 481/1570 [34:46<53:03, 2.92s/it]
31%|███ | 482/1570 [34:49<52:55, 2.92s/it]
31%|███ | 483/1570 [34:52<52:47, 2.91s/it]
31%|███ | 484/1570 [34:55<52:37, 2.91s/it]
31%|███ | 485/1570 [34:58<52:41, 2.91s/it]
31%|███ | 486/1570 [35:01<52:40, 2.92s/it]
31%|███ | 487/1570 [35:04<52:45, 2.92s/it]
31%|███ | 488/1570 [35:07<52:34, 2.92s/it]
31%|███ | 489/1570 [35:10<52:25, 2.91s/it]
31%|███ | 490/1570 [35:13<52:32, 2.92s/it]
31%|███▏ | 491/1570 [35:16<52:21, 2.91s/it]
31%|███▏ | 492/1570 [35:18<52:28, 2.92s/it]
31%|███▏ | 493/1570 [35:21<52:27, 2.92s/it]
31%|███▏ | 494/1570 [35:24<52:17, 2.92s/it]
32%|███▏ | 495/1570 [35:27<52:18, 2.92s/it]
32%|███▏ | 496/1570 [35:30<52:02, 2.91s/it]
32%|███▏ | 497/1570 [35:33<52:03, 2.91s/it]
32%|███▏ | 498/1570 [35:36<52:25, 2.93s/it]
32%|███▏ | 499/1570 [35:39<52:24, 2.94s/it]
32%|███▏ | 500/1570 [35:42<52:19, 2.93s/it]
{'loss': '0.00627', 'grad_norm': '0.09828', 'learning_rate': '7.71e-05', 'epoch': '1.594'}
+
32%|███▏ | 500/1570 [35:42<52:19, 2.93s/it]
32%|███▏ | 501/1570 [35:45<52:06, 2.92s/it]
32%|███▏ | 502/1570 [35:48<52:18, 2.94s/it]
32%|███▏ | 503/1570 [35:51<52:25, 2.95s/it]
32%|███▏ | 504/1570 [35:54<52:29, 2.95s/it]
32%|███▏ | 505/1570 [35:57<52:34, 2.96s/it]
32%|███▏ | 506/1570 [36:00<52:18, 2.95s/it]
32%|███▏ | 507/1570 [36:03<52:17, 2.95s/it]
32%|███▏ | 508/1570 [36:05<52:12, 2.95s/it]
32%|███▏ | 509/1570 [36:08<52:01, 2.94s/it]
32%|███▏ | 510/1570 [36:11<52:00, 2.94s/it]
33%|███▎ | 511/1570 [36:14<51:57, 2.94s/it]
33%|███▎ | 512/1570 [36:17<51:49, 2.94s/it]
33%|███▎ | 513/1570 [36:20<51:49, 2.94s/it]
33%|███▎ | 514/1570 [36:23<51:47, 2.94s/it]
33%|███▎ | 515/1570 [36:26<51:32, 2.93s/it]
33%|███▎ | 516/1570 [36:29<51:27, 2.93s/it]
33%|███▎ | 517/1570 [36:32<50:58, 2.90s/it]
33%|███▎ | 518/1570 [36:35<50:56, 2.91s/it]
33%|███▎ | 519/1570 [36:38<51:12, 2.92s/it]
33%|███▎ | 520/1570 [36:41<51:12, 2.93s/it]
{'loss': '0.005865', 'grad_norm': '0.09949', 'learning_rate': '7.54e-05', 'epoch': '1.657'}
+
33%|███▎ | 520/1570 [36:41<51:12, 2.93s/it]
33%|███▎ | 521/1570 [36:44<51:25, 2.94s/it]
33%|███▎ | 522/1570 [36:47<51:34, 2.95s/it]
33%|███▎ | 523/1570 [36:49<51:22, 2.94s/it]
33%|███▎ | 524/1570 [36:52<51:25, 2.95s/it]
33%|███▎ | 525/1570 [36:55<51:16, 2.94s/it]
34%|███▎ | 526/1570 [36:58<51:12, 2.94s/it]
34%|███▎ | 527/1570 [37:01<51:04, 2.94s/it]
34%|███▎ | 528/1570 [37:04<50:48, 2.93s/it]
34%|███▎ | 529/1570 [37:07<50:41, 2.92s/it]
34%|███▍ | 530/1570 [37:10<50:27, 2.91s/it]
34%|███▍ | 531/1570 [37:13<50:32, 2.92s/it]
34%|███▍ | 532/1570 [37:16<50:31, 2.92s/it]
34%|███▍ | 533/1570 [37:19<50:32, 2.92s/it]
34%|███▍ | 534/1570 [37:22<50:46, 2.94s/it]
34%|███▍ | 535/1570 [37:25<50:35, 2.93s/it]
34%|███▍ | 536/1570 [37:28<50:31, 2.93s/it]
34%|███▍ | 537/1570 [37:30<50:16, 2.92s/it]
34%|███▍ | 538/1570 [37:33<50:23, 2.93s/it]
34%|███▍ | 539/1570 [37:36<50:10, 2.92s/it]
34%|███▍ | 540/1570 [37:39<50:07, 2.92s/it]
{'loss': '0.007865', 'grad_norm': '0.3055', 'learning_rate': '7.366e-05', 'epoch': '1.721'}
+
34%|███▍ | 540/1570 [37:39<50:07, 2.92s/it]
34%|███▍ | 541/1570 [37:42<50:14, 2.93s/it]
35%|███▍ | 542/1570 [37:45<50:08, 2.93s/it]
35%|███▍ | 543/1570 [37:48<50:11, 2.93s/it]
35%|███▍ | 544/1570 [37:51<50:03, 2.93s/it]
35%|███▍ | 545/1570 [37:54<49:56, 2.92s/it]
35%|███▍ | 546/1570 [37:57<49:51, 2.92s/it]
35%|███▍ | 547/1570 [38:00<49:54, 2.93s/it]
35%|███▍ | 548/1570 [38:03<49:42, 2.92s/it]
35%|███▍ | 549/1570 [38:05<49:19, 2.90s/it]
35%|███▌ | 550/1570 [38:08<49:27, 2.91s/it]
35%|███▌ | 551/1570 [38:11<49:45, 2.93s/it]
35%|███▌ | 552/1570 [38:14<49:28, 2.92s/it]
35%|███▌ | 553/1570 [38:17<49:15, 2.91s/it]
35%|███▌ | 554/1570 [38:20<49:14, 2.91s/it]
35%|███▌ | 555/1570 [38:23<49:14, 2.91s/it]
35%|███▌ | 556/1570 [38:26<49:14, 2.91s/it]
35%|███▌ | 557/1570 [38:29<49:09, 2.91s/it]
36%|███▌ | 558/1570 [38:32<49:08, 2.91s/it]
36%|███▌ | 559/1570 [38:35<49:06, 2.91s/it]
36%|███▌ | 560/1570 [38:38<49:06, 2.92s/it]
{'loss': '0.00261', 'grad_norm': '0.334', 'learning_rate': '7.188e-05', 'epoch': '1.785'}
+
36%|███▌ | 560/1570 [38:38<49:06, 2.92s/it]
36%|███▌ | 561/1570 [38:40<49:04, 2.92s/it]
36%|███▌ | 562/1570 [38:43<48:53, 2.91s/it]
36%|███▌ | 563/1570 [38:46<48:40, 2.90s/it]
36%|███▌ | 564/1570 [38:49<48:40, 2.90s/it]
36%|███▌ | 565/1570 [38:52<49:03, 2.93s/it]
36%|███▌ | 566/1570 [38:55<48:49, 2.92s/it]
36%|███▌ | 567/1570 [38:58<48:45, 2.92s/it]
36%|███▌ | 568/1570 [39:01<48:41, 2.92s/it]
36%|███▌ | 569/1570 [39:04<48:45, 2.92s/it]
36%|███▋ | 570/1570 [39:07<48:35, 2.92s/it]
36%|███▋ | 571/1570 [39:10<48:26, 2.91s/it]
36%|███▋ | 572/1570 [39:12<48:18, 2.90s/it]
36%|███▋ | 573/1570 [39:15<48:30, 2.92s/it]
37%|███▋ | 574/1570 [39:18<48:15, 2.91s/it]
37%|███▋ | 575/1570 [39:21<48:07, 2.90s/it]
37%|███▋ | 576/1570 [39:24<47:48, 2.89s/it]
37%|███▋ | 577/1570 [39:27<48:04, 2.91s/it]
37%|███▋ | 578/1570 [39:30<47:56, 2.90s/it]
37%|███▋ | 579/1570 [39:33<47:51, 2.90s/it]
37%|███▋ | 580/1570 [39:36<48:07, 2.92s/it]
{'loss': '0.005318', 'grad_norm': '0.3238', 'learning_rate': '7.006e-05', 'epoch': '1.849'}
+
37%|███▋ | 580/1570 [39:36<48:07, 2.92s/it]
37%|███▋ | 581/1570 [39:39<47:53, 2.91s/it]
37%|███▋ | 582/1570 [39:42<47:52, 2.91s/it]
37%|███▋ | 583/1570 [39:44<47:48, 2.91s/it]
37%|███▋ | 584/1570 [39:47<47:54, 2.91s/it]
37%|███▋ | 585/1570 [39:50<47:58, 2.92s/it]
37%|███▋ | 586/1570 [39:53<47:49, 2.92s/it]
37%|███▋ | 587/1570 [39:56<47:33, 2.90s/it]
37%|███▋ | 588/1570 [39:59<47:11, 2.88s/it]
38%|███▊ | 589/1570 [40:02<47:28, 2.90s/it]
38%|███▊ | 590/1570 [40:05<47:26, 2.90s/it]
38%|███▊ | 591/1570 [40:08<47:27, 2.91s/it]
38%|███▊ | 592/1570 [40:11<47:32, 2.92s/it]
38%|███▊ | 593/1570 [40:14<47:15, 2.90s/it]
38%|███▊ | 594/1570 [40:16<47:00, 2.89s/it]
38%|███▊ | 595/1570 [40:19<46:54, 2.89s/it]
38%|███▊ | 596/1570 [40:22<47:04, 2.90s/it]
38%|███▊ | 597/1570 [40:25<47:10, 2.91s/it]
38%|███▊ | 598/1570 [40:28<47:05, 2.91s/it]
38%|███▊ | 599/1570 [40:31<47:07, 2.91s/it]
38%|███▊ | 600/1570 [40:34<46:50, 2.90s/it]
{'loss': '0.0031', 'grad_norm': '0.06956', 'learning_rate': '6.821e-05', 'epoch': '1.913'}
+
38%|███▊ | 600/1570 [40:34<46:50, 2.90s/it]
38%|███▊ | 601/1570 [40:37<46:55, 2.91s/it]
38%|███▊ | 602/1570 [40:40<46:57, 2.91s/it]
38%|███▊ | 603/1570 [40:43<47:01, 2.92s/it]
38%|███▊ | 604/1570 [40:46<47:09, 2.93s/it]
39%|███▊ | 605/1570 [40:48<46:59, 2.92s/it]
39%|███▊ | 606/1570 [40:51<47:06, 2.93s/it]
39%|███▊ | 607/1570 [40:54<46:54, 2.92s/it]
39%|███▊ | 608/1570 [40:57<46:46, 2.92s/it]
39%|███▉ | 609/1570 [41:00<46:31, 2.90s/it]
39%|███▉ | 610/1570 [41:03<46:28, 2.90s/it]
39%|███▉ | 611/1570 [41:06<46:21, 2.90s/it]
39%|███▉ | 612/1570 [41:09<46:18, 2.90s/it]
39%|███▉ | 613/1570 [41:12<46:09, 2.89s/it]
39%|███▉ | 614/1570 [41:15<46:06, 2.89s/it]
39%|███▉ | 615/1570 [41:17<46:07, 2.90s/it]
39%|███▉ | 616/1570 [41:20<46:18, 2.91s/it]
39%|███▉ | 617/1570 [41:23<46:18, 2.92s/it]
39%|███▉ | 618/1570 [41:26<46:08, 2.91s/it]
39%|███▉ | 619/1570 [41:29<46:16, 2.92s/it]
39%|███▉ | 620/1570 [41:32<46:20, 2.93s/it]
{'loss': '0.003379', 'grad_norm': '0.0299', 'learning_rate': '6.634e-05', 'epoch': '1.976'}
+
39%|███▉ | 620/1570 [41:32<46:20, 2.93s/it]
40%|███▉ | 621/1570 [41:35<46:22, 2.93s/it]
40%|███▉ | 622/1570 [41:38<45:56, 2.91s/it]
40%|███▉ | 623/1570 [41:41<45:57, 2.91s/it]
40%|███▉ | 624/1570 [41:44<46:08, 2.93s/it]
40%|███▉ | 625/1570 [41:47<46:13, 2.93s/it]
40%|███▉ | 626/1570 [41:50<45:57, 2.92s/it]
40%|███▉ | 627/1570 [41:53<46:01, 2.93s/it]
40%|████ | 628/1570 [41:54<37:40, 2.40s/it][INFO|trainer.py:2668] 2026-08-22 17:58:13,121 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 17:58:13,121 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 17:58:13,121 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/srkl/checkpoint-628
+[INFO|configuration_utils.py:780] 2026-08-22 18:08:24,515 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 18:08:24,517 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 18:08:24,552 >> chat template saved in results/qwen3/srkl/checkpoint-628/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 18:08:24,552 >> tokenizer config file saved in results/qwen3/srkl/checkpoint-628/tokenizer_config.json
+
40%|████ | 629/1570 [52:09<48:41:27, 186.28s/it]
40%|████ | 630/1570 [52:12<34:16:35, 131.27s/it]
40%|████ | 631/1570 [52:15<24:11:41, 92.76s/it]
40%|████ | 632/1570 [52:18<17:08:37, 65.80s/it]
40%|████ | 633/1570 [52:21<12:12:55, 46.93s/it]
40%|████ | 634/1570 [52:24<8:46:13, 33.73s/it]
40%|████ | 635/1570 [52:27<6:21:56, 24.51s/it]
41%|████ | 636/1570 [52:30<4:40:38, 18.03s/it]
41%|████ | 637/1570 [52:32<3:29:44, 13.49s/it]
41%|████ | 638/1570 [52:36<2:45:15, 10.64s/it]
41%|████ | 639/1570 [52:40<2:14:09, 8.65s/it]
41%|████ | 640/1570 [52:45<1:52:57, 7.29s/it]
{'loss': '0.002889', 'grad_norm': '0.08394', 'learning_rate': '6.444e-05', 'epoch': '2.038'}
+
41%|████ | 640/1570 [52:45<1:52:57, 7.29s/it]
41%|████ | 641/1570 [52:49<1:38:11, 6.34s/it]
41%|████ | 642/1570 [52:53<1:27:36, 5.66s/it]
41%|████ | 643/1570 [52:57<1:20:06, 5.19s/it]
41%|████ | 644/1570 [53:01<1:14:51, 4.85s/it]
41%|████ | 645/1570 [53:05<1:11:02, 4.61s/it]
41%|████ | 646/1570 [53:09<1:08:10, 4.43s/it]
41%|████ | 647/1570 [53:13<1:06:16, 4.31s/it]
41%|████▏ | 648/1570 [53:17<1:05:06, 4.24s/it]
41%|████▏ | 649/1570 [53:21<1:03:49, 4.16s/it]
41%|████▏ | 650/1570 [53:25<1:03:37, 4.15s/it]
41%|████▏ | 651/1570 [53:29<1:03:10, 4.12s/it]
42%|████▏ | 652/1570 [53:33<1:01:49, 4.04s/it]
42%|████▏ | 653/1570 [53:37<1:01:58, 4.05s/it]
42%|████▏ | 654/1570 [53:41<1:01:49, 4.05s/it]
42%|████▏ | 655/1570 [53:45<1:01:33, 4.04s/it]
42%|████▏ | 656/1570 [53:49<1:01:32, 4.04s/it]
42%|████▏ | 657/1570 [53:53<1:00:45, 3.99s/it]
42%|████▏ | 658/1570 [53:57<1:00:54, 4.01s/it]
42%|████▏ | 659/1570 [54:01<1:00:38, 3.99s/it]
42%|████▏ | 660/1570 [54:05<1:00:57, 4.02s/it]
{'loss': '0.002997', 'grad_norm': '0.3179', 'learning_rate': '6.251e-05', 'epoch': '2.102'}
+
42%|████▏ | 660/1570 [54:05<1:00:57, 4.02s/it]
42%|████▏ | 661/1570 [54:09<1:01:11, 4.04s/it]
42%|████▏ | 662/1570 [54:13<1:00:40, 4.01s/it]
42%|████▏ | 663/1570 [54:17<1:00:24, 4.00s/it]
42%|████▏ | 664/1570 [54:21<58:18, 3.86s/it]
42%|████▏ | 665/1570 [54:24<56:59, 3.78s/it]
42%|████▏ | 666/1570 [54:28<55:57, 3.71s/it]
42%|████▏ | 667/1570 [54:31<55:07, 3.66s/it]
43%|████▎ | 668/1570 [54:35<54:35, 3.63s/it]
43%|████▎ | 669/1570 [54:39<54:10, 3.61s/it]
43%|████▎ | 670/1570 [54:42<54:06, 3.61s/it]
43%|████▎ | 671/1570 [54:45<52:42, 3.52s/it]
43%|████▎ | 672/1570 [54:48<50:22, 3.37s/it]
43%|████▎ | 673/1570 [54:51<48:53, 3.27s/it]
43%|████▎ | 674/1570 [54:55<50:08, 3.36s/it]
43%|████▎ | 675/1570 [54:59<51:45, 3.47s/it]
43%|████▎ | 676/1570 [55:02<52:18, 3.51s/it]
43%|████▎ | 677/1570 [55:06<52:10, 3.51s/it]
43%|████▎ | 678/1570 [55:09<52:05, 3.50s/it]
43%|████▎ | 679/1570 [55:13<51:36, 3.48s/it]
43%|████▎ | 680/1570 [55:16<49:18, 3.32s/it]
{'loss': '0.001807', 'grad_norm': '0.03328', 'learning_rate': '6.057e-05', 'epoch': '2.166'}
+
43%|████▎ | 680/1570 [55:16<49:18, 3.32s/it]
43%|████▎ | 681/1570 [55:19<47:27, 3.20s/it]
43%|████▎ | 682/1570 [55:22<46:09, 3.12s/it]
44%|████▎ | 683/1570 [55:24<44:50, 3.03s/it]
44%|████▎ | 684/1570 [55:27<44:17, 3.00s/it]
44%|████▎ | 685/1570 [55:30<44:03, 2.99s/it]
44%|████▎ | 686/1570 [55:33<43:36, 2.96s/it]
44%|████▍ | 687/1570 [55:36<43:16, 2.94s/it]
44%|████▍ | 688/1570 [55:39<43:06, 2.93s/it]
44%|████▍ | 689/1570 [55:42<42:57, 2.93s/it]
44%|████▍ | 690/1570 [55:45<42:42, 2.91s/it]
44%|████▍ | 691/1570 [55:48<42:53, 2.93s/it]
44%|████▍ | 692/1570 [55:51<42:47, 2.92s/it]
44%|████▍ | 693/1570 [55:54<42:39, 2.92s/it]
44%|████▍ | 694/1570 [55:57<42:44, 2.93s/it]
44%|████▍ | 695/1570 [56:00<42:49, 2.94s/it]
44%|████▍ | 696/1570 [56:02<42:33, 2.92s/it]
44%|████▍ | 697/1570 [56:05<42:19, 2.91s/it]
44%|████▍ | 698/1570 [56:08<42:15, 2.91s/it]
45%|████▍ | 699/1570 [56:11<42:15, 2.91s/it]
45%|████▍ | 700/1570 [56:14<42:08, 2.91s/it]
{'loss': '0.001691', 'grad_norm': '0.06479', 'learning_rate': '5.86e-05', 'epoch': '2.23'}
+
45%|████▍ | 700/1570 [56:14<42:08, 2.91s/it]
45%|████▍ | 701/1570 [56:17<42:06, 2.91s/it]
45%|████▍ | 702/1570 [56:20<42:23, 2.93s/it]
45%|████▍ | 703/1570 [56:23<42:13, 2.92s/it]
45%|████▍ | 704/1570 [56:26<42:06, 2.92s/it]
45%|████▍ | 705/1570 [56:29<41:57, 2.91s/it]
45%|████▍ | 706/1570 [56:32<42:12, 2.93s/it]
45%|████▌ | 707/1570 [56:34<41:59, 2.92s/it]
45%|████▌ | 708/1570 [56:37<42:00, 2.92s/it]
45%|████▌ | 709/1570 [56:40<41:54, 2.92s/it]
45%|████▌ | 710/1570 [56:43<41:34, 2.90s/it]
45%|████▌ | 711/1570 [56:46<41:35, 2.90s/it]
45%|████▌ | 712/1570 [56:49<41:46, 2.92s/it]
45%|████▌ | 713/1570 [56:52<41:49, 2.93s/it]
45%|████▌ | 714/1570 [56:55<41:42, 2.92s/it]
46%|████▌ | 715/1570 [56:58<41:36, 2.92s/it]
46%|████▌ | 716/1570 [57:01<41:31, 2.92s/it]
46%|████▌ | 717/1570 [57:04<41:24, 2.91s/it]
46%|████▌ | 718/1570 [57:07<41:32, 2.93s/it]
46%|████▌ | 719/1570 [57:10<41:37, 2.94s/it]
46%|████▌ | 720/1570 [57:13<41:46, 2.95s/it]
{'loss': '0.001684', 'grad_norm': '0.1992', 'learning_rate': '5.663e-05', 'epoch': '2.294'}
+
46%|████▌ | 720/1570 [57:13<41:46, 2.95s/it]
46%|████▌ | 721/1570 [57:15<41:28, 2.93s/it]
46%|████▌ | 722/1570 [57:18<41:16, 2.92s/it]
46%|████▌ | 723/1570 [57:21<41:11, 2.92s/it]
46%|████▌ | 724/1570 [57:24<41:13, 2.92s/it]
46%|████▌ | 725/1570 [57:27<41:15, 2.93s/it]
46%|████▌ | 726/1570 [57:30<41:20, 2.94s/it]
46%|████▋ | 727/1570 [57:33<41:15, 2.94s/it]
46%|████▋ | 728/1570 [57:36<41:19, 2.94s/it]
46%|████▋ | 729/1570 [57:39<41:35, 2.97s/it]
46%|████▋ | 730/1570 [57:42<41:43, 2.98s/it]
47%|████▋ | 731/1570 [57:45<41:47, 2.99s/it]
47%|████▋ | 732/1570 [57:48<41:42, 2.99s/it]
47%|████▋ | 733/1570 [57:52<44:39, 3.20s/it]
47%|████▋ | 734/1570 [57:55<43:31, 3.12s/it]
47%|████▋ | 735/1570 [57:58<43:19, 3.11s/it]
47%|████▋ | 736/1570 [58:01<43:14, 3.11s/it]
47%|████▋ | 737/1570 [58:04<43:05, 3.10s/it]
47%|████▋ | 738/1570 [58:07<42:47, 3.09s/it]
47%|████▋ | 739/1570 [58:10<42:59, 3.10s/it]
47%|████▋ | 740/1570 [58:13<42:43, 3.09s/it]
{'loss': '0.001978', 'grad_norm': '0.04859', 'learning_rate': '5.464e-05', 'epoch': '2.357'}
+
47%|████▋ | 740/1570 [58:13<42:43, 3.09s/it]
47%|████▋ | 741/1570 [58:16<41:58, 3.04s/it]
47%|████▋ | 742/1570 [58:19<41:33, 3.01s/it]
47%|████▋ | 743/1570 [58:22<41:13, 2.99s/it]
47%|████▋ | 744/1570 [58:25<41:03, 2.98s/it]
47%|████▋ | 745/1570 [58:28<40:44, 2.96s/it]
48%|████▊ | 746/1570 [58:31<40:23, 2.94s/it]
48%|████▊ | 747/1570 [58:34<40:18, 2.94s/it]
48%|████▊ | 748/1570 [58:37<40:11, 2.93s/it]
48%|████▊ | 749/1570 [58:40<40:17, 2.94s/it]
48%|████▊ | 750/1570 [58:42<39:57, 2.92s/it]
48%|████▊ | 751/1570 [58:45<40:00, 2.93s/it]
48%|████▊ | 752/1570 [58:48<39:52, 2.92s/it]
48%|████▊ | 753/1570 [58:51<39:47, 2.92s/it]
48%|████▊ | 754/1570 [58:54<39:46, 2.92s/it]
48%|████▊ | 755/1570 [58:57<39:40, 2.92s/it]
48%|████▊ | 756/1570 [59:00<39:33, 2.92s/it]
48%|████▊ | 757/1570 [59:03<39:19, 2.90s/it]
48%|████▊ | 758/1570 [59:06<39:23, 2.91s/it]
48%|████▊ | 759/1570 [59:09<39:25, 2.92s/it]
48%|████▊ | 760/1570 [59:12<39:24, 2.92s/it]
{'loss': '0.001009', 'grad_norm': '0.01044', 'learning_rate': '5.265e-05', 'epoch': '2.421'}
+
48%|████▊ | 760/1570 [59:12<39:24, 2.92s/it]
48%|████▊ | 761/1570 [59:15<39:27, 2.93s/it]
49%|████▊ | 762/1570 [59:17<39:30, 2.93s/it]
49%|████▊ | 763/1570 [59:20<39:19, 2.92s/it]
49%|████▊ | 764/1570 [59:23<39:16, 2.92s/it]
49%|████▊ | 765/1570 [59:26<39:03, 2.91s/it]
49%|████▉ | 766/1570 [59:29<39:04, 2.92s/it]
49%|████▉ | 767/1570 [59:32<38:57, 2.91s/it]
49%|████▉ | 768/1570 [59:35<38:56, 2.91s/it]
49%|████▉ | 769/1570 [59:38<38:40, 2.90s/it]
49%|████▉ | 770/1570 [59:41<38:46, 2.91s/it]
49%|████▉ | 771/1570 [59:44<38:42, 2.91s/it]
49%|████▉ | 772/1570 [59:47<38:43, 2.91s/it]
49%|████▉ | 773/1570 [59:49<38:35, 2.91s/it]
49%|████▉ | 774/1570 [59:52<38:29, 2.90s/it]
49%|████▉ | 775/1570 [59:55<38:37, 2.91s/it]
49%|████▉ | 776/1570 [59:58<38:41, 2.92s/it]
49%|████▉ | 777/1570 [1:00:01<38:35, 2.92s/it]
50%|████▉ | 778/1570 [1:00:04<38:27, 2.91s/it]
50%|████▉ | 779/1570 [1:00:07<38:17, 2.90s/it]
50%|████▉ | 780/1570 [1:00:10<38:12, 2.90s/it]
{'loss': '0.0007452', 'grad_norm': '0.04027', 'learning_rate': '5.065e-05', 'epoch': '2.485'}
+
50%|████▉ | 780/1570 [1:00:10<38:12, 2.90s/it]
50%|████▉ | 781/1570 [1:00:13<38:08, 2.90s/it]
50%|████▉ | 782/1570 [1:00:16<38:11, 2.91s/it]
50%|████▉ | 783/1570 [1:00:19<38:08, 2.91s/it]
50%|████▉ | 784/1570 [1:00:22<41:55, 3.20s/it]
50%|█████ | 785/1570 [1:00:27<45:24, 3.47s/it]
50%|█████ | 786/1570 [1:00:30<46:41, 3.57s/it]
50%|█████ | 787/1570 [1:00:34<46:32, 3.57s/it]
50%|█████ | 788/1570 [1:00:37<46:31, 3.57s/it]
50%|█████ | 789/1570 [1:00:41<46:34, 3.58s/it]
50%|█████ | 790/1570 [1:00:45<46:39, 3.59s/it]
50%|█████ | 791/1570 [1:00:48<46:51, 3.61s/it]
50%|█████ | 792/1570 [1:00:52<46:42, 3.60s/it]
51%|█████ | 793/1570 [1:00:55<46:09, 3.56s/it]
51%|█████ | 794/1570 [1:00:58<44:03, 3.41s/it]
51%|█████ | 795/1570 [1:01:01<42:27, 3.29s/it]
51%|█████ | 796/1570 [1:01:05<43:10, 3.35s/it]
51%|█████ | 797/1570 [1:01:09<44:29, 3.45s/it]
51%|█████ | 798/1570 [1:01:12<45:14, 3.52s/it]
51%|█████ | 799/1570 [1:01:16<45:28, 3.54s/it]
51%|█████ | 800/1570 [1:01:20<45:45, 3.57s/it]
{'loss': '0.003089', 'grad_norm': '0.07828', 'learning_rate': '4.865e-05', 'epoch': '2.549'}
+
51%|█████ | 800/1570 [1:01:20<45:45, 3.57s/it]
51%|█████ | 801/1570 [1:01:23<44:21, 3.46s/it]
51%|█████ | 802/1570 [1:01:26<42:12, 3.30s/it]
51%|█████ | 803/1570 [1:01:29<40:42, 3.18s/it]
51%|█████ | 804/1570 [1:01:31<39:34, 3.10s/it]
51%|█████▏ | 805/1570 [1:01:34<38:43, 3.04s/it]
51%|█████▏ | 806/1570 [1:01:37<38:12, 3.00s/it]
51%|█████▏ | 807/1570 [1:01:40<37:47, 2.97s/it]
51%|█████▏ | 808/1570 [1:01:43<37:22, 2.94s/it]
52%|█████▏ | 809/1570 [1:01:46<37:10, 2.93s/it]
52%|█████▏ | 810/1570 [1:01:49<37:01, 2.92s/it]
52%|█████▏ | 811/1570 [1:01:52<36:56, 2.92s/it]
52%|█████▏ | 812/1570 [1:01:55<36:51, 2.92s/it]
52%|█████▏ | 813/1570 [1:01:58<37:01, 2.93s/it]
52%|█████▏ | 814/1570 [1:02:01<36:49, 2.92s/it]
52%|█████▏ | 815/1570 [1:02:03<36:38, 2.91s/it]
52%|█████▏ | 816/1570 [1:02:06<36:46, 2.93s/it]
52%|█████▏ | 817/1570 [1:02:09<36:45, 2.93s/it]
52%|█████▏ | 818/1570 [1:02:12<36:45, 2.93s/it]
52%|█████▏ | 819/1570 [1:02:15<36:44, 2.93s/it]
52%|█████▏ | 820/1570 [1:02:18<36:35, 2.93s/it]
{'loss': '0.001661', 'grad_norm': '0.2067', 'learning_rate': '4.665e-05', 'epoch': '2.613'}
+
52%|█████▏ | 820/1570 [1:02:18<36:35, 2.93s/it]
52%|█████▏ | 821/1570 [1:02:21<36:42, 2.94s/it]
52%|█████▏ | 822/1570 [1:02:24<36:34, 2.93s/it]
52%|█████▏ | 823/1570 [1:02:27<36:30, 2.93s/it]
52%|█████▏ | 824/1570 [1:02:30<36:44, 2.95s/it]
53%|█████▎ | 825/1570 [1:02:33<36:29, 2.94s/it]
53%|█████▎ | 826/1570 [1:02:36<36:34, 2.95s/it]
53%|█████▎ | 827/1570 [1:02:39<36:19, 2.93s/it]
53%|█████▎ | 828/1570 [1:02:42<36:18, 2.94s/it]
53%|█████▎ | 829/1570 [1:02:45<36:08, 2.93s/it]
53%|█████▎ | 830/1570 [1:02:48<36:12, 2.94s/it]
53%|█████▎ | 831/1570 [1:02:50<36:05, 2.93s/it]
53%|█████▎ | 832/1570 [1:02:53<35:53, 2.92s/it]
53%|█████▎ | 833/1570 [1:02:56<36:26, 2.97s/it]
53%|█████▎ | 834/1570 [1:02:59<36:17, 2.96s/it]
53%|█████▎ | 835/1570 [1:03:02<36:09, 2.95s/it]
53%|█████▎ | 836/1570 [1:03:05<35:54, 2.94s/it]
53%|█████▎ | 837/1570 [1:03:08<35:46, 2.93s/it]
53%|█████▎ | 838/1570 [1:03:11<35:42, 2.93s/it]
53%|█████▎ | 839/1570 [1:03:14<35:35, 2.92s/it]
54%|█████▎ | 840/1570 [1:03:17<35:30, 2.92s/it]
{'loss': '0.001747', 'grad_norm': '0.06613', 'learning_rate': '4.466e-05', 'epoch': '2.677'}
+
54%|█████▎ | 840/1570 [1:03:17<35:30, 2.92s/it]
54%|█████▎ | 841/1570 [1:03:20<35:54, 2.96s/it]
54%|█████▎ | 842/1570 [1:03:23<35:46, 2.95s/it]
54%|█████▎ | 843/1570 [1:03:26<35:34, 2.94s/it]
54%|█████▍ | 844/1570 [1:03:29<35:32, 2.94s/it]
54%|█████▍ | 845/1570 [1:03:32<35:20, 2.92s/it]
54%|█████▍ | 846/1570 [1:03:35<35:24, 2.93s/it]
54%|█████▍ | 847/1570 [1:03:37<35:28, 2.94s/it]
54%|█████▍ | 848/1570 [1:03:40<35:38, 2.96s/it]
54%|█████▍ | 849/1570 [1:03:43<35:14, 2.93s/it]
54%|█████▍ | 850/1570 [1:03:47<36:07, 3.01s/it]
54%|█████▍ | 851/1570 [1:03:50<35:50, 2.99s/it]
54%|█████▍ | 852/1570 [1:03:53<36:14, 3.03s/it]
54%|█████▍ | 853/1570 [1:03:56<36:22, 3.04s/it]
54%|█████▍ | 854/1570 [1:03:59<37:02, 3.10s/it]
54%|█████▍ | 855/1570 [1:04:02<36:37, 3.07s/it]
55%|█████▍ | 856/1570 [1:04:05<36:15, 3.05s/it]
55%|█████▍ | 857/1570 [1:04:08<36:51, 3.10s/it]
55%|█████▍ | 858/1570 [1:04:11<37:18, 3.14s/it]
55%|█████▍ | 859/1570 [1:04:15<37:18, 3.15s/it]
55%|█████▍ | 860/1570 [1:04:18<38:10, 3.23s/it]
{'loss': '0.003954', 'grad_norm': '0.1408', 'learning_rate': '4.268e-05', 'epoch': '2.74'}
+
55%|█████▍ | 860/1570 [1:04:18<38:10, 3.23s/it]
55%|█████▍ | 861/1570 [1:04:21<38:40, 3.27s/it]
55%|█████▍ | 862/1570 [1:04:24<37:32, 3.18s/it]
55%|█████▍ | 863/1570 [1:04:27<36:26, 3.09s/it]
55%|█████▌ | 864/1570 [1:04:30<35:54, 3.05s/it]
55%|█████▌ | 865/1570 [1:04:33<35:21, 3.01s/it]
55%|█████▌ | 866/1570 [1:04:36<34:50, 2.97s/it]
55%|█████▌ | 867/1570 [1:04:39<34:28, 2.94s/it]
55%|█████▌ | 868/1570 [1:04:42<34:09, 2.92s/it]
55%|█████▌ | 869/1570 [1:04:45<34:11, 2.93s/it]
55%|█████▌ | 870/1570 [1:04:48<34:09, 2.93s/it]
55%|█████▌ | 871/1570 [1:04:50<34:01, 2.92s/it]
56%|█████▌ | 872/1570 [1:04:53<33:51, 2.91s/it]
56%|█████▌ | 873/1570 [1:04:56<33:42, 2.90s/it]
56%|█████▌ | 874/1570 [1:04:59<33:38, 2.90s/it]
56%|█████▌ | 875/1570 [1:05:02<33:39, 2.91s/it]
56%|█████▌ | 876/1570 [1:05:05<33:32, 2.90s/it]
56%|█████▌ | 877/1570 [1:05:08<33:39, 2.91s/it]
56%|█████▌ | 878/1570 [1:05:11<33:40, 2.92s/it]
56%|█████▌ | 879/1570 [1:05:14<33:41, 2.92s/it]
56%|█████▌ | 880/1570 [1:05:17<33:40, 2.93s/it]
{'loss': '0.001689', 'grad_norm': '0.05687', 'learning_rate': '4.071e-05', 'epoch': '2.804'}
+
56%|█████▌ | 880/1570 [1:05:17<33:40, 2.93s/it]
56%|█████▌ | 881/1570 [1:05:20<33:34, 2.92s/it]
56%|█████▌ | 882/1570 [1:05:23<33:32, 2.93s/it]
56%|█████▌ | 883/1570 [1:05:25<33:21, 2.91s/it]
56%|█████▋ | 884/1570 [1:05:28<33:19, 2.91s/it]
56%|█████▋ | 885/1570 [1:05:31<33:06, 2.90s/it]
56%|█████▋ | 886/1570 [1:05:34<33:16, 2.92s/it]
56%|█████▋ | 887/1570 [1:05:37<33:25, 2.94s/it]
57%|█████▋ | 888/1570 [1:05:40<33:19, 2.93s/it]
57%|█████▋ | 889/1570 [1:05:43<33:20, 2.94s/it]
57%|█████▋ | 890/1570 [1:05:46<33:03, 2.92s/it]
57%|█████▋ | 891/1570 [1:05:49<32:55, 2.91s/it]
57%|█████▋ | 892/1570 [1:05:52<32:51, 2.91s/it]
57%|█████▋ | 893/1570 [1:05:55<32:49, 2.91s/it]
57%|█████▋ | 894/1570 [1:05:58<32:57, 2.92s/it]
57%|█████▋ | 895/1570 [1:06:00<32:45, 2.91s/it]
57%|█████▋ | 896/1570 [1:06:03<32:41, 2.91s/it]
57%|█████▋ | 897/1570 [1:06:06<32:43, 2.92s/it]
57%|█████▋ | 898/1570 [1:06:09<32:50, 2.93s/it]
57%|█████▋ | 899/1570 [1:06:12<32:41, 2.92s/it]
57%|█████▋ | 900/1570 [1:06:15<32:33, 2.92s/it]
{'loss': '0.001372', 'grad_norm': '0.1041', 'learning_rate': '3.875e-05', 'epoch': '2.868'}
+
57%|█████▋ | 900/1570 [1:06:15<32:33, 2.92s/it]
57%|█████▋ | 901/1570 [1:06:18<32:33, 2.92s/it]
57%|█████▋ | 902/1570 [1:06:21<32:27, 2.92s/it]
58%|█████▊ | 903/1570 [1:06:24<32:24, 2.92s/it]
58%|█████▊ | 904/1570 [1:06:27<32:12, 2.90s/it]
58%|█████▊ | 905/1570 [1:06:30<32:11, 2.90s/it]
58%|█████▊ | 906/1570 [1:06:32<31:57, 2.89s/it]
58%|█████▊ | 907/1570 [1:06:35<31:56, 2.89s/it]
58%|█████▊ | 908/1570 [1:06:38<31:56, 2.89s/it]
58%|█████▊ | 909/1570 [1:06:41<31:54, 2.90s/it]
58%|█████▊ | 910/1570 [1:06:44<31:49, 2.89s/it]
58%|█████▊ | 911/1570 [1:06:47<31:58, 2.91s/it]
58%|█████▊ | 912/1570 [1:06:50<31:54, 2.91s/it]
58%|█████▊ | 913/1570 [1:06:53<31:52, 2.91s/it]
58%|█████▊ | 914/1570 [1:06:56<31:57, 2.92s/it]
58%|█████▊ | 915/1570 [1:06:59<32:49, 3.01s/it]
58%|█████▊ | 916/1570 [1:07:02<33:35, 3.08s/it]
58%|█████▊ | 917/1570 [1:07:05<32:56, 3.03s/it]
58%|█████▊ | 918/1570 [1:07:08<32:24, 2.98s/it]
59%|█████▊ | 919/1570 [1:07:11<32:17, 2.98s/it]
59%|█████▊ | 920/1570 [1:07:14<32:06, 2.96s/it]
{'loss': '0.001446', 'grad_norm': '0.003308', 'learning_rate': '3.682e-05', 'epoch': '2.932'}
+
59%|█████▊ | 920/1570 [1:07:14<32:06, 2.96s/it]
59%|█████▊ | 921/1570 [1:07:17<32:03, 2.96s/it]
59%|█████▊ | 922/1570 [1:07:20<31:58, 2.96s/it]
59%|█████▉ | 923/1570 [1:07:23<31:57, 2.96s/it]
59%|█████▉ | 924/1570 [1:07:26<31:51, 2.96s/it]
59%|█████▉ | 925/1570 [1:07:29<31:45, 2.95s/it]
59%|█████▉ | 926/1570 [1:07:32<31:35, 2.94s/it]
59%|█████▉ | 927/1570 [1:07:34<31:16, 2.92s/it]
59%|█████▉ | 928/1570 [1:07:37<31:09, 2.91s/it]
59%|█████▉ | 929/1570 [1:07:40<31:15, 2.93s/it]
59%|█████▉ | 930/1570 [1:07:43<31:12, 2.93s/it]
59%|█████▉ | 931/1570 [1:07:46<31:06, 2.92s/it]
59%|█████▉ | 932/1570 [1:07:49<31:06, 2.93s/it]
59%|█████▉ | 933/1570 [1:07:52<31:11, 2.94s/it]
59%|█████▉ | 934/1570 [1:07:55<31:17, 2.95s/it]
60%|█████▉ | 935/1570 [1:07:58<31:11, 2.95s/it]
60%|█████▉ | 936/1570 [1:08:01<31:12, 2.95s/it]
60%|█████▉ | 937/1570 [1:08:04<30:59, 2.94s/it]
60%|█████▉ | 938/1570 [1:08:07<30:53, 2.93s/it]
60%|█████▉ | 939/1570 [1:08:10<30:42, 2.92s/it]
60%|█████▉ | 940/1570 [1:08:13<30:39, 2.92s/it]
{'loss': '0.001163', 'grad_norm': '0.07255', 'learning_rate': '3.49e-05', 'epoch': '2.996'}
+
60%|█████▉ | 940/1570 [1:08:13<30:39, 2.92s/it]
60%|█████▉ | 941/1570 [1:08:15<30:40, 2.93s/it]
60%|██████ | 942/1570 [1:08:17<25:06, 2.40s/it][INFO|trainer.py:2668] 2026-08-22 18:24:36,036 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 18:24:36,036 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 18:24:36,036 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/srkl/checkpoint-942
+[INFO|configuration_utils.py:780] 2026-08-22 18:34:37,960 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 18:34:37,963 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 18:34:37,997 >> chat template saved in results/qwen3/srkl/checkpoint-942/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 18:34:37,997 >> tokenizer config file saved in results/qwen3/srkl/checkpoint-942/tokenizer_config.json
+
60%|██████ | 943/1570 [1:18:22<31:56:46, 183.42s/it]
60%|██████ | 944/1570 [1:18:25<22:28:34, 129.26s/it]
60%|██████ | 945/1570 [1:18:28<15:51:41, 91.36s/it]
60%|██████ | 946/1570 [1:18:31<11:14:09, 64.82s/it]
60%|██████ | 947/1570 [1:18:34<8:00:00, 46.23s/it]
60%|██████ | 948/1570 [1:18:37<5:44:28, 33.23s/it]
60%|██████ | 949/1570 [1:18:40<4:09:41, 24.13s/it]
61%|██████ | 950/1570 [1:18:43<3:03:18, 17.74s/it]
61%|██████ | 951/1570 [1:18:45<2:16:50, 13.26s/it]
61%|██████ | 952/1570 [1:18:48<1:44:24, 10.14s/it]
61%|██████ | 953/1570 [1:18:51<1:21:43, 7.95s/it]
61%|██████ | 954/1570 [1:18:54<1:06:02, 6.43s/it]
61%|██████ | 955/1570 [1:18:57<55:03, 5.37s/it]
61%|██████ | 956/1570 [1:19:00<47:19, 4.62s/it]
61%|██████ | 957/1570 [1:19:03<41:57, 4.11s/it]
61%|██████ | 958/1570 [1:19:06<38:00, 3.73s/it]
61%|██████ | 959/1570 [1:19:08<35:18, 3.47s/it]
61%|██████ | 960/1570 [1:19:11<33:23, 3.28s/it]
{'loss': '0.002454', 'grad_norm': '0.07352', 'learning_rate': '3.301e-05', 'epoch': '3.057'}
+
61%|██████ | 960/1570 [1:19:11<33:23, 3.28s/it]
61%|██████ | 961/1570 [1:19:14<31:59, 3.15s/it]
61%|██████▏ | 962/1570 [1:19:17<31:05, 3.07s/it]
61%|██████▏ | 963/1570 [1:19:20<30:28, 3.01s/it]
61%|██████▏ | 964/1570 [1:19:23<30:00, 2.97s/it]
61%|██████▏ | 965/1570 [1:19:26<29:46, 2.95s/it]
62%|██████▏ | 966/1570 [1:19:29<29:31, 2.93s/it]
62%|██████▏ | 967/1570 [1:19:31<29:06, 2.90s/it]
62%|██████▏ | 968/1570 [1:19:34<29:10, 2.91s/it]
62%|██████▏ | 969/1570 [1:19:37<29:07, 2.91s/it]
62%|██████▏ | 970/1570 [1:19:40<29:04, 2.91s/it]
62%|██████▏ | 971/1570 [1:19:43<28:50, 2.89s/it]
62%|██████▏ | 972/1570 [1:19:46<28:31, 2.86s/it]
62%|██████▏ | 973/1570 [1:19:49<28:30, 2.87s/it]
62%|██████▏ | 974/1570 [1:19:51<28:30, 2.87s/it]
62%|██████▏ | 975/1570 [1:19:54<28:19, 2.86s/it]
62%|██████▏ | 976/1570 [1:19:57<28:17, 2.86s/it]
62%|██████▏ | 977/1570 [1:20:00<28:21, 2.87s/it]
62%|██████▏ | 978/1570 [1:20:03<28:17, 2.87s/it]
62%|██████▏ | 979/1570 [1:20:06<28:14, 2.87s/it]
62%|██████▏ | 980/1570 [1:20:09<28:20, 2.88s/it]
{'loss': '0.0009018', 'grad_norm': '0.001421', 'learning_rate': '3.114e-05', 'epoch': '3.121'}
+
62%|██████▏ | 980/1570 [1:20:09<28:20, 2.88s/it]
62%|██████▏ | 981/1570 [1:20:12<28:16, 2.88s/it]
63%|██████▎ | 982/1570 [1:20:14<28:09, 2.87s/it]
63%|██████▎ | 983/1570 [1:20:17<28:08, 2.88s/it]
63%|██████▎ | 984/1570 [1:20:20<27:59, 2.87s/it]
63%|██████▎ | 985/1570 [1:20:23<27:59, 2.87s/it]
63%|██████▎ | 986/1570 [1:20:26<27:59, 2.88s/it]
63%|██████▎ | 987/1570 [1:20:29<27:49, 2.86s/it]
63%|██████▎ | 988/1570 [1:20:32<27:48, 2.87s/it]
63%|██████▎ | 989/1570 [1:20:35<27:44, 2.87s/it]
63%|██████▎ | 990/1570 [1:20:37<27:36, 2.86s/it]
63%|██████▎ | 991/1570 [1:20:40<27:36, 2.86s/it]
63%|██████▎ | 992/1570 [1:20:43<27:39, 2.87s/it]
63%|██████▎ | 993/1570 [1:20:46<27:29, 2.86s/it]
63%|██████▎ | 994/1570 [1:20:49<27:27, 2.86s/it]
63%|██████▎ | 995/1570 [1:20:52<27:37, 2.88s/it]
63%|██████▎ | 996/1570 [1:20:55<27:39, 2.89s/it]
64%|██████▎ | 997/1570 [1:20:57<27:23, 2.87s/it]
64%|██████▎ | 998/1570 [1:21:00<27:22, 2.87s/it]
64%|██████▎ | 999/1570 [1:21:03<27:23, 2.88s/it]
64%|██████▎ | 1000/1570 [1:21:06<27:25, 2.89s/it]
{'loss': '0.001016', 'grad_norm': '0.1232', 'learning_rate': '2.931e-05', 'epoch': '3.185'}
+
64%|██████▎ | 1000/1570 [1:21:06<27:25, 2.89s/it]
64%|██████▍ | 1001/1570 [1:21:09<27:19, 2.88s/it]
64%|██████▍ | 1002/1570 [1:21:12<27:08, 2.87s/it]
64%|██████▍ | 1003/1570 [1:21:15<27:09, 2.87s/it]
64%|██████▍ | 1004/1570 [1:21:18<27:05, 2.87s/it]
64%|██████▍ | 1005/1570 [1:21:20<26:58, 2.87s/it]
64%|██████▍ | 1006/1570 [1:21:23<26:56, 2.87s/it]
64%|██████▍ | 1007/1570 [1:21:26<26:58, 2.87s/it]
64%|██████▍ | 1008/1570 [1:21:29<26:54, 2.87s/it]
64%|██████▍ | 1009/1570 [1:21:32<26:47, 2.86s/it]
64%|██████▍ | 1010/1570 [1:21:35<26:43, 2.86s/it]
64%|██████▍ | 1011/1570 [1:21:38<26:52, 2.89s/it]
64%|██████▍ | 1012/1570 [1:21:41<26:49, 2.88s/it]
65%|██████▍ | 1013/1570 [1:21:44<26:47, 2.89s/it]
65%|██████▍ | 1014/1570 [1:21:46<26:47, 2.89s/it]
65%|██████▍ | 1015/1570 [1:21:49<26:41, 2.89s/it]
65%|██████▍ | 1016/1570 [1:21:52<26:41, 2.89s/it]
65%|██████▍ | 1017/1570 [1:21:55<26:38, 2.89s/it]
65%|██████▍ | 1018/1570 [1:21:58<26:28, 2.88s/it]
65%|██████▍ | 1019/1570 [1:22:01<26:23, 2.87s/it]
65%|██████▍ | 1020/1570 [1:22:04<26:21, 2.88s/it]
{'loss': '0.002848', 'grad_norm': '0.01639', 'learning_rate': '2.751e-05', 'epoch': '3.249'}
+
65%|██████▍ | 1020/1570 [1:22:04<26:21, 2.88s/it]
65%|██████▌ | 1021/1570 [1:22:07<26:21, 2.88s/it]
65%|██████▌ | 1022/1570 [1:22:09<26:17, 2.88s/it]
65%|██████▌ | 1023/1570 [1:22:12<26:10, 2.87s/it]
65%|██████▌ | 1024/1570 [1:22:15<26:05, 2.87s/it]
65%|██████▌ | 1025/1570 [1:22:18<26:03, 2.87s/it]
65%|██████▌ | 1026/1570 [1:22:21<26:05, 2.88s/it]
65%|██████▌ | 1027/1570 [1:22:24<26:08, 2.89s/it]
65%|██████▌ | 1028/1570 [1:22:27<26:01, 2.88s/it]
66%|██████▌ | 1029/1570 [1:22:30<25:59, 2.88s/it]
66%|██████▌ | 1030/1570 [1:22:32<25:53, 2.88s/it]
66%|██████▌ | 1031/1570 [1:22:35<25:50, 2.88s/it]
66%|██████▌ | 1032/1570 [1:22:38<25:57, 2.90s/it]
66%|██████▌ | 1033/1570 [1:22:41<25:57, 2.90s/it]
66%|██████▌ | 1034/1570 [1:22:44<26:00, 2.91s/it]
66%|██████▌ | 1035/1570 [1:22:47<25:51, 2.90s/it]
66%|██████▌ | 1036/1570 [1:22:50<25:44, 2.89s/it]
66%|██████▌ | 1037/1570 [1:22:53<25:50, 2.91s/it]
66%|██████▌ | 1038/1570 [1:22:56<25:50, 2.91s/it]
66%|██████▌ | 1039/1570 [1:22:59<25:50, 2.92s/it]
66%|██████▌ | 1040/1570 [1:23:02<25:40, 2.91s/it]
{'loss': '0.0006675', 'grad_norm': '0.1458', 'learning_rate': '2.574e-05', 'epoch': '3.313'}
+
66%|██████▌ | 1040/1570 [1:23:02<25:40, 2.91s/it]
66%|██████▋ | 1041/1570 [1:23:04<25:22, 2.88s/it]
66%|██████▋ | 1042/1570 [1:23:07<25:19, 2.88s/it]
66%|██████▋ | 1043/1570 [1:23:10<25:19, 2.88s/it]
66%|██████▋ | 1044/1570 [1:23:13<25:19, 2.89s/it]
67%|██████▋ | 1045/1570 [1:23:16<25:12, 2.88s/it]
67%|██████▋ | 1046/1570 [1:23:19<25:06, 2.88s/it]
67%|██████▋ | 1047/1570 [1:23:22<25:05, 2.88s/it]
67%|██████▋ | 1048/1570 [1:23:25<25:04, 2.88s/it]
67%|██████▋ | 1049/1570 [1:23:27<25:13, 2.91s/it]
67%|██████▋ | 1050/1570 [1:23:30<25:10, 2.90s/it]
67%|██████▋ | 1051/1570 [1:23:33<25:02, 2.89s/it]
67%|██████▋ | 1052/1570 [1:23:36<25:01, 2.90s/it]
67%|██████▋ | 1053/1570 [1:23:39<24:59, 2.90s/it]
67%|██████▋ | 1054/1570 [1:23:42<24:49, 2.89s/it]
67%|██████▋ | 1055/1570 [1:23:45<24:36, 2.87s/it]
67%|██████▋ | 1056/1570 [1:23:48<24:36, 2.87s/it]
67%|██████▋ | 1057/1570 [1:23:50<24:23, 2.85s/it]
67%|██████▋ | 1058/1570 [1:23:53<24:27, 2.87s/it]
67%|██████▋ | 1059/1570 [1:23:56<24:27, 2.87s/it]
68%|██████▊ | 1060/1570 [1:23:59<24:28, 2.88s/it]
{'loss': '0.001183', 'grad_norm': '0.001941', 'learning_rate': '2.402e-05', 'epoch': '3.377'}
+
68%|██████▊ | 1060/1570 [1:23:59<24:28, 2.88s/it]
68%|██████▊ | 1061/1570 [1:24:02<24:38, 2.90s/it]
68%|██████▊ | 1062/1570 [1:24:05<24:32, 2.90s/it]
68%|██████▊ | 1063/1570 [1:24:08<24:17, 2.87s/it]
68%|██████▊ | 1064/1570 [1:24:11<24:20, 2.89s/it]
68%|██████▊ | 1065/1570 [1:24:14<24:22, 2.90s/it]
68%|██████▊ | 1066/1570 [1:24:16<24:16, 2.89s/it]
68%|██████▊ | 1067/1570 [1:24:19<24:10, 2.88s/it]
68%|██████▊ | 1068/1570 [1:24:22<24:05, 2.88s/it]
68%|██████▊ | 1069/1570 [1:24:25<24:11, 2.90s/it]
68%|██████▊ | 1070/1570 [1:24:28<24:02, 2.89s/it]
68%|██████▊ | 1071/1570 [1:24:31<24:05, 2.90s/it]
68%|██████▊ | 1072/1570 [1:24:34<24:03, 2.90s/it]
68%|██████▊ | 1073/1570 [1:24:37<23:55, 2.89s/it]
68%|██████▊ | 1074/1570 [1:24:40<23:53, 2.89s/it]
68%|██████▊ | 1075/1570 [1:24:42<23:45, 2.88s/it]
69%|██████▊ | 1076/1570 [1:24:45<23:41, 2.88s/it]
69%|██████▊ | 1077/1570 [1:24:48<23:35, 2.87s/it]
69%|██████▊ | 1078/1570 [1:24:51<23:44, 2.90s/it]
69%|██████▊ | 1079/1570 [1:24:54<23:33, 2.88s/it]
69%|██████▉ | 1080/1570 [1:24:57<23:30, 2.88s/it]
{'loss': '0.002316', 'grad_norm': '0.07083', 'learning_rate': '2.233e-05', 'epoch': '3.44'}
+
69%|██████▉ | 1080/1570 [1:24:57<23:30, 2.88s/it]
69%|██████▉ | 1081/1570 [1:25:00<23:29, 2.88s/it]
69%|██████▉ | 1082/1570 [1:25:03<23:27, 2.88s/it]
69%|██████▉ | 1083/1570 [1:25:06<23:25, 2.89s/it]
69%|██████▉ | 1084/1570 [1:25:08<23:29, 2.90s/it]
69%|██████▉ | 1085/1570 [1:25:11<23:23, 2.89s/it]
69%|██████▉ | 1086/1570 [1:25:14<23:19, 2.89s/it]
69%|██████▉ | 1087/1570 [1:25:17<23:15, 2.89s/it]
69%|██████▉ | 1088/1570 [1:25:20<23:10, 2.89s/it]
69%|██████▉ | 1089/1570 [1:25:23<23:07, 2.88s/it]
69%|██████▉ | 1090/1570 [1:25:26<23:00, 2.88s/it]
69%|██████▉ | 1091/1570 [1:25:29<22:56, 2.87s/it]
70%|██████▉ | 1092/1570 [1:25:32<22:58, 2.88s/it]
70%|██████▉ | 1093/1570 [1:25:34<22:53, 2.88s/it]
70%|██████▉ | 1094/1570 [1:25:37<22:52, 2.88s/it]
70%|██████▉ | 1095/1570 [1:25:40<22:46, 2.88s/it]
70%|██████▉ | 1096/1570 [1:25:43<22:47, 2.88s/it]
70%|██████▉ | 1097/1570 [1:25:46<22:44, 2.89s/it]
70%|██████▉ | 1098/1570 [1:25:49<22:48, 2.90s/it]
70%|███████ | 1099/1570 [1:25:52<22:52, 2.91s/it]
70%|███████ | 1100/1570 [1:25:55<22:45, 2.90s/it]
{'loss': '0.002298', 'grad_norm': '0.01226', 'learning_rate': '2.069e-05', 'epoch': '3.504'}
+
70%|███████ | 1100/1570 [1:25:55<22:45, 2.90s/it]
70%|███████ | 1101/1570 [1:25:58<22:48, 2.92s/it]
70%|███████ | 1102/1570 [1:26:01<22:46, 2.92s/it]
70%|███████ | 1103/1570 [1:26:03<22:38, 2.91s/it]
70%|███████ | 1104/1570 [1:26:06<22:29, 2.90s/it]
70%|███████ | 1105/1570 [1:26:09<22:25, 2.89s/it]
70%|███████ | 1106/1570 [1:26:12<22:24, 2.90s/it]
71%|███████ | 1107/1570 [1:26:15<22:26, 2.91s/it]
71%|███████ | 1108/1570 [1:26:18<22:18, 2.90s/it]
71%|███████ | 1109/1570 [1:26:21<22:12, 2.89s/it]
71%|███████ | 1110/1570 [1:26:24<22:08, 2.89s/it]
71%|███████ | 1111/1570 [1:26:27<22:01, 2.88s/it]
71%|███████ | 1112/1570 [1:26:29<21:59, 2.88s/it]
71%|███████ | 1113/1570 [1:26:32<22:03, 2.90s/it]
71%|███████ | 1114/1570 [1:26:35<21:57, 2.89s/it]
71%|███████ | 1115/1570 [1:26:38<21:57, 2.90s/it]
71%|███████ | 1116/1570 [1:26:41<21:52, 2.89s/it]
71%|███████ | 1117/1570 [1:26:44<21:50, 2.89s/it]
71%|███████ | 1118/1570 [1:26:47<21:44, 2.89s/it]
71%|███████▏ | 1119/1570 [1:26:50<21:48, 2.90s/it]
71%|███████▏ | 1120/1570 [1:26:53<21:40, 2.89s/it]
{'loss': '0.001326', 'grad_norm': '0.01279', 'learning_rate': '1.91e-05', 'epoch': '3.568'}
+
71%|███████▏ | 1120/1570 [1:26:53<21:40, 2.89s/it]
71%|███████▏ | 1121/1570 [1:26:55<21:40, 2.90s/it]
71%|███████▏ | 1122/1570 [1:26:58<21:42, 2.91s/it]
72%|███████▏ | 1123/1570 [1:27:01<21:36, 2.90s/it]
72%|███████▏ | 1124/1570 [1:27:04<21:24, 2.88s/it]
72%|███████▏ | 1125/1570 [1:27:07<21:18, 2.87s/it]
72%|███████▏ | 1126/1570 [1:27:10<21:23, 2.89s/it]
72%|███████▏ | 1127/1570 [1:27:13<21:13, 2.87s/it]
72%|███████▏ | 1128/1570 [1:27:16<21:15, 2.89s/it]
72%|███████▏ | 1129/1570 [1:27:19<21:14, 2.89s/it]
72%|███████▏ | 1130/1570 [1:27:21<21:14, 2.90s/it]
72%|███████▏ | 1131/1570 [1:27:24<21:12, 2.90s/it]
72%|███████▏ | 1132/1570 [1:27:27<21:08, 2.90s/it]
72%|███████▏ | 1133/1570 [1:27:30<21:05, 2.90s/it]
72%|███████▏ | 1134/1570 [1:27:33<20:59, 2.89s/it]
72%|███████▏ | 1135/1570 [1:27:36<21:03, 2.90s/it]
72%|███████▏ | 1136/1570 [1:27:39<20:58, 2.90s/it]
72%|███████▏ | 1137/1570 [1:27:42<20:50, 2.89s/it]
72%|███████▏ | 1138/1570 [1:27:45<20:49, 2.89s/it]
73%|███████▎ | 1139/1570 [1:27:47<20:40, 2.88s/it]
73%|███████▎ | 1140/1570 [1:27:50<20:45, 2.90s/it]
{'loss': '0.0007607', 'grad_norm': '0.07144', 'learning_rate': '1.755e-05', 'epoch': '3.632'}
+
73%|███████▎ | 1140/1570 [1:27:50<20:45, 2.90s/it]
73%|███████▎ | 1141/1570 [1:27:53<20:47, 2.91s/it]
73%|███████▎ | 1142/1570 [1:27:56<20:40, 2.90s/it]
73%|███████▎ | 1143/1570 [1:27:59<20:37, 2.90s/it]
73%|███████▎ | 1144/1570 [1:28:02<20:38, 2.91s/it]
73%|███████▎ | 1145/1570 [1:28:05<20:34, 2.90s/it]
73%|███████▎ | 1146/1570 [1:28:08<20:30, 2.90s/it]
73%|███████▎ | 1147/1570 [1:28:11<20:26, 2.90s/it]
73%|███████▎ | 1148/1570 [1:28:14<20:25, 2.90s/it]
73%|███████▎ | 1149/1570 [1:28:17<20:20, 2.90s/it]
73%|███████▎ | 1150/1570 [1:28:19<20:12, 2.89s/it]
73%|███████▎ | 1151/1570 [1:28:22<20:05, 2.88s/it]
73%|███████▎ | 1152/1570 [1:28:25<19:58, 2.87s/it]
73%|███████▎ | 1153/1570 [1:28:28<19:55, 2.87s/it]
74%|███████▎ | 1154/1570 [1:28:31<19:53, 2.87s/it]
74%|███████▎ | 1155/1570 [1:28:34<19:55, 2.88s/it]
74%|███████▎ | 1156/1570 [1:28:37<19:45, 2.86s/it]
74%|███████▎ | 1157/1570 [1:28:39<19:40, 2.86s/it]
74%|███████▍ | 1158/1570 [1:28:42<19:41, 2.87s/it]
74%|███████▍ | 1159/1570 [1:28:45<19:35, 2.86s/it]
74%|███████▍ | 1160/1570 [1:28:48<19:33, 2.86s/it]
{'loss': '0.002803', 'grad_norm': '0.5528', 'learning_rate': '1.606e-05', 'epoch': '3.696'}
+
74%|███████▍ | 1160/1570 [1:28:48<19:33, 2.86s/it]
74%|███████▍ | 1161/1570 [1:28:51<19:32, 2.87s/it]
74%|███████▍ | 1162/1570 [1:28:54<19:32, 2.87s/it]
74%|███████▍ | 1163/1570 [1:28:57<19:29, 2.87s/it]
74%|███████▍ | 1164/1570 [1:29:00<19:22, 2.86s/it]
74%|███████▍ | 1165/1570 [1:29:02<19:20, 2.87s/it]
74%|███████▍ | 1166/1570 [1:29:05<19:19, 2.87s/it]
74%|███████▍ | 1167/1570 [1:29:08<19:08, 2.85s/it]
74%|███████▍ | 1168/1570 [1:29:11<19:08, 2.86s/it]
74%|███████▍ | 1169/1570 [1:29:14<19:05, 2.86s/it]
75%|███████▍ | 1170/1570 [1:29:17<19:07, 2.87s/it]
75%|███████▍ | 1171/1570 [1:29:20<19:03, 2.87s/it]
75%|███████▍ | 1172/1570 [1:29:22<18:59, 2.86s/it]
75%|███████▍ | 1173/1570 [1:29:25<18:55, 2.86s/it]
75%|███████▍ | 1174/1570 [1:29:28<19:01, 2.88s/it]
75%|███████▍ | 1175/1570 [1:29:31<18:57, 2.88s/it]
75%|███████▍ | 1176/1570 [1:29:34<18:58, 2.89s/it]
75%|███████▍ | 1177/1570 [1:29:37<18:57, 2.89s/it]
75%|███████▌ | 1178/1570 [1:29:40<18:46, 2.87s/it]
75%|███████▌ | 1179/1570 [1:29:43<18:44, 2.88s/it]
75%|███████▌ | 1180/1570 [1:29:45<18:40, 2.87s/it]
{'loss': '0.001243', 'grad_norm': '0.01241', 'learning_rate': '1.462e-05', 'epoch': '3.759'}
+
75%|███████▌ | 1180/1570 [1:29:45<18:40, 2.87s/it]
75%|███████▌ | 1181/1570 [1:29:48<18:46, 2.90s/it]
75%|███████▌ | 1182/1570 [1:29:51<18:42, 2.89s/it]
75%|███████▌ | 1183/1570 [1:29:54<18:40, 2.90s/it]
75%|███████▌ | 1184/1570 [1:29:57<18:34, 2.89s/it]
75%|███████▌ | 1185/1570 [1:30:00<18:32, 2.89s/it]
76%|███████▌ | 1186/1570 [1:30:03<18:35, 2.90s/it]
76%|███████▌ | 1187/1570 [1:30:06<18:32, 2.90s/it]
76%|███████▌ | 1188/1570 [1:30:09<18:31, 2.91s/it]
76%|███████▌ | 1189/1570 [1:30:12<18:28, 2.91s/it]
76%|███████▌ | 1190/1570 [1:30:15<18:29, 2.92s/it]
76%|███████▌ | 1191/1570 [1:30:17<18:20, 2.90s/it]
76%|███████▌ | 1192/1570 [1:30:20<18:18, 2.91s/it]
76%|███████▌ | 1193/1570 [1:30:23<18:18, 2.91s/it]
76%|███████▌ | 1194/1570 [1:30:26<18:14, 2.91s/it]
76%|███████▌ | 1195/1570 [1:30:29<18:09, 2.91s/it]
76%|███████▌ | 1196/1570 [1:30:32<18:06, 2.90s/it]
76%|███████▌ | 1197/1570 [1:30:35<18:01, 2.90s/it]
76%|███████▋ | 1198/1570 [1:30:38<18:01, 2.91s/it]
76%|███████▋ | 1199/1570 [1:30:41<18:03, 2.92s/it]
76%|███████▋ | 1200/1570 [1:30:44<18:00, 2.92s/it]
{'loss': '0.001273', 'grad_norm': '0.05056', 'learning_rate': '1.324e-05', 'epoch': '3.823'}
+
76%|███████▋ | 1200/1570 [1:30:44<18:00, 2.92s/it]
76%|███████▋ | 1201/1570 [1:30:47<17:58, 2.92s/it]
77%|███████▋ | 1202/1570 [1:30:50<17:55, 2.92s/it]
77%|███████▋ | 1203/1570 [1:30:52<17:49, 2.91s/it]
77%|███████▋ | 1204/1570 [1:30:55<17:50, 2.93s/it]
77%|███████▋ | 1205/1570 [1:30:58<17:42, 2.91s/it]
77%|███████▋ | 1206/1570 [1:31:01<17:39, 2.91s/it]
77%|███████▋ | 1207/1570 [1:31:04<17:33, 2.90s/it]
77%|███████▋ | 1208/1570 [1:31:07<17:34, 2.91s/it]
77%|███████▋ | 1209/1570 [1:31:10<17:31, 2.91s/it]
77%|███████▋ | 1210/1570 [1:31:13<17:28, 2.91s/it]
77%|███████▋ | 1211/1570 [1:31:16<17:26, 2.91s/it]
77%|███████▋ | 1212/1570 [1:31:19<17:20, 2.91s/it]
77%|███████▋ | 1213/1570 [1:31:21<17:11, 2.89s/it]
77%|███████▋ | 1214/1570 [1:31:24<17:11, 2.90s/it]
77%|███████▋ | 1215/1570 [1:31:27<17:08, 2.90s/it]
77%|███████▋ | 1216/1570 [1:31:30<17:08, 2.90s/it]
78%|███████▊ | 1217/1570 [1:31:33<17:04, 2.90s/it]
78%|███████▊ | 1218/1570 [1:31:36<16:59, 2.90s/it]
78%|███████▊ | 1219/1570 [1:31:39<16:58, 2.90s/it]
78%|███████▊ | 1220/1570 [1:31:42<16:50, 2.89s/it]
{'loss': '0.0007077', 'grad_norm': '0.0344', 'learning_rate': '1.192e-05', 'epoch': '3.887'}
+
78%|███████▊ | 1220/1570 [1:31:42<16:50, 2.89s/it]
78%|███████▊ | 1221/1570 [1:31:45<16:49, 2.89s/it]
78%|███████▊ | 1222/1570 [1:31:48<16:43, 2.88s/it]
78%|███████▊ | 1223/1570 [1:31:50<16:37, 2.87s/it]
78%|███████▊ | 1224/1570 [1:31:53<16:36, 2.88s/it]
78%|███████▊ | 1225/1570 [1:31:56<16:34, 2.88s/it]
78%|███████▊ | 1226/1570 [1:31:59<16:32, 2.88s/it]
78%|███████▊ | 1227/1570 [1:32:02<16:34, 2.90s/it]
78%|███████▊ | 1228/1570 [1:32:05<16:33, 2.90s/it]
78%|███████▊ | 1229/1570 [1:32:08<16:29, 2.90s/it]
78%|███████▊ | 1230/1570 [1:32:11<16:28, 2.91s/it]
78%|███████▊ | 1231/1570 [1:32:14<16:25, 2.91s/it]
78%|███████▊ | 1232/1570 [1:32:16<16:19, 2.90s/it]
79%|███████▊ | 1233/1570 [1:32:19<16:12, 2.89s/it]
79%|███████▊ | 1234/1570 [1:32:22<16:11, 2.89s/it]
79%|███████▊ | 1235/1570 [1:32:25<16:04, 2.88s/it]
79%|███████▊ | 1236/1570 [1:32:28<16:02, 2.88s/it]
79%|███████▉ | 1237/1570 [1:32:31<15:56, 2.87s/it]
79%|███████▉ | 1238/1570 [1:32:34<15:51, 2.87s/it]
79%|███████▉ | 1239/1570 [1:32:37<15:51, 2.88s/it]
79%|███████▉ | 1240/1570 [1:32:39<15:49, 2.88s/it]
{'loss': '0.0005449', 'grad_norm': '0.01693', 'learning_rate': '1.066e-05', 'epoch': '3.951'}
+
79%|███████▉ | 1240/1570 [1:32:39<15:49, 2.88s/it]
79%|███████▉ | 1241/1570 [1:32:42<15:49, 2.89s/it]
79%|███████▉ | 1242/1570 [1:32:45<15:42, 2.87s/it]
79%|███████▉ | 1243/1570 [1:32:48<15:45, 2.89s/it]
79%|███████▉ | 1244/1570 [1:32:51<15:42, 2.89s/it]
79%|███████▉ | 1245/1570 [1:32:54<15:38, 2.89s/it]
79%|███████▉ | 1246/1570 [1:32:57<15:40, 2.90s/it]
79%|███████▉ | 1247/1570 [1:33:00<15:39, 2.91s/it]
79%|███████▉ | 1248/1570 [1:33:03<15:36, 2.91s/it]
80%|███████▉ | 1249/1570 [1:33:06<15:30, 2.90s/it]
80%|███████▉ | 1250/1570 [1:33:09<15:31, 2.91s/it]
80%|███████▉ | 1251/1570 [1:33:11<15:26, 2.90s/it]
80%|███████▉ | 1252/1570 [1:33:14<15:20, 2.89s/it]
80%|███████▉ | 1253/1570 [1:33:17<15:17, 2.89s/it]
80%|███████▉ | 1254/1570 [1:33:20<15:12, 2.89s/it]
80%|███████▉ | 1255/1570 [1:33:23<15:15, 2.91s/it]
80%|████████ | 1256/1570 [1:33:24<12:26, 2.38s/it][INFO|trainer.py:2668] 2026-08-22 18:49:43,505 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 18:49:43,505 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 18:49:43,505 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/srkl/checkpoint-1256
+[INFO|configuration_utils.py:780] 2026-08-22 18:59:46,316 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 18:59:46,319 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 18:59:46,353 >> chat template saved in results/qwen3/srkl/checkpoint-1256/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 18:59:46,353 >> tokenizer config file saved in results/qwen3/srkl/checkpoint-1256/tokenizer_config.json
+
80%|████████ | 1257/1570 [1:43:31<15:58:14, 183.69s/it]
80%|████████ | 1258/1570 [1:43:34<11:13:09, 129.45s/it]
80%|████████ | 1259/1570 [1:43:37<7:54:13, 91.49s/it]
80%|████████ | 1260/1570 [1:43:40<5:35:24, 64.92s/it]
{'loss': '0.0009123', 'grad_norm': '0.02643', 'learning_rate': '9.464e-06', 'epoch': '4.013'}
+
80%|████████ | 1260/1570 [1:43:40<5:35:24, 64.92s/it]
80%|████████ | 1261/1570 [1:43:42<3:58:30, 46.31s/it]
80%|████████ | 1262/1570 [1:43:45<2:50:50, 33.28s/it]
80%|████████ | 1263/1570 [1:43:48<2:03:38, 24.16s/it]
81%|████████ | 1264/1570 [1:43:51<1:30:40, 17.78s/it]
81%|████████ | 1265/1570 [1:43:54<1:07:37, 13.30s/it]
81%|████████ | 1266/1570 [1:43:57<51:32, 10.17s/it]
81%|████████ | 1267/1570 [1:44:00<40:18, 7.98s/it]
81%|████████ | 1268/1570 [1:44:03<32:28, 6.45s/it]
81%|████████ | 1269/1570 [1:44:06<26:59, 5.38s/it]
81%|████████ | 1270/1570 [1:44:08<23:09, 4.63s/it]
81%|████████ | 1271/1570 [1:44:11<20:31, 4.12s/it]
81%|████████ | 1272/1570 [1:44:14<18:37, 3.75s/it]
81%|████████ | 1273/1570 [1:44:17<17:18, 3.50s/it]
81%|████████ | 1274/1570 [1:44:20<16:21, 3.32s/it]
81%|████████ | 1275/1570 [1:44:23<15:43, 3.20s/it]
81%|████████▏ | 1276/1570 [1:44:26<15:10, 3.10s/it]
81%|████████▏ | 1277/1570 [1:44:29<14:48, 3.03s/it]
81%|████████▏ | 1278/1570 [1:44:32<14:28, 2.98s/it]
81%|████████▏ | 1279/1570 [1:44:34<14:16, 2.94s/it]
82%|████████▏ | 1280/1570 [1:44:37<14:08, 2.92s/it]
{'loss': '0.0002503', 'grad_norm': '0.0545', 'learning_rate': '8.332e-06', 'epoch': '4.077'}
+
82%|████████▏ | 1280/1570 [1:44:37<14:08, 2.92s/it]
82%|████████▏ | 1281/1570 [1:44:40<13:59, 2.91s/it]
82%|████████▏ | 1282/1570 [1:44:43<13:54, 2.90s/it]
82%|████████▏ | 1283/1570 [1:44:46<13:46, 2.88s/it]
82%|████████▏ | 1284/1570 [1:44:49<13:47, 2.89s/it]
82%|████████▏ | 1285/1570 [1:44:52<13:39, 2.87s/it]
82%|████████▏ | 1286/1570 [1:44:54<13:36, 2.88s/it]
82%|████████▏ | 1287/1570 [1:44:57<13:34, 2.88s/it]
82%|████████▏ | 1288/1570 [1:45:00<13:30, 2.88s/it]
82%|████████▏ | 1289/1570 [1:45:03<13:25, 2.87s/it]
82%|████████▏ | 1290/1570 [1:45:06<13:27, 2.88s/it]
82%|████████▏ | 1291/1570 [1:45:09<13:25, 2.89s/it]
82%|████████▏ | 1292/1570 [1:45:12<13:23, 2.89s/it]
82%|████████▏ | 1293/1570 [1:45:15<13:23, 2.90s/it]
82%|████████▏ | 1294/1570 [1:45:18<13:19, 2.90s/it]
82%|████████▏ | 1295/1570 [1:45:21<13:18, 2.90s/it]
83%|████████▎ | 1296/1570 [1:45:23<13:15, 2.90s/it]
83%|████████▎ | 1297/1570 [1:45:26<13:14, 2.91s/it]
83%|████████▎ | 1298/1570 [1:45:29<13:05, 2.89s/it]
83%|████████▎ | 1299/1570 [1:45:32<13:04, 2.89s/it]
83%|████████▎ | 1300/1570 [1:45:35<12:58, 2.88s/it]
{'loss': '0.0005663', 'grad_norm': '0.004517', 'learning_rate': '7.266e-06', 'epoch': '4.14'}
+
83%|████████▎ | 1300/1570 [1:45:35<12:58, 2.88s/it]
83%|████████▎ | 1301/1570 [1:45:38<12:58, 2.89s/it]
83%|████████▎ | 1302/1570 [1:45:41<12:54, 2.89s/it]
83%|████████▎ | 1303/1570 [1:45:44<12:53, 2.90s/it]
83%|████████▎ | 1304/1570 [1:45:47<12:48, 2.89s/it]
83%|████████▎ | 1305/1570 [1:45:49<12:49, 2.90s/it]
83%|████████▎ | 1306/1570 [1:45:52<12:45, 2.90s/it]
83%|████████▎ | 1307/1570 [1:45:55<12:43, 2.90s/it]
83%|████████▎ | 1308/1570 [1:45:58<12:39, 2.90s/it]
83%|████████▎ | 1309/1570 [1:46:01<12:36, 2.90s/it]
83%|████████▎ | 1310/1570 [1:46:04<12:33, 2.90s/it]
84%|████████▎ | 1311/1570 [1:46:07<12:31, 2.90s/it]
84%|████████▎ | 1312/1570 [1:46:10<12:25, 2.89s/it]
84%|████████▎ | 1313/1570 [1:46:13<12:19, 2.88s/it]
84%|████████▎ | 1314/1570 [1:46:15<12:14, 2.87s/it]
84%|████████▍ | 1315/1570 [1:46:18<12:13, 2.88s/it]
84%|████████▍ | 1316/1570 [1:46:21<12:11, 2.88s/it]
84%|████████▍ | 1317/1570 [1:46:24<12:06, 2.87s/it]
84%|████████▍ | 1318/1570 [1:46:27<12:05, 2.88s/it]
84%|████████▍ | 1319/1570 [1:46:30<12:01, 2.88s/it]
84%|████████▍ | 1320/1570 [1:46:33<12:03, 2.89s/it]
{'loss': '0.0004728', 'grad_norm': '0.01496', 'learning_rate': '6.269e-06', 'epoch': '4.204'}
+
84%|████████▍ | 1320/1570 [1:46:33<12:03, 2.89s/it]
84%|████████▍ | 1321/1570 [1:46:36<11:59, 2.89s/it]
84%|████████▍ | 1322/1570 [1:46:39<11:56, 2.89s/it]
84%|████████▍ | 1323/1570 [1:46:41<11:54, 2.89s/it]
84%|████████▍ | 1324/1570 [1:46:44<11:51, 2.89s/it]
84%|████████▍ | 1325/1570 [1:46:47<11:52, 2.91s/it]
84%|████████▍ | 1326/1570 [1:46:50<11:50, 2.91s/it]
85%|████████▍ | 1327/1570 [1:46:53<11:47, 2.91s/it]
85%|████████▍ | 1328/1570 [1:46:56<11:41, 2.90s/it]
85%|████████▍ | 1329/1570 [1:46:59<11:41, 2.91s/it]
85%|████████▍ | 1330/1570 [1:47:02<11:34, 2.89s/it]
85%|████████▍ | 1331/1570 [1:47:05<11:25, 2.87s/it]
85%|████████▍ | 1332/1570 [1:47:07<11:27, 2.89s/it]
85%|████████▍ | 1333/1570 [1:47:10<11:25, 2.89s/it]
85%|████████▍ | 1334/1570 [1:47:13<11:20, 2.88s/it]
85%|████████▌ | 1335/1570 [1:47:16<11:13, 2.87s/it]
85%|████████▌ | 1336/1570 [1:47:19<11:08, 2.86s/it]
85%|████████▌ | 1337/1570 [1:47:22<11:09, 2.87s/it]
85%|████████▌ | 1338/1570 [1:47:25<11:07, 2.88s/it]
85%|████████▌ | 1339/1570 [1:47:28<11:06, 2.89s/it]
85%|████████▌ | 1340/1570 [1:47:31<11:04, 2.89s/it]
{'loss': '0.000712', 'grad_norm': '0.03497', 'learning_rate': '5.342e-06', 'epoch': '4.268'}
+
85%|████████▌ | 1340/1570 [1:47:31<11:04, 2.89s/it]
85%|████████▌ | 1341/1570 [1:47:33<11:01, 2.89s/it]
85%|████████▌ | 1342/1570 [1:47:36<10:59, 2.89s/it]
86%|████████▌ | 1343/1570 [1:47:39<10:56, 2.89s/it]
86%|████████▌ | 1344/1570 [1:47:42<10:54, 2.90s/it]
86%|████████▌ | 1345/1570 [1:47:45<10:52, 2.90s/it]
86%|████████▌ | 1346/1570 [1:47:48<10:51, 2.91s/it]
86%|████████▌ | 1347/1570 [1:47:51<10:46, 2.90s/it]
86%|████████▌ | 1348/1570 [1:47:54<10:39, 2.88s/it]
86%|████████▌ | 1349/1570 [1:47:57<10:35, 2.88s/it]
86%|████████▌ | 1350/1570 [1:47:59<10:33, 2.88s/it]
86%|████████▌ | 1351/1570 [1:48:03<11:01, 3.02s/it]
86%|████████▌ | 1352/1570 [1:48:06<10:50, 2.99s/it]
86%|████████▌ | 1353/1570 [1:48:09<10:41, 2.96s/it]
86%|████████▌ | 1354/1570 [1:48:11<10:35, 2.94s/it]
86%|████████▋ | 1355/1570 [1:48:14<10:28, 2.92s/it]
86%|████████▋ | 1356/1570 [1:48:17<10:20, 2.90s/it]
86%|████████▋ | 1357/1570 [1:48:20<10:16, 2.89s/it]
86%|████████▋ | 1358/1570 [1:48:23<10:10, 2.88s/it]
87%|████████▋ | 1359/1570 [1:48:26<10:07, 2.88s/it]
87%|████████▋ | 1360/1570 [1:48:29<10:08, 2.90s/it]
{'loss': '0.001403', 'grad_norm': '0.006169', 'learning_rate': '4.486e-06', 'epoch': '4.332'}
+
87%|████████▋ | 1360/1570 [1:48:29<10:08, 2.90s/it]
87%|████████▋ | 1361/1570 [1:48:32<10:05, 2.90s/it]
87%|████████▋ | 1362/1570 [1:48:34<09:59, 2.88s/it]
87%|████████▋ | 1363/1570 [1:48:37<09:54, 2.87s/it]
87%|████████▋ | 1364/1570 [1:48:40<09:53, 2.88s/it]
87%|████████▋ | 1365/1570 [1:48:43<09:50, 2.88s/it]
87%|████████▋ | 1366/1570 [1:48:46<09:47, 2.88s/it]
87%|████████▋ | 1367/1570 [1:48:49<09:43, 2.88s/it]
87%|████████▋ | 1368/1570 [1:48:52<09:41, 2.88s/it]
87%|████████▋ | 1369/1570 [1:48:55<09:37, 2.87s/it]
87%|████████▋ | 1370/1570 [1:48:57<09:35, 2.88s/it]
87%|████████▋ | 1371/1570 [1:49:00<09:31, 2.87s/it]
87%|████████▋ | 1372/1570 [1:49:03<09:31, 2.89s/it]
87%|████████▋ | 1373/1570 [1:49:06<09:25, 2.87s/it]
88%|████████▊ | 1374/1570 [1:49:09<09:23, 2.87s/it]
88%|████████▊ | 1375/1570 [1:49:12<09:22, 2.89s/it]
88%|████████▊ | 1376/1570 [1:49:15<09:19, 2.88s/it]
88%|████████▊ | 1377/1570 [1:49:18<09:12, 2.86s/it]
88%|████████▊ | 1378/1570 [1:49:20<09:11, 2.87s/it]
88%|████████▊ | 1379/1570 [1:49:23<09:10, 2.88s/it]
88%|████████▊ | 1380/1570 [1:49:26<09:07, 2.88s/it]
{'loss': '0.0008245', 'grad_norm': '0.02854', 'learning_rate': '3.704e-06', 'epoch': '4.396'}
+
88%|████████▊ | 1380/1570 [1:49:26<09:07, 2.88s/it]
88%|████████▊ | 1381/1570 [1:49:29<09:07, 2.90s/it]
88%|████████▊ | 1382/1570 [1:49:32<09:03, 2.89s/it]
88%|████████▊ | 1383/1570 [1:49:35<09:01, 2.89s/it]
88%|████████▊ | 1384/1570 [1:49:38<08:56, 2.88s/it]
88%|████████▊ | 1385/1570 [1:49:41<08:54, 2.89s/it]
88%|████████▊ | 1386/1570 [1:49:44<08:49, 2.88s/it]
88%|████████▊ | 1387/1570 [1:49:46<08:48, 2.89s/it]
88%|████████▊ | 1388/1570 [1:49:49<08:46, 2.89s/it]
88%|████████▊ | 1389/1570 [1:49:52<08:43, 2.89s/it]
89%|████████▊ | 1390/1570 [1:49:56<09:02, 3.01s/it]
89%|████████▊ | 1391/1570 [1:49:58<08:53, 2.98s/it]
89%|████████▊ | 1392/1570 [1:50:01<08:42, 2.93s/it]
89%|████████▊ | 1393/1570 [1:50:04<08:36, 2.92s/it]
89%|████████▉ | 1394/1570 [1:50:07<08:31, 2.91s/it]
89%|████████▉ | 1395/1570 [1:50:10<08:27, 2.90s/it]
89%|████████▉ | 1396/1570 [1:50:13<08:25, 2.90s/it]
89%|████████▉ | 1397/1570 [1:50:16<08:20, 2.89s/it]
89%|████████▉ | 1398/1570 [1:50:19<08:19, 2.90s/it]
89%|████████▉ | 1399/1570 [1:50:22<08:16, 2.90s/it]
89%|████████▉ | 1400/1570 [1:50:24<08:13, 2.90s/it]
{'loss': '0.0009157', 'grad_norm': '0.03609', 'learning_rate': '2.996e-06', 'epoch': '4.46'}
+
89%|████████▉ | 1400/1570 [1:50:24<08:13, 2.90s/it]
89%|████████▉ | 1401/1570 [1:50:27<08:07, 2.88s/it]
89%|████████▉ | 1402/1570 [1:50:30<08:07, 2.90s/it]
89%|████████▉ | 1403/1570 [1:50:33<08:03, 2.90s/it]
89%|████████▉ | 1404/1570 [1:50:36<08:03, 2.91s/it]
89%|████████▉ | 1405/1570 [1:50:39<07:59, 2.90s/it]
90%|████████▉ | 1406/1570 [1:50:42<07:55, 2.90s/it]
90%|████████▉ | 1407/1570 [1:50:45<07:51, 2.89s/it]
90%|████████▉ | 1408/1570 [1:50:48<07:49, 2.90s/it]
90%|████████▉ | 1409/1570 [1:50:51<07:46, 2.90s/it]
90%|████████▉ | 1410/1570 [1:50:53<07:43, 2.89s/it]
90%|████████▉ | 1411/1570 [1:50:56<07:42, 2.91s/it]
90%|████████▉ | 1412/1570 [1:50:59<07:40, 2.92s/it]
90%|█████████ | 1413/1570 [1:51:02<07:37, 2.91s/it]
90%|█████████ | 1414/1570 [1:51:05<07:32, 2.90s/it]
90%|█████████ | 1415/1570 [1:51:08<07:30, 2.91s/it]
90%|█████████ | 1416/1570 [1:51:11<07:25, 2.89s/it]
90%|█████████ | 1417/1570 [1:51:14<07:21, 2.88s/it]
90%|█████████ | 1418/1570 [1:51:17<07:17, 2.88s/it]
90%|█████████ | 1419/1570 [1:51:19<07:14, 2.88s/it]
90%|█████████ | 1420/1570 [1:51:22<07:13, 2.89s/it]
{'loss': '0.00066', 'grad_norm': '0.04012', 'learning_rate': '2.363e-06', 'epoch': '4.523'}
+
90%|█████████ | 1420/1570 [1:51:22<07:13, 2.89s/it]
91%|█████████ | 1421/1570 [1:51:25<07:10, 2.89s/it]
91%|█████████ | 1422/1570 [1:51:28<07:05, 2.87s/it]
91%|█████████ | 1423/1570 [1:51:31<07:02, 2.87s/it]
91%|█████████ | 1424/1570 [1:51:34<07:01, 2.88s/it]
91%|█████████ | 1425/1570 [1:51:37<06:59, 2.89s/it]
91%|█████████ | 1426/1570 [1:51:40<06:57, 2.90s/it]
91%|█████████ | 1427/1570 [1:51:43<06:52, 2.88s/it]
91%|█████████ | 1428/1570 [1:51:45<06:49, 2.89s/it]
91%|█████████ | 1429/1570 [1:51:48<06:47, 2.89s/it]
91%|█████████ | 1430/1570 [1:51:51<06:43, 2.88s/it]
91%|█████████ | 1431/1570 [1:51:54<06:40, 2.88s/it]
91%|█████████ | 1432/1570 [1:51:57<06:39, 2.89s/it]
91%|█████████▏| 1433/1570 [1:52:00<06:36, 2.90s/it]
91%|█████████▏| 1434/1570 [1:52:03<06:35, 2.91s/it]
91%|█████████▏| 1435/1570 [1:52:06<06:33, 2.92s/it]
91%|█████████▏| 1436/1570 [1:52:09<06:30, 2.91s/it]
92%|█████████▏| 1437/1570 [1:52:12<06:27, 2.91s/it]
92%|█████████▏| 1438/1570 [1:52:15<06:26, 2.93s/it]
92%|█████████▏| 1439/1570 [1:52:17<06:23, 2.93s/it]
92%|█████████▏| 1440/1570 [1:52:20<06:21, 2.93s/it]
{'loss': '0.0006631', 'grad_norm': '0.01892', 'learning_rate': '1.806e-06', 'epoch': '4.587'}
+
92%|█████████▏| 1440/1570 [1:52:20<06:21, 2.93s/it]
92%|█████████▏| 1441/1570 [1:52:23<06:16, 2.92s/it]
92%|█████████▏| 1442/1570 [1:52:26<06:10, 2.90s/it]
92%|█████████▏| 1443/1570 [1:52:29<06:08, 2.90s/it]
92%|█████████▏| 1444/1570 [1:52:32<06:07, 2.91s/it]
92%|█████████▏| 1445/1570 [1:52:35<06:02, 2.90s/it]
92%|█████████▏| 1446/1570 [1:52:38<06:00, 2.91s/it]
92%|█████████▏| 1447/1570 [1:52:41<05:56, 2.90s/it]
92%|█████████▏| 1448/1570 [1:52:44<05:53, 2.90s/it]
92%|█████████▏| 1449/1570 [1:52:46<05:50, 2.90s/it]
92%|█████████▏| 1450/1570 [1:52:49<05:46, 2.89s/it]
92%|█████████▏| 1451/1570 [1:52:52<05:45, 2.90s/it]
92%|█████████▏| 1452/1570 [1:52:55<05:42, 2.90s/it]
93%|█████████▎| 1453/1570 [1:52:58<05:41, 2.92s/it]
93%|█████████▎| 1454/1570 [1:53:01<05:37, 2.91s/it]
93%|█████████▎| 1455/1570 [1:53:04<05:32, 2.90s/it]
93%|█████████▎| 1456/1570 [1:53:07<05:28, 2.89s/it]
93%|█████████▎| 1457/1570 [1:53:10<05:26, 2.89s/it]
93%|█████████▎| 1458/1570 [1:53:13<05:25, 2.91s/it]
93%|█████████▎| 1459/1570 [1:53:16<05:22, 2.91s/it]
93%|█████████▎| 1460/1570 [1:53:18<05:19, 2.91s/it]
{'loss': '0.0003777', 'grad_norm': '0.02151', 'learning_rate': '1.327e-06', 'epoch': '4.651'}
+
93%|█████████▎| 1460/1570 [1:53:18<05:19, 2.91s/it]
93%|█████████▎| 1461/1570 [1:53:21<05:15, 2.89s/it]
93%|█████████▎| 1462/1570 [1:53:24<05:12, 2.90s/it]
93%|█████████▎| 1463/1570 [1:53:27<05:11, 2.91s/it]
93%|█████████▎| 1464/1570 [1:53:30<05:08, 2.91s/it]
93%|█████████▎| 1465/1570 [1:53:33<05:04, 2.90s/it]
93%|█████████▎| 1466/1570 [1:53:36<05:02, 2.91s/it]
93%|█████████▎| 1467/1570 [1:53:39<04:59, 2.91s/it]
94%|█████████▎| 1468/1570 [1:53:42<04:56, 2.91s/it]
94%|█████████▎| 1469/1570 [1:53:45<04:52, 2.90s/it]
94%|█████████▎| 1470/1570 [1:53:47<04:48, 2.89s/it]
94%|█████████▎| 1471/1570 [1:53:50<04:45, 2.88s/it]
94%|█████████▍| 1472/1570 [1:53:53<04:42, 2.88s/it]
94%|█████████▍| 1473/1570 [1:53:56<04:38, 2.87s/it]
94%|█████████▍| 1474/1570 [1:53:59<04:36, 2.88s/it]
94%|█████████▍| 1475/1570 [1:54:02<04:35, 2.90s/it]
94%|█████████▍| 1476/1570 [1:54:05<04:31, 2.89s/it]
94%|█████████▍| 1477/1570 [1:54:08<04:27, 2.88s/it]
94%|█████████▍| 1478/1570 [1:54:10<04:24, 2.88s/it]
94%|█████████▍| 1479/1570 [1:54:13<04:21, 2.87s/it]
94%|█████████▍| 1480/1570 [1:54:16<04:18, 2.88s/it]
{'loss': '0.0005044', 'grad_norm': '0.008279', 'learning_rate': '9.258e-07', 'epoch': '4.715'}
+
94%|█████████▍| 1480/1570 [1:54:16<04:18, 2.88s/it]
94%|█████████▍| 1481/1570 [1:54:19<04:16, 2.88s/it]
94%|█████████▍| 1482/1570 [1:54:22<04:11, 2.86s/it]
94%|█████████▍| 1483/1570 [1:54:25<04:10, 2.88s/it]
95%|█████████▍| 1484/1570 [1:54:28<04:07, 2.88s/it]
95%|█████████▍| 1485/1570 [1:54:31<04:03, 2.87s/it]
95%|█████████▍| 1486/1570 [1:54:33<04:01, 2.87s/it]
95%|█████████▍| 1487/1570 [1:54:36<03:58, 2.88s/it]
95%|█████████▍| 1488/1570 [1:54:39<03:57, 2.89s/it]
95%|█████████▍| 1489/1570 [1:54:42<03:54, 2.90s/it]
95%|█████████▍| 1490/1570 [1:54:45<03:51, 2.89s/it]
95%|█████████▍| 1491/1570 [1:54:48<03:48, 2.89s/it]
95%|█████████▌| 1492/1570 [1:54:51<03:45, 2.90s/it]
95%|█████████▌| 1493/1570 [1:54:54<03:42, 2.90s/it]
95%|█████████▌| 1494/1570 [1:54:57<03:40, 2.90s/it]
95%|█████████▌| 1495/1570 [1:54:59<03:36, 2.89s/it]
95%|█████████▌| 1496/1570 [1:55:02<03:34, 2.90s/it]
95%|█████████▌| 1497/1570 [1:55:05<03:31, 2.90s/it]
95%|█████████▌| 1498/1570 [1:55:08<03:28, 2.90s/it]
95%|█████████▌| 1499/1570 [1:55:11<03:25, 2.89s/it]
96%|█████████▌| 1500/1570 [1:55:14<03:21, 2.88s/it]
{'loss': '0.000692', 'grad_norm': '0.01255', 'learning_rate': '6.033e-07', 'epoch': '4.779'}
+
96%|█████████▌| 1500/1570 [1:55:14<03:21, 2.88s/it]
96%|█████████▌| 1501/1570 [1:55:17<03:19, 2.89s/it]
96%|█████████▌| 1502/1570 [1:55:20<03:15, 2.88s/it]
96%|█████████▌| 1503/1570 [1:55:23<03:13, 2.88s/it]
96%|█████████▌| 1504/1570 [1:55:25<03:11, 2.90s/it]
96%|█████████▌| 1505/1570 [1:55:28<03:08, 2.89s/it]
96%|█████████▌| 1506/1570 [1:55:31<03:05, 2.91s/it]
96%|█████████▌| 1507/1570 [1:55:34<03:02, 2.90s/it]
96%|█████████▌| 1508/1570 [1:55:37<02:59, 2.90s/it]
96%|█████████▌| 1509/1570 [1:55:40<02:56, 2.90s/it]
96%|█████████▌| 1510/1570 [1:55:43<02:54, 2.91s/it]
96%|█████████▌| 1511/1570 [1:55:46<02:50, 2.90s/it]
96%|█████████▋| 1512/1570 [1:55:49<02:48, 2.90s/it]
96%|█████████▋| 1513/1570 [1:55:52<02:44, 2.89s/it]
96%|█████████▋| 1514/1570 [1:55:54<02:42, 2.89s/it]
96%|█████████▋| 1515/1570 [1:55:57<02:39, 2.90s/it]
97%|█████████▋| 1516/1570 [1:56:00<02:35, 2.88s/it]
97%|█████████▋| 1517/1570 [1:56:03<02:33, 2.90s/it]
97%|█████████▋| 1518/1570 [1:56:06<02:30, 2.90s/it]
97%|█████████▋| 1519/1570 [1:56:09<02:27, 2.90s/it]
97%|█████████▋| 1520/1570 [1:56:12<02:24, 2.88s/it]
{'loss': '0.004239', 'grad_norm': '0.003553', 'learning_rate': '3.599e-07', 'epoch': '4.842'}
+
97%|█████████▋| 1520/1570 [1:56:12<02:24, 2.88s/it]
97%|█████████▋| 1521/1570 [1:56:15<02:21, 2.90s/it]
97%|█████████▋| 1522/1570 [1:56:18<02:19, 2.90s/it]
97%|█████████▋| 1523/1570 [1:56:21<02:16, 2.90s/it]
97%|█████████▋| 1524/1570 [1:56:23<02:13, 2.90s/it]
97%|█████████▋| 1525/1570 [1:56:26<02:10, 2.89s/it]
97%|█████████▋| 1526/1570 [1:56:29<02:07, 2.90s/it]
97%|█████████▋| 1527/1570 [1:56:32<02:04, 2.90s/it]
97%|█████████▋| 1528/1570 [1:56:35<02:01, 2.90s/it]
97%|█████████▋| 1529/1570 [1:56:38<01:58, 2.89s/it]
97%|█████████▋| 1530/1570 [1:56:41<01:55, 2.90s/it]
98%|█████████▊| 1531/1570 [1:56:44<01:52, 2.89s/it]
98%|█████████▊| 1532/1570 [1:56:47<01:49, 2.89s/it]
98%|█████████▊| 1533/1570 [1:56:49<01:47, 2.89s/it]
98%|█████████▊| 1534/1570 [1:56:52<01:43, 2.88s/it]
98%|█████████▊| 1535/1570 [1:56:55<01:41, 2.90s/it]
98%|█████████▊| 1536/1570 [1:56:58<01:38, 2.91s/it]
98%|█████████▊| 1537/1570 [1:57:01<01:36, 2.91s/it]
98%|█████████▊| 1538/1570 [1:57:04<01:32, 2.90s/it]
98%|█████████▊| 1539/1570 [1:57:07<01:29, 2.89s/it]
98%|█████████▊| 1540/1570 [1:57:10<01:26, 2.89s/it]
{'loss': '0.0004378', 'grad_norm': '0.07334', 'learning_rate': '1.961e-07', 'epoch': '4.906'}
+
98%|█████████▊| 1540/1570 [1:57:10<01:26, 2.89s/it]
98%|█████████▊| 1541/1570 [1:57:13<01:23, 2.90s/it]
98%|█████████▊| 1542/1570 [1:57:16<01:20, 2.89s/it]
98%|█████████▊| 1543/1570 [1:57:18<01:18, 2.90s/it]
98%|█████████▊| 1544/1570 [1:57:21<01:15, 2.89s/it]
98%|█████████▊| 1545/1570 [1:57:24<01:12, 2.90s/it]
98%|█████████▊| 1546/1570 [1:57:27<01:09, 2.88s/it]
99%|█████████▊| 1547/1570 [1:57:30<01:06, 2.89s/it]
99%|█████████▊| 1548/1570 [1:57:33<01:03, 2.89s/it]
99%|█████████▊| 1549/1570 [1:57:36<01:00, 2.90s/it]
99%|█████████▊| 1550/1570 [1:57:39<00:58, 2.91s/it]
99%|█████████▉| 1551/1570 [1:57:42<00:55, 2.91s/it]
99%|█████████▉| 1552/1570 [1:57:45<00:52, 2.92s/it]
99%|█████████▉| 1553/1570 [1:57:48<00:49, 2.91s/it]
99%|█████████▉| 1554/1570 [1:57:50<00:46, 2.91s/it]
99%|█████████▉| 1555/1570 [1:57:53<00:43, 2.90s/it]
99%|█████████▉| 1556/1570 [1:57:56<00:40, 2.90s/it]
99%|█████████▉| 1557/1570 [1:57:59<00:37, 2.90s/it]
99%|█████████▉| 1558/1570 [1:58:02<00:34, 2.89s/it]
99%|█████████▉| 1559/1570 [1:58:05<00:31, 2.89s/it]
99%|█████████▉| 1560/1570 [1:58:08<00:28, 2.89s/it]
{'loss': '0.0004438', 'grad_norm': '0.02573', 'learning_rate': '1.121e-07', 'epoch': '4.97'}
+
99%|█████████▉| 1560/1570 [1:58:08<00:28, 2.89s/it]
99%|█████████▉| 1561/1570 [1:58:11<00:26, 2.90s/it]
99%|█████████▉| 1562/1570 [1:58:14<00:23, 2.91s/it]
100%|█████████▉| 1563/1570 [1:58:16<00:20, 2.91s/it]
100%|█████████▉| 1564/1570 [1:58:19<00:17, 2.90s/it]
100%|█████████▉| 1565/1570 [1:58:22<00:14, 2.91s/it]
100%|█████████▉| 1566/1570 [1:58:25<00:11, 2.91s/it]
100%|█████████▉| 1567/1570 [1:58:28<00:08, 2.91s/it]
100%|█████████▉| 1568/1570 [1:58:31<00:05, 2.91s/it]
100%|█████████▉| 1569/1570 [1:58:34<00:02, 2.90s/it]
100%|██████████| 1570/1570 [1:58:35<00:00, 2.38s/it][INFO|trainer.py:2668] 2026-08-22 19:14:54,448 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 19:14:54,448 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 19:14:54,448 >> Batch size = 16
+
+
0%| | 0/107 [00:00> Saving model checkpoint to results/qwen3/srkl/checkpoint-1570
+[INFO|configuration_utils.py:780] 2026-08-22 19:24:58,255 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 19:24:58,257 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 19:24:58,293 >> chat template saved in results/qwen3/srkl/checkpoint-1570/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 19:24:58,293 >> tokenizer config file saved in results/qwen3/srkl/checkpoint-1570/tokenizer_config.json
+[INFO|trainer.py:1823] 2026-08-22 19:24:59,106 >>
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
{'train_runtime': '7720', 'train_samples_per_second': '6.494', 'train_steps_per_second': '0.203', 'train_loss': '0.01169', 'epoch': '5'}
+
100%|██████████| 1570/1570 [2:08:40<00:00, 2.38s/it]
100%|██████████| 1570/1570 [2:08:40<00:00, 4.92s/it]
+[INFO|trainer.py:3815] 2026-08-22 19:24:59,573 >> Saving model checkpoint to results/qwen3/srkl
+[INFO|configuration_utils.py:780] 2026-08-22 19:25:00,057 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B/snapshots/c1899de289a04d12100db370d81485cdf75e47ca/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 19:25:00,059 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 1024,
+ "initializer_range": 0.02,
+ "intermediate_size": 3072,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 40960,
+ "max_window_layers": 28,
+ "model_type": "qwen3",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 28,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 1000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 19:25:00,094 >> chat template saved in results/qwen3/srkl/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 19:25:00,094 >> tokenizer config file saved in results/qwen3/srkl/tokenizer_config.json
+[transformers] The following generation flags are not valid and may be ignored: ['temperature', 'top_p']. Set `TRANSFORMERS_VERBOSITY=info` for more details.
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+***** train metrics *****
+ epoch = 5.0
+ total_flos = 54682091GF
+ train_loss = 0.0117
+ train_runtime = 2:08:40.23
+ train_samples_per_second = 6.494
+ train_steps_per_second = 0.203
+[INFO|trainer.py:2668] 2026-08-22 19:25:00,219 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 19:25:00,220 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 19:25:00,220 >> Batch size = 16
+[WARNING|logging.py:340] 2026-08-22 19:25:00,331 >> The following generation flags are not valid and may be ignored: ['temperature', 'top_p'].
+[INFO|logging.py:355] 2026-08-22 19:25:00,332 >> - `temperature`: `do_sample` is not set to `True`. However, `temperature` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.
+- `top_p`: `do_sample` is not set to `True`. However, `top_p` is set to `0.7` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.
+If you're using a pretrained model, note that some of these attributes may be set through the model's `generation_config.json` file.
+[WARNING|utils.py:1642] 2026-08-22 19:25:00,334 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
0%| | 0/107 [00:00> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
2%|▏ | 2/107 [00:05<04:48, 2.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:25:11,520 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
3%|▎ | 3/107 [00:10<06:44, 3.89s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:25:17,002 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
4%|▎ | 4/107 [00:16<07:47, 4.54s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:25:22,644 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
5%|▍ | 5/107 [00:22<08:16, 4.87s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:25:28,124 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
6%|▌ | 6/107 [00:26<08:12, 4.88s/it][WARNING|utils.py:1642] 2026-08-22 19:25:33,014 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 7/107 [00:32<08:27, 5.08s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:25:38,518 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 8/107 [00:37<08:34, 5.20s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:25:43,983 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
8%|▊ | 9/107 [00:43<08:55, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:25:50,028 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
9%|▉ | 10/107 [00:49<08:55, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:25:55,689 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
10%|█ | 11/107 [00:54<08:24, 5.25s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:26:00,318 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
11%|█ | 12/107 [01:00<08:32, 5.40s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:26:06,045 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
12%|█▏ | 13/107 [01:05<08:33, 5.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:26:11,669 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
13%|█▎ | 14/107 [01:11<08:26, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:26:17,087 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
14%|█▍ | 15/107 [01:16<08:27, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:26:22,743 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
15%|█▍ | 16/107 [01:22<08:23, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:26:28,324 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
16%|█▌ | 17/107 [01:28<08:26, 5.63s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:26:34,174 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
17%|█▋ | 18/107 [01:33<08:22, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:26:39,871 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
18%|█▊ | 19/107 [01:39<08:10, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:26:45,283 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
19%|█▊ | 20/107 [01:44<08:03, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:26:50,813 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
20%|█▉ | 21/107 [01:49<07:40, 5.35s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:26:55,669 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██ | 22/107 [01:55<07:41, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:27:01,267 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██▏ | 23/107 [02:00<07:36, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:27:06,717 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
22%|██▏ | 24/107 [02:06<07:31, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:27:12,168 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
23%|██▎ | 25/107 [02:12<07:50, 5.74s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:27:18,617 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
24%|██▍ | 26/107 [02:18<07:44, 5.73s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:27:24,328 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
25%|██▌ | 27/107 [02:23<07:34, 5.69s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:27:29,907 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
26%|██▌ | 28/107 [02:29<07:27, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:27:35,502 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
27%|██▋ | 29/107 [02:35<07:28, 5.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:27:41,479 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
28%|██▊ | 30/107 [02:40<07:16, 5.67s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:27:46,966 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
29%|██▉ | 31/107 [02:45<06:45, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:27:51,499 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
30%|██▉ | 32/107 [02:50<06:40, 5.35s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:27:56,881 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
31%|███ | 33/107 [02:56<06:40, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:28:02,437 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
32%|███▏ | 34/107 [03:02<06:55, 5.70s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:28:08,808 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
33%|███▎ | 35/107 [03:08<06:48, 5.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:28:14,438 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
34%|███▎ | 36/107 [03:13<06:36, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:28:19,806 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
35%|███▍ | 37/107 [03:18<06:12, 5.32s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:28:24,510 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▌ | 38/107 [03:24<06:15, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:28:30,247 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▋ | 39/107 [03:29<06:02, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:28:35,305 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
37%|███▋ | 40/107 [03:34<05:58, 5.35s/it][WARNING|utils.py:1642] 2026-08-22 19:28:40,705 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
38%|███▊ | 41/107 [03:40<05:57, 5.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:28:46,297 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
39%|███▉ | 42/107 [03:46<06:07, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:28:52,505 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
40%|████ | 43/107 [03:52<06:08, 5.76s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:28:58,506 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
41%|████ | 44/107 [03:57<05:56, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:29:03,917 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
42%|████▏ | 45/107 [04:02<05:40, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:29:09,004 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
43%|████▎ | 46/107 [04:08<05:32, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:29:14,347 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
44%|████▍ | 47/107 [04:14<05:35, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:29:20,268 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
45%|████▍ | 48/107 [04:20<05:33, 5.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:29:26,066 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
46%|████▌ | 49/107 [04:25<05:24, 5.60s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:29:31,537 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
47%|████▋ | 50/107 [04:30<05:15, 5.54s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:29:36,949 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
48%|████▊ | 51/107 [04:36<05:13, 5.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:29:42,663 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
49%|████▊ | 52/107 [04:41<05:02, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:29:47,954 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|████▉ | 53/107 [04:47<05:05, 5.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:29:53,995 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|█████ | 54/107 [04:53<04:53, 5.54s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:29:59,241 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
51%|█████▏ | 55/107 [04:58<04:48, 5.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:30:04,803 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
52%|█████▏ | 56/107 [05:04<04:45, 5.60s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:30:10,529 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
53%|█████▎ | 57/107 [05:10<04:41, 5.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:30:16,261 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
54%|█████▍ | 58/107 [05:15<04:32, 5.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:30:21,629 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
55%|█████▌ | 59/107 [05:21<04:24, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:30:27,038 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
56%|█████▌ | 60/107 [05:26<04:17, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:30:32,409 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
57%|█████▋ | 61/107 [05:31<04:02, 5.27s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:30:37,217 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
58%|█████▊ | 62/107 [05:36<03:57, 5.29s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:30:42,546 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
59%|█████▉ | 63/107 [05:42<03:56, 5.37s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:30:48,092 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
60%|█████▉ | 64/107 [05:47<03:49, 5.33s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:30:53,342 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
61%|██████ | 65/107 [05:52<03:43, 5.32s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:30:58,633 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
62%|██████▏ | 66/107 [05:58<03:39, 5.35s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:31:04,070 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
63%|██████▎ | 67/107 [06:03<03:35, 5.39s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:31:09,528 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▎ | 68/107 [06:08<03:24, 5.24s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:31:14,437 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▍ | 69/107 [06:13<03:18, 5.23s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:31:19,643 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
65%|██████▌ | 70/107 [06:19<03:22, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:31:25,719 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
66%|██████▋ | 71/107 [06:25<03:20, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:31:31,490 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
67%|██████▋ | 72/107 [06:30<03:12, 5.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:31:36,806 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
68%|██████▊ | 73/107 [06:36<03:07, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:31:42,352 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
69%|██████▉ | 74/107 [06:41<03:00, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:31:47,757 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
70%|███████ | 75/107 [06:47<02:54, 5.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:31:53,153 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
71%|███████ | 76/107 [06:52<02:50, 5.49s/it][WARNING|utils.py:1642] 2026-08-22 19:31:58,727 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
72%|███████▏ | 77/107 [06:58<02:45, 5.51s/it][WARNING|utils.py:1642] 2026-08-22 19:32:04,289 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
73%|███████▎ | 78/107 [07:03<02:40, 5.54s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:32:09,897 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
74%|███████▍ | 79/107 [07:09<02:34, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:32:15,414 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
75%|███████▍ | 80/107 [07:15<02:32, 5.64s/it][WARNING|utils.py:1642] 2026-08-22 19:32:21,288 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
76%|███████▌ | 81/107 [07:20<02:24, 5.57s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:32:26,710 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
77%|███████▋ | 82/107 [07:26<02:17, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:32:32,031 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
78%|███████▊ | 83/107 [07:31<02:10, 5.44s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:32:37,342 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▊ | 84/107 [07:36<02:03, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:32:42,589 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▉ | 85/107 [07:40<01:51, 5.07s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:32:46,928 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
80%|████████ | 86/107 [07:46<01:49, 5.23s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:32:52,522 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
81%|████████▏ | 87/107 [07:51<01:40, 5.03s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:32:57,107 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
82%|████████▏ | 88/107 [07:56<01:38, 5.16s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:33:02,570 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
83%|████████▎ | 89/107 [08:02<01:39, 5.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:33:08,956 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
84%|████████▍ | 90/107 [08:08<01:33, 5.51s/it][WARNING|utils.py:1642] 2026-08-22 19:33:14,417 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
85%|████████▌ | 91/107 [08:13<01:26, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:33:19,582 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
86%|████████▌ | 92/107 [08:18<01:20, 5.34s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:33:24,762 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
87%|████████▋ | 93/107 [08:24<01:14, 5.34s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:33:30,090 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
88%|████████▊ | 94/107 [08:29<01:10, 5.41s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:33:35,670 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
89%|████████▉ | 95/107 [08:35<01:05, 5.42s/it][WARNING|utils.py:1642] 2026-08-22 19:33:41,119 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
90%|████████▉ | 96/107 [08:40<00:59, 5.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:33:46,406 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
91%|█████████ | 97/107 [08:45<00:54, 5.43s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:33:51,951 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
92%|█████████▏| 98/107 [08:51<00:49, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:33:57,682 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 99/107 [08:57<00:44, 5.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:34:03,213 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 100/107 [09:02<00:38, 5.47s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:34:08,543 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
94%|█████████▍| 101/107 [09:08<00:33, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:34:14,380 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
95%|█████████▌| 102/107 [09:13<00:27, 5.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:34:19,982 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
96%|█████████▋| 103/107 [09:19<00:22, 5.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:34:25,286 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
97%|█████████▋| 104/107 [09:24<00:16, 5.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:34:30,832 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
98%|█████████▊| 105/107 [09:30<00:11, 5.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:34:36,678 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
99%|█████████▉| 106/107 [09:36<00:05, 5.62s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 19:34:42,387 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
100%|██████████| 107/107 [09:42<00:00, 5.65s/it]
100%|██████████| 107/107 [09:43<00:00, 5.45s/it]
+***** eval metrics *****
+ epoch = 5.0
+ eval_generator_count = 1712.0
+ eval_generator_exact_match = 96.729
+ eval_generator_rouge1 = 99.894
+ eval_generator_rouge2 = 99.8306
+ eval_generator_rougeL = 99.7434
+ eval_loss = 0.0037
+ eval_runtime = 0:09:48.95
+ eval_samples_per_second = 5.797
+ eval_selector_accuracy = 98.715
+ eval_selector_count = 1712.0
+ eval_steps_per_second = 0.182
+[INFO|modelcard.py:263] 2026-08-22 19:34:49,170 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
diff --git a/qwen3/run_all_logs/20260822_031516/teacher_sft.log b/qwen3/run_all_logs/20260822_031516/teacher_sft.log
new file mode 100644
index 0000000000000000000000000000000000000000..f32a49e7a13e24ff11691e5328b0dbaee7e307eb
--- /dev/null
+++ b/qwen3/run_all_logs/20260822_031516/teacher_sft.log
@@ -0,0 +1,2629 @@
+W0822 03:15:19.341000 1336276 torch/distributed/run.py:803]
+W0822 03:15:19.341000 1336276 torch/distributed/run.py:803] *****************************************
+W0822 03:15:19.341000 1336276 torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+W0822 03:15:19.341000 1336276 torch/distributed/run.py:803] *****************************************
+[INFO|2026-08-22 03:15:35] llamafactory.hparams.parser:651 >> Process rank: 1, world size: 2, device: cuda:1, distributed training: True, compute dtype: torch.bfloat16
+[INFO|2026-08-22 03:15:35] llamafactory.hparams.parser:144 >> Set `ddp_find_unused_parameters` to False in DDP training since LoRA is enabled.
+[INFO|2026-08-22 03:15:35] llamafactory.hparams.parser:651 >> Process rank: 0, world size: 2, device: cuda:0, distributed training: True, compute dtype: torch.bfloat16
+Starting sft with LlamaFactory and DeepSpeed=True.
+/mnt/hungpv/projects/cypher-extract/.venv/lib/python3.13/site-packages/jieba/__init__.py:44: SyntaxWarning: invalid escape sequence '\.'
+ re_han_default = re.compile("([\u4E00-\u9FD5a-zA-Z0-9+#&\._%\-]+)", re.U)
+/mnt/hungpv/projects/cypher-extract/.venv/lib/python3.13/site-packages/jieba/__init__.py:46: SyntaxWarning: invalid escape sequence '\s'
+ re_skip_default = re.compile("(\r\n|\s)", re.U)
+/mnt/hungpv/projects/cypher-extract/.venv/lib/python3.13/site-packages/jieba/__init__.py:44: SyntaxWarning: invalid escape sequence '\.'
+ re_han_default = re.compile("([\u4E00-\u9FD5a-zA-Z0-9+#&\._%\-]+)", re.U)
+/mnt/hungpv/projects/cypher-extract/.venv/lib/python3.13/site-packages/jieba/__init__.py:46: SyntaxWarning: invalid escape sequence '\s'
+ re_skip_default = re.compile("(\r\n|\s)", re.U)
+/mnt/hungpv/projects/cypher-extract/.venv/lib/python3.13/site-packages/jieba/finalseg/__init__.py:78: SyntaxWarning: invalid escape sequence '\.'
+ re_skip = re.compile("([a-zA-Z0-9]+(?:\.\d+)?%?)")
+/mnt/hungpv/projects/cypher-extract/.venv/lib/python3.13/site-packages/jieba/finalseg/__init__.py:78: SyntaxWarning: invalid escape sequence '\.'
+ re_skip = re.compile("([a-zA-Z0-9]+(?:\.\d+)?%?)")
+/mnt/hungpv/projects/cypher-extract/.venv/lib/python3.13/site-packages/rouge_chinese/rouge.py:93: SyntaxWarning: invalid escape sequence '\?'
+ para = re.sub('([。！？\?])([^”’])', r"\1\n\2", para)
+/mnt/hungpv/projects/cypher-extract/.venv/lib/python3.13/site-packages/rouge_chinese/rouge.py:93: SyntaxWarning: invalid escape sequence '\?'
+ para = re.sub('([。！？\?])([^”’])', r"\1\n\2", para)
+/mnt/hungpv/projects/cypher-extract/.venv/lib/python3.13/site-packages/rouge_chinese/rouge.py:94: SyntaxWarning: invalid escape sequence '\.'
+ para = re.sub('(\.{6})([^”’])', r"\1\n\2", para)
+/mnt/hungpv/projects/cypher-extract/.venv/lib/python3.13/site-packages/rouge_chinese/rouge.py:94: SyntaxWarning: invalid escape sequence '\.'
+ para = re.sub('(\.{6})([^”’])', r"\1\n\2", para)
+/mnt/hungpv/projects/cypher-extract/.venv/lib/python3.13/site-packages/rouge_chinese/rouge.py:96: SyntaxWarning: invalid escape sequence '\?'
+ para = re.sub('([。！？\?][”’])([^，。！？\?])', r'\1\n\2', para)
+/mnt/hungpv/projects/cypher-extract/.venv/lib/python3.13/site-packages/rouge_chinese/rouge.py:96: SyntaxWarning: invalid escape sequence '\?'
+ para = re.sub('([。！？\?][”’])([^，。！？\?])', r'\1\n\2', para)
+[INFO|configuration_utils.py:780] 2026-08-22 03:15:38,988 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 03:15:38,992 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 03:15:42,128 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 03:15:42,132 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|configuration_utils.py:780] 2026-08-22 03:15:42,365 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 03:15:42,367 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 03:15:43] llamafactory.data.loader:144 >> Loading dataset cypher_prepared_train.jsonl...
+
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 10027 examples [00:00, 119580.34 examples/s]
+
Converting format of dataset: 0%| | 0/10027 [00:00> Loading dataset cypher_prepared_eval.jsonl...
+
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 3414 examples [00:00, 124575.70 examples/s]
+
Converting format of dataset: 0%| | 0/3414 [00:00system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of art movements linked to paintings in the genre art style that ended before the year 1600?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "ArtMovement",
+ "properties": {
+ "end_year": "INTEGER",
+ "start_year": "INTEGER"
+ }
+ },
+ {
+ "label": "Genre",
+ "properties": {}
+ },
+ {
+ "label": "Painting",
+ "properties": {
+ "country_of_origin": "STRING",
+ "creation_year": "INTEGER"
+ }
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "ArtMovement",
+ "type": "associatedWith"
+ },
+ {
+ "properties": {},
+ "source": "Painting",
+ "target": "Genre",
+ "type": "hasGenre"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 74398, 320, 77, 25, 9286, 38050, 26432, 40995, 81, 15, 25, 49087, 2354, 60, 8013, 76, 15, 25, 18098, 287, 7287, 58, 81, 16, 25, 4648, 37525, 6294, 7, 76, 16, 25, 37525, 314, 606, 25, 364, 33613, 1947, 65260, 4769, 59086, 308, 5288, 308, 5073, 14645, 366, 220, 16, 21, 15, 15, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "MATCH (n:ArtMovement)<-[r0:associatedWith]-(m0:Painting)-[r1:hasGenre]->(m1:Genre {name: 'genre art'}) WITH DISTINCT n WHERE n.end_year < 1600 RETURN n.name"}<|im_end|>
+
+
Running tokenizer on dataset: 0%| | 0/3414 [00:00system
+You are a Cypher Query Generator.
+
+Your task is to generate a valid Cypher query that answers the user's natural language question using only the provided graph schema.
+
+Rules:
+- Use only labels, relationships, and properties that exist in the schema.
+- Do not invent any schema elements.
+- Return only the fields needed to answer the question.
+- If duplicates are possible, use DISTINCT when appropriate.
+- If the question requires aggregation, sorting, or limiting, use the correct Cypher clauses.
+- Ensure the query is syntactically valid.
+
+Output format:
+{
+ "cypher": "The complete Cypher query"
+}
+
+Return only the JSON object and nothing else.
+detailed thinking off<|im_end|>
+<|im_start|>user
+QUESTION:
+What are the names of weapons used in either the 2021 Baghdad suicide bombings or the Botroseya Church bombing?
+
+SCHEMA:
+{
+ "nodes": [
+ {
+ "label": "TerroristAttack",
+ "properties": {
+ "date": "DATE",
+ "locations": "LIST",
+ "number_of_deaths": "INTEGER",
+ "number_of_injuries": "INTEGER"
+ }
+ },
+ {
+ "label": "Weapon",
+ "properties": {}
+ }
+ ],
+ "relationships": [
+ {
+ "properties": {},
+ "source": "TerroristAttack",
+ "target": "Weapon",
+ "type": "employs"
+ }
+ ]
+}
+
+Generate a Cypher query that answers the question using only the provided schema.
+Return only the JSON object in the required format.<|im_end|>
+<|im_start|>assistant
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+label_ids:
+[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 4913, 11130, 28082, 788, 330, 25427, 314, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 15, 25, 30210, 1047, 60, 8013, 76, 15, 68661, 841, 380, 28602, 314, 606, 25, 364, 17, 15, 17, 16, 55218, 18144, 73379, 65260, 30880, 308, 11, 296, 15, 438, 296, 77553, 48171, 320, 77, 25, 28214, 26432, 40995, 81, 16, 25, 30210, 1047, 60, 8013, 76, 16, 68661, 841, 380, 28602, 314, 606, 25, 364, 23502, 24847, 7755, 9257, 36331, 65260, 30880, 308, 11, 296, 16, 438, 296, 335, 4769, 59086, 308, 30880, 308, 2644, 9207, 151645, 198]
+labels:
+{"cypher": "CALL { MATCH (n:Weapon)<-[r0:employs]-(m0:TerroristAttack {name: '2021 Baghdad suicide bombings'}) RETURN n, m0 as m UNION MATCH (n:Weapon)<-[r1:employs]-(m1:TerroristAttack {name: 'Botroseya Church bombing'}) RETURN n, m1 as m } WITH DISTINCT n RETURN n.name"}<|im_end|>
+
+[INFO|configuration_utils.py:780] 2026-08-22 03:16:01,028 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 03:16:01,031 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|2026-08-22 03:16:01] llamafactory.model.model_utils.kv_cache:144 >> KV cache is disabled during training.
+[INFO|modeling_utils.py:769] 2026-08-22 03:16:02,205 >> loading weights file model.safetensors from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model.safetensors.index.json
+[INFO|modeling_utils.py:839] 2026-08-22 03:16:02,206 >> Will use dtype=torch.bfloat16 as defined in model's config object
+[INFO|configuration_utils.py:1086] 2026-08-22 03:16:02,207 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "eos_token_id": 151645,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "use_cache": false
+}
+
+
Loading weights: 0%| | 0/398 [00:00> loading configuration file generation_config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/generation_config.json
+[INFO|configuration_utils.py:1086] 2026-08-22 03:16:04,254 >> Generate config GenerationConfig {
+ "bos_token_id": 151643,
+ "do_sample": true,
+ "eos_token_id": [
+ 151645,
+ 151643
+ ],
+ "pad_token_id": 151643,
+ "temperature": 0.7,
+ "top_k": 20,
+ "top_p": 0.8
+}
+
+[INFO|2026-08-22 03:16:04] llamafactory.model.model_utils.checkpointing:144 >> Gradient checkpointing enabled.
+[INFO|2026-08-22 03:16:04] llamafactory.model.model_utils.attention:144 >> Using torch SDPA for faster training and inference.
+[INFO|2026-08-22 03:16:04] llamafactory.model.adapter:144 >> Upcasting trainable params to float32.
+[INFO|2026-08-22 03:16:04] llamafactory.model.adapter:144 >> Fine-tuning method: LoRA
+[INFO|2026-08-22 03:16:04] llamafactory.model.model_utils.misc:144 >> Found linear modules: down_proj,k_proj,o_proj,q_proj,up_proj,v_proj,gate_proj
+[INFO|2026-08-22 03:16:05] llamafactory.model.loader:144 >> trainable params: 66,060,288 || all params: 4,088,528,384 || trainable%: 1.6157
+[WARNING|trainer_utils.py:1238] 2026-08-22 03:16:05,075 >> The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'bos_token_id': None, 'pad_token_id': 151643}.
+
Loading weights: 0%| | 0/398 [00:00> ***** Running training *****
+[INFO|trainer.py:1470] 2026-08-22 03:16:09,160 >> Num examples = 10,027
+[INFO|trainer.py:1471] 2026-08-22 03:16:09,161 >> Num Epochs = 5
+[INFO|trainer.py:1472] 2026-08-22 03:16:09,161 >> Num update steps per epoch = 314
+[INFO|trainer.py:1473] 2026-08-22 03:16:09,161 >> Instantaneous batch size per device = 1
+[INFO|trainer.py:1476] 2026-08-22 03:16:09,161 >> Total train batch size (w. parallel, distributed & accumulation) = 32
+[INFO|trainer.py:1477] 2026-08-22 03:16:09,161 >> Gradient Accumulation steps = 16
+[INFO|trainer.py:1478] 2026-08-22 03:16:09,161 >> Total optimization steps = 1,570
+[INFO|trainer.py:1479] 2026-08-22 03:16:09,165 >> Number of trainable parameters = 66,060,288
+
0%| | 0/1570 [00:00>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 03:45:30,286 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 03:45:30,286 >> Batch size = 8
+
+
0%| | 0/214 [00:00> Saving model checkpoint to results/qwen3/teacher_lora/checkpoint-314
+[INFO|configuration_utils.py:780] 2026-08-22 04:10:57,985 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 04:10:57,987 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 04:10:58,083 >> chat template saved in results/qwen3/teacher_lora/checkpoint-314/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 04:10:58,083 >> tokenizer config file saved in results/qwen3/teacher_lora/checkpoint-314/tokenizer_config.json
+
20%|██ | 315/1570 [54:56<161:42:20, 463.86s/it]
20%|██ | 316/1570 [55:03<113:50:03, 326.80s/it]
20%|██ | 317/1570 [55:10<80:19:37, 230.79s/it]
20%|██ | 318/1570 [55:17<56:52:00, 163.51s/it]
20%|██ | 319/1570 [55:23<40:27:57, 116.45s/it]
20%|██ | 320/1570 [55:30<28:58:00, 83.42s/it]
{'loss': '0.01131', 'grad_norm': '0.2329', 'learning_rate': '9.016e-05', 'epoch': '1.019'}
+
20%|██ | 320/1570 [55:30<28:58:00, 83.42s/it]
20%|██ | 321/1570 [55:37<20:59:28, 60.50s/it]
21%|██ | 322/1570 [55:44<15:23:04, 44.38s/it]
21%|██ | 323/1570 [55:50<11:27:45, 33.09s/it]
21%|██ | 324/1570 [55:56<8:36:03, 24.85s/it]
21%|██ | 325/1570 [56:02<6:37:11, 19.14s/it]
21%|██ | 326/1570 [56:08<5:16:32, 15.27s/it]
21%|██ | 327/1570 [56:14<4:18:01, 12.45s/it]
21%|██ | 328/1570 [56:20<3:36:13, 10.45s/it]
21%|██ | 329/1570 [56:25<3:06:23, 9.01s/it]
21%|██ | 330/1570 [56:31<2:45:21, 8.00s/it]
21%|██ | 331/1570 [56:37<2:30:43, 7.30s/it]
21%|██ | 332/1570 [56:42<2:20:17, 6.80s/it]
21%|██ | 333/1570 [56:48<2:12:53, 6.45s/it]
21%|██▏ | 334/1570 [56:54<2:08:12, 6.22s/it]
21%|██▏ | 335/1570 [56:59<2:04:42, 6.06s/it]
21%|██▏ | 336/1570 [57:05<2:02:12, 5.94s/it]
21%|██▏ | 337/1570 [57:11<2:00:25, 5.86s/it]
22%|██▏ | 338/1570 [57:16<1:59:11, 5.80s/it]
22%|██▏ | 339/1570 [57:22<1:58:15, 5.76s/it]
22%|██▏ | 340/1570 [57:28<1:57:28, 5.73s/it]
{'loss': '0.00655', 'grad_norm': '0.07233', 'learning_rate': '8.894e-05', 'epoch': '1.083'}
+
22%|██▏ | 340/1570 [57:28<1:57:28, 5.73s/it]
22%|██▏ | 341/1570 [57:33<1:56:52, 5.71s/it]
22%|██▏ | 342/1570 [57:39<1:56:29, 5.69s/it]
22%|██▏ | 343/1570 [57:44<1:56:10, 5.68s/it]
22%|██▏ | 344/1570 [57:50<1:56:04, 5.68s/it]
22%|██▏ | 345/1570 [57:56<1:55:50, 5.67s/it]
22%|██▏ | 346/1570 [58:01<1:55:39, 5.67s/it]
22%|██▏ | 347/1570 [58:07<1:55:01, 5.64s/it]
22%|██▏ | 348/1570 [58:13<1:54:57, 5.64s/it]
22%|██▏ | 349/1570 [58:18<1:54:57, 5.65s/it]
22%|██▏ | 350/1570 [58:24<1:55:06, 5.66s/it]
22%|██▏ | 351/1570 [58:30<1:54:41, 5.65s/it]
22%|██▏ | 352/1570 [58:35<1:55:02, 5.67s/it]
22%|██▏ | 353/1570 [58:41<1:54:56, 5.67s/it]
23%|██▎ | 354/1570 [58:47<1:55:14, 5.69s/it]
23%|██▎ | 355/1570 [58:52<1:55:01, 5.68s/it]
23%|██▎ | 356/1570 [58:58<1:55:34, 5.71s/it]
23%|██▎ | 357/1570 [59:04<1:56:57, 5.79s/it]
23%|██▎ | 358/1570 [59:11<2:01:24, 6.01s/it]
23%|██▎ | 359/1570 [59:17<2:00:22, 5.96s/it]
23%|██▎ | 360/1570 [59:22<1:58:09, 5.86s/it]
{'loss': '0.003336', 'grad_norm': '0.05371', 'learning_rate': '8.766e-05', 'epoch': '1.147'}
+
23%|██▎ | 360/1570 [59:22<1:58:09, 5.86s/it]
23%|██▎ | 361/1570 [59:28<1:56:33, 5.78s/it]
23%|██▎ | 362/1570 [59:33<1:55:12, 5.72s/it]
23%|██▎ | 363/1570 [59:39<1:55:01, 5.72s/it]
23%|██▎ | 364/1570 [59:45<1:54:20, 5.69s/it]
23%|██▎ | 365/1570 [59:50<1:53:50, 5.67s/it]
23%|██▎ | 366/1570 [59:56<1:53:19, 5.65s/it]
23%|██▎ | 367/1570 [1:00:02<1:52:53, 5.63s/it]
23%|██▎ | 368/1570 [1:00:07<1:52:35, 5.62s/it]
24%|██▎ | 369/1570 [1:00:13<1:53:00, 5.65s/it]
24%|██▎ | 370/1570 [1:00:18<1:52:44, 5.64s/it]
24%|██▎ | 371/1570 [1:00:24<1:52:36, 5.64s/it]
24%|██▎ | 372/1570 [1:00:30<1:52:24, 5.63s/it]
24%|██▍ | 373/1570 [1:00:35<1:52:16, 5.63s/it]
24%|██▍ | 374/1570 [1:00:41<1:52:20, 5.64s/it]
24%|██▍ | 375/1570 [1:00:47<1:52:05, 5.63s/it]
24%|██▍ | 376/1570 [1:00:52<1:51:55, 5.62s/it]
24%|██▍ | 377/1570 [1:00:58<1:51:45, 5.62s/it]
24%|██▍ | 378/1570 [1:01:03<1:51:53, 5.63s/it]
24%|██▍ | 379/1570 [1:01:09<1:51:42, 5.63s/it]
24%|██▍ | 380/1570 [1:01:15<1:51:38, 5.63s/it]
{'loss': '0.008411', 'grad_norm': '0.1449', 'learning_rate': '8.631e-05', 'epoch': '1.211'}
+
24%|██▍ | 380/1570 [1:01:15<1:51:38, 5.63s/it]
24%|██▍ | 381/1570 [1:01:21<1:52:43, 5.69s/it]
24%|██▍ | 382/1570 [1:01:26<1:53:34, 5.74s/it]
24%|██▍ | 383/1570 [1:01:32<1:54:41, 5.80s/it]
24%|██▍ | 384/1570 [1:01:38<1:56:47, 5.91s/it]
25%|██▍ | 385/1570 [1:01:44<1:55:59, 5.87s/it]
25%|██▍ | 386/1570 [1:01:50<1:54:20, 5.79s/it]
25%|██▍ | 387/1570 [1:01:56<1:53:14, 5.74s/it]
25%|██▍ | 388/1570 [1:02:01<1:52:41, 5.72s/it]
25%|██▍ | 389/1570 [1:02:07<1:53:00, 5.74s/it]
25%|██▍ | 390/1570 [1:02:13<1:52:08, 5.70s/it]
25%|██▍ | 391/1570 [1:02:18<1:52:08, 5.71s/it]
25%|██▍ | 392/1570 [1:02:24<1:52:02, 5.71s/it]
25%|██▌ | 393/1570 [1:02:30<1:51:02, 5.66s/it]
25%|██▌ | 394/1570 [1:02:35<1:50:40, 5.65s/it]
25%|██▌ | 395/1570 [1:02:41<1:51:46, 5.71s/it]
25%|██▌ | 396/1570 [1:02:47<1:50:46, 5.66s/it]
25%|██▌ | 397/1570 [1:02:52<1:50:01, 5.63s/it]
25%|██▌ | 398/1570 [1:02:58<1:50:48, 5.67s/it]
25%|██▌ | 399/1570 [1:03:04<1:50:30, 5.66s/it]
25%|██▌ | 400/1570 [1:03:09<1:49:49, 5.63s/it]
{'loss': '0.006243', 'grad_norm': '0.06368', 'learning_rate': '8.491e-05', 'epoch': '1.274'}
+
25%|██▌ | 400/1570 [1:03:09<1:49:49, 5.63s/it]
26%|██▌ | 401/1570 [1:03:15<1:49:23, 5.61s/it]
26%|██▌ | 402/1570 [1:03:20<1:49:19, 5.62s/it]
26%|██▌ | 403/1570 [1:03:26<1:50:34, 5.69s/it]
26%|██▌ | 404/1570 [1:03:32<1:51:10, 5.72s/it]
26%|██▌ | 405/1570 [1:03:38<1:50:19, 5.68s/it]
26%|██▌ | 406/1570 [1:03:43<1:49:54, 5.67s/it]
26%|██▌ | 407/1570 [1:03:49<1:49:20, 5.64s/it]
26%|██▌ | 408/1570 [1:03:54<1:49:38, 5.66s/it]
26%|██▌ | 409/1570 [1:04:00<1:51:44, 5.77s/it]
26%|██▌ | 410/1570 [1:04:07<1:54:21, 5.91s/it]
26%|██▌ | 411/1570 [1:04:13<1:56:31, 6.03s/it]
26%|██▌ | 412/1570 [1:04:19<1:54:37, 5.94s/it]
26%|██▋ | 413/1570 [1:04:24<1:52:51, 5.85s/it]
26%|██▋ | 414/1570 [1:04:30<1:52:13, 5.82s/it]
26%|██▋ | 415/1570 [1:04:36<1:50:51, 5.76s/it]
26%|██▋ | 416/1570 [1:04:42<1:53:03, 5.88s/it]
27%|██▋ | 417/1570 [1:04:48<1:52:15, 5.84s/it]
27%|██▋ | 418/1570 [1:04:53<1:50:56, 5.78s/it]
27%|██▋ | 419/1570 [1:04:59<1:50:20, 5.75s/it]
27%|██▋ | 420/1570 [1:05:05<1:50:15, 5.75s/it]
{'loss': '0.007444', 'grad_norm': '0.0505', 'learning_rate': '8.345e-05', 'epoch': '1.338'}
+
27%|██▋ | 420/1570 [1:05:05<1:50:15, 5.75s/it]
27%|██▋ | 421/1570 [1:05:10<1:49:17, 5.71s/it]
27%|██▋ | 422/1570 [1:05:16<1:49:43, 5.73s/it]
27%|██▋ | 423/1570 [1:05:22<1:48:59, 5.70s/it]
27%|██▋ | 424/1570 [1:05:28<1:49:13, 5.72s/it]
27%|██▋ | 425/1570 [1:05:33<1:48:28, 5.68s/it]
27%|██▋ | 426/1570 [1:05:39<1:47:56, 5.66s/it]
27%|██▋ | 427/1570 [1:05:45<1:48:30, 5.70s/it]
27%|██▋ | 428/1570 [1:05:50<1:47:58, 5.67s/it]
27%|██▋ | 429/1570 [1:05:56<1:47:36, 5.66s/it]
27%|██▋ | 430/1570 [1:06:01<1:47:37, 5.66s/it]
27%|██▋ | 431/1570 [1:06:07<1:47:52, 5.68s/it]
28%|██▊ | 432/1570 [1:06:13<1:48:48, 5.74s/it]
28%|██▊ | 433/1570 [1:06:19<1:48:13, 5.71s/it]
28%|██▊ | 434/1570 [1:06:25<1:49:25, 5.78s/it]
28%|██▊ | 435/1570 [1:06:30<1:48:44, 5.75s/it]
28%|██▊ | 436/1570 [1:06:36<1:47:57, 5.71s/it]
28%|██▊ | 437/1570 [1:06:42<1:47:26, 5.69s/it]
28%|██▊ | 438/1570 [1:06:47<1:48:23, 5.75s/it]
28%|██▊ | 439/1570 [1:06:53<1:47:47, 5.72s/it]
28%|██▊ | 440/1570 [1:06:59<1:47:10, 5.69s/it]
{'loss': '0.003112', 'grad_norm': '0.03241', 'learning_rate': '8.194e-05', 'epoch': '1.402'}
+
28%|██▊ | 440/1570 [1:06:59<1:47:10, 5.69s/it]
28%|██▊ | 441/1570 [1:07:04<1:47:12, 5.70s/it]
28%|██▊ | 442/1570 [1:07:10<1:46:29, 5.66s/it]
28%|██▊ | 443/1570 [1:07:16<1:47:07, 5.70s/it]
28%|██▊ | 444/1570 [1:07:22<1:47:37, 5.73s/it]
28%|██▊ | 445/1570 [1:07:27<1:46:42, 5.69s/it]
28%|██▊ | 446/1570 [1:07:33<1:46:11, 5.67s/it]
28%|██▊ | 447/1570 [1:07:39<1:47:11, 5.73s/it]
29%|██▊ | 448/1570 [1:07:44<1:46:38, 5.70s/it]
29%|██▊ | 449/1570 [1:07:50<1:46:05, 5.68s/it]
29%|██▊ | 450/1570 [1:07:56<1:45:49, 5.67s/it]
29%|██▊ | 451/1570 [1:08:01<1:45:43, 5.67s/it]
29%|██▉ | 452/1570 [1:08:07<1:45:38, 5.67s/it]
29%|██▉ | 453/1570 [1:08:13<1:45:44, 5.68s/it]
29%|██▉ | 454/1570 [1:08:18<1:44:58, 5.64s/it]
29%|██▉ | 455/1570 [1:08:24<1:44:43, 5.64s/it]
29%|██▉ | 456/1570 [1:08:30<1:45:15, 5.67s/it]
29%|██▉ | 457/1570 [1:08:35<1:44:46, 5.65s/it]
29%|██▉ | 458/1570 [1:08:41<1:46:46, 5.76s/it]
29%|██▉ | 459/1570 [1:08:47<1:46:37, 5.76s/it]
29%|██▉ | 460/1570 [1:08:53<1:45:31, 5.70s/it]
{'loss': '0.00526', 'grad_norm': '0.1364', 'learning_rate': '8.037e-05', 'epoch': '1.466'}
+
29%|██▉ | 460/1570 [1:08:53<1:45:31, 5.70s/it]
29%|██▉ | 461/1570 [1:08:58<1:46:10, 5.74s/it]
29%|██▉ | 462/1570 [1:09:04<1:44:57, 5.68s/it]
29%|██▉ | 463/1570 [1:09:09<1:43:59, 5.64s/it]
30%|██▉ | 464/1570 [1:09:15<1:44:00, 5.64s/it]
30%|██▉ | 465/1570 [1:09:21<1:43:50, 5.64s/it]
30%|██▉ | 466/1570 [1:09:26<1:43:28, 5.62s/it]
30%|██▉ | 467/1570 [1:09:32<1:43:45, 5.64s/it]
30%|██▉ | 468/1570 [1:09:38<1:44:37, 5.70s/it]
30%|██▉ | 469/1570 [1:09:43<1:43:53, 5.66s/it]
30%|██▉ | 470/1570 [1:09:49<1:45:12, 5.74s/it]
30%|███ | 471/1570 [1:09:55<1:45:05, 5.74s/it]
30%|███ | 472/1570 [1:10:02<1:49:20, 5.98s/it]
30%|███ | 473/1570 [1:10:08<1:52:02, 6.13s/it]
30%|███ | 474/1570 [1:10:14<1:49:13, 5.98s/it]
30%|███ | 475/1570 [1:10:19<1:47:31, 5.89s/it]
30%|███ | 476/1570 [1:10:25<1:46:17, 5.83s/it]
30%|███ | 477/1570 [1:10:31<1:45:17, 5.78s/it]
30%|███ | 478/1570 [1:10:37<1:44:59, 5.77s/it]
31%|███ | 479/1570 [1:10:42<1:45:20, 5.79s/it]
31%|███ | 480/1570 [1:10:48<1:43:56, 5.72s/it]
{'loss': '0.004165', 'grad_norm': '0.022', 'learning_rate': '7.876e-05', 'epoch': '1.53'}
+
31%|███ | 480/1570 [1:10:48<1:43:56, 5.72s/it]
31%|███ | 481/1570 [1:10:54<1:44:33, 5.76s/it]
31%|███ | 482/1570 [1:10:59<1:43:14, 5.69s/it]
31%|███ | 483/1570 [1:11:05<1:43:12, 5.70s/it]
31%|███ | 484/1570 [1:11:11<1:42:30, 5.66s/it]
31%|███ | 485/1570 [1:11:16<1:42:00, 5.64s/it]
31%|███ | 486/1570 [1:11:22<1:41:45, 5.63s/it]
31%|███ | 487/1570 [1:11:27<1:41:18, 5.61s/it]
31%|███ | 488/1570 [1:11:33<1:41:53, 5.65s/it]
31%|███ | 489/1570 [1:11:39<1:42:07, 5.67s/it]
31%|███ | 490/1570 [1:11:44<1:41:32, 5.64s/it]
31%|███▏ | 491/1570 [1:11:50<1:40:56, 5.61s/it]
31%|███▏ | 492/1570 [1:11:56<1:41:24, 5.64s/it]
31%|███▏ | 493/1570 [1:12:01<1:40:58, 5.63s/it]
31%|███▏ | 494/1570 [1:12:07<1:41:31, 5.66s/it]
32%|███▏ | 495/1570 [1:12:13<1:42:08, 5.70s/it]
32%|███▏ | 496/1570 [1:12:19<1:42:15, 5.71s/it]
32%|███▏ | 497/1570 [1:12:25<1:44:16, 5.83s/it]
32%|███▏ | 498/1570 [1:12:30<1:42:36, 5.74s/it]
32%|███▏ | 499/1570 [1:12:36<1:42:19, 5.73s/it]
32%|███▏ | 500/1570 [1:12:42<1:41:57, 5.72s/it]
{'loss': '0.001552', 'grad_norm': '0.01941', 'learning_rate': '7.71e-05', 'epoch': '1.594'}
+
32%|███▏ | 500/1570 [1:12:42<1:41:57, 5.72s/it]
32%|███▏ | 501/1570 [1:12:48<1:44:05, 5.84s/it]
32%|███▏ | 502/1570 [1:12:53<1:42:37, 5.77s/it]
32%|███▏ | 503/1570 [1:12:59<1:41:36, 5.71s/it]
32%|███▏ | 504/1570 [1:13:04<1:40:41, 5.67s/it]
32%|███▏ | 505/1570 [1:13:10<1:39:58, 5.63s/it]
32%|███▏ | 506/1570 [1:13:16<1:39:33, 5.61s/it]
32%|███▏ | 507/1570 [1:13:21<1:39:11, 5.60s/it]
32%|███▏ | 508/1570 [1:13:27<1:38:55, 5.59s/it]
32%|███▏ | 509/1570 [1:13:32<1:38:38, 5.58s/it]
32%|███▏ | 510/1570 [1:13:38<1:38:29, 5.58s/it]
33%|███▎ | 511/1570 [1:13:43<1:38:26, 5.58s/it]
33%|███▎ | 512/1570 [1:13:49<1:38:13, 5.57s/it]
33%|███▎ | 513/1570 [1:13:54<1:37:52, 5.56s/it]
33%|███▎ | 514/1570 [1:14:00<1:37:30, 5.54s/it]
33%|███▎ | 515/1570 [1:14:05<1:37:15, 5.53s/it]
33%|███▎ | 516/1570 [1:14:11<1:37:00, 5.52s/it]
33%|███▎ | 517/1570 [1:14:16<1:36:52, 5.52s/it]
33%|███▎ | 518/1570 [1:14:22<1:36:50, 5.52s/it]
33%|███▎ | 519/1570 [1:14:28<1:36:38, 5.52s/it]
33%|███▎ | 520/1570 [1:14:33<1:36:37, 5.52s/it]
{'loss': '0.004233', 'grad_norm': '0.1271', 'learning_rate': '7.54e-05', 'epoch': '1.657'}
+
33%|███▎ | 520/1570 [1:14:33<1:36:37, 5.52s/it]
33%|███▎ | 521/1570 [1:14:39<1:36:37, 5.53s/it]
33%|███▎ | 522/1570 [1:14:44<1:36:26, 5.52s/it]
33%|███▎ | 523/1570 [1:14:50<1:36:26, 5.53s/it]
33%|███▎ | 524/1570 [1:14:55<1:36:23, 5.53s/it]
33%|███▎ | 525/1570 [1:15:01<1:36:22, 5.53s/it]
34%|███▎ | 526/1570 [1:15:06<1:36:11, 5.53s/it]
34%|███▎ | 527/1570 [1:15:12<1:36:15, 5.54s/it]
34%|███▎ | 528/1570 [1:15:17<1:36:00, 5.53s/it]
34%|███▎ | 529/1570 [1:15:23<1:35:58, 5.53s/it]
34%|███▍ | 530/1570 [1:15:28<1:35:55, 5.53s/it]
34%|███▍ | 531/1570 [1:15:34<1:35:47, 5.53s/it]
34%|███▍ | 532/1570 [1:15:39<1:35:39, 5.53s/it]
34%|███▍ | 533/1570 [1:15:45<1:35:36, 5.53s/it]
34%|███▍ | 534/1570 [1:15:51<1:35:37, 5.54s/it]
34%|███▍ | 535/1570 [1:15:56<1:35:34, 5.54s/it]
34%|███▍ | 536/1570 [1:16:02<1:35:29, 5.54s/it]
34%|███▍ | 537/1570 [1:16:07<1:35:24, 5.54s/it]
34%|███▍ | 538/1570 [1:16:13<1:35:27, 5.55s/it]
34%|███▍ | 539/1570 [1:16:18<1:35:19, 5.55s/it]
34%|███▍ | 540/1570 [1:16:24<1:35:17, 5.55s/it]
{'loss': '0.00293', 'grad_norm': '0.226', 'learning_rate': '7.366e-05', 'epoch': '1.721'}
+
34%|███▍ | 540/1570 [1:16:24<1:35:17, 5.55s/it]
34%|███▍ | 541/1570 [1:16:29<1:35:08, 5.55s/it]
35%|███▍ | 542/1570 [1:16:35<1:35:05, 5.55s/it]
35%|███▍ | 543/1570 [1:16:40<1:35:00, 5.55s/it]
35%|███▍ | 544/1570 [1:16:46<1:34:48, 5.54s/it]
35%|███▍ | 545/1570 [1:16:52<1:34:41, 5.54s/it]
35%|███▍ | 546/1570 [1:16:57<1:34:38, 5.55s/it]
35%|███▍ | 547/1570 [1:17:03<1:34:39, 5.55s/it]
35%|███▍ | 548/1570 [1:17:08<1:34:40, 5.56s/it]
35%|███▍ | 549/1570 [1:17:14<1:34:41, 5.56s/it]
35%|███▌ | 550/1570 [1:17:19<1:34:43, 5.57s/it]
35%|███▌ | 551/1570 [1:17:25<1:34:48, 5.58s/it]
35%|███▌ | 552/1570 [1:17:31<1:34:48, 5.59s/it]
35%|███▌ | 553/1570 [1:17:36<1:34:51, 5.60s/it]
35%|███▌ | 554/1570 [1:17:42<1:34:52, 5.60s/it]
35%|███▌ | 555/1570 [1:17:47<1:34:45, 5.60s/it]
35%|███▌ | 556/1570 [1:17:53<1:34:31, 5.59s/it]
35%|███▌ | 557/1570 [1:17:59<1:34:28, 5.60s/it]
36%|███▌ | 558/1570 [1:18:04<1:34:24, 5.60s/it]
36%|███▌ | 559/1570 [1:18:10<1:34:11, 5.59s/it]
36%|███▌ | 560/1570 [1:18:15<1:34:14, 5.60s/it]
{'loss': '0.004469', 'grad_norm': '0.0756', 'learning_rate': '7.188e-05', 'epoch': '1.785'}
+
36%|███▌ | 560/1570 [1:18:15<1:34:14, 5.60s/it]
36%|███▌ | 561/1570 [1:18:21<1:34:03, 5.59s/it]
36%|███▌ | 562/1570 [1:18:27<1:33:50, 5.59s/it]
36%|███▌ | 563/1570 [1:18:32<1:33:54, 5.60s/it]
36%|███▌ | 564/1570 [1:18:38<1:33:42, 5.59s/it]
36%|███▌ | 565/1570 [1:18:43<1:33:30, 5.58s/it]
36%|███▌ | 566/1570 [1:18:49<1:33:27, 5.58s/it]
36%|███▌ | 567/1570 [1:18:54<1:33:06, 5.57s/it]
36%|███▌ | 568/1570 [1:19:00<1:32:53, 5.56s/it]
36%|███▌ | 569/1570 [1:19:06<1:32:47, 5.56s/it]
36%|███▋ | 570/1570 [1:19:11<1:32:25, 5.55s/it]
36%|███▋ | 571/1570 [1:19:17<1:32:27, 5.55s/it]
36%|███▋ | 572/1570 [1:19:22<1:32:22, 5.55s/it]
36%|███▋ | 573/1570 [1:19:28<1:32:25, 5.56s/it]
37%|███▋ | 574/1570 [1:19:33<1:32:12, 5.55s/it]
37%|███▋ | 575/1570 [1:19:39<1:32:11, 5.56s/it]
37%|███▋ | 576/1570 [1:19:44<1:31:58, 5.55s/it]
37%|███▋ | 577/1570 [1:19:50<1:32:05, 5.56s/it]
37%|███▋ | 578/1570 [1:19:56<1:32:01, 5.57s/it]
37%|███▋ | 579/1570 [1:20:01<1:32:01, 5.57s/it]
37%|███▋ | 580/1570 [1:20:07<1:31:55, 5.57s/it]
{'loss': '0.00552', 'grad_norm': '0.0852', 'learning_rate': '7.006e-05', 'epoch': '1.849'}
+
37%|███▋ | 580/1570 [1:20:07<1:31:55, 5.57s/it]
37%|███▋ | 581/1570 [1:20:12<1:31:53, 5.58s/it]
37%|███▋ | 582/1570 [1:20:18<1:31:43, 5.57s/it]
37%|███▋ | 583/1570 [1:20:23<1:31:56, 5.59s/it]
37%|███▋ | 584/1570 [1:20:29<1:31:47, 5.59s/it]
37%|███▋ | 585/1570 [1:20:35<1:31:38, 5.58s/it]
37%|███▋ | 586/1570 [1:20:40<1:31:26, 5.58s/it]
37%|███▋ | 587/1570 [1:20:46<1:31:14, 5.57s/it]
37%|███▋ | 588/1570 [1:20:51<1:31:11, 5.57s/it]
38%|███▊ | 589/1570 [1:20:57<1:31:00, 5.57s/it]
38%|███▊ | 590/1570 [1:21:02<1:31:03, 5.57s/it]
38%|███▊ | 591/1570 [1:21:08<1:31:01, 5.58s/it]
38%|███▊ | 592/1570 [1:21:14<1:30:53, 5.58s/it]
38%|███▊ | 593/1570 [1:21:19<1:30:43, 5.57s/it]
38%|███▊ | 594/1570 [1:21:25<1:30:44, 5.58s/it]
38%|███▊ | 595/1570 [1:21:30<1:30:38, 5.58s/it]
38%|███▊ | 596/1570 [1:21:36<1:30:33, 5.58s/it]
38%|███▊ | 597/1570 [1:21:42<1:30:26, 5.58s/it]
38%|███▊ | 598/1570 [1:21:47<1:30:19, 5.58s/it]
38%|███▊ | 599/1570 [1:21:53<1:30:14, 5.58s/it]
38%|███▊ | 600/1570 [1:21:58<1:30:04, 5.57s/it]
{'loss': '0.001579', 'grad_norm': '0.008491', 'learning_rate': '6.821e-05', 'epoch': '1.913'}
+
38%|███▊ | 600/1570 [1:21:58<1:30:04, 5.57s/it]
38%|███▊ | 601/1570 [1:22:04<1:30:01, 5.57s/it]
38%|███▊ | 602/1570 [1:22:09<1:29:57, 5.58s/it]
38%|███▊ | 603/1570 [1:22:15<1:29:53, 5.58s/it]
38%|███▊ | 604/1570 [1:22:21<1:29:42, 5.57s/it]
39%|███▊ | 605/1570 [1:22:26<1:29:28, 5.56s/it]
39%|███▊ | 606/1570 [1:22:32<1:29:18, 5.56s/it]
39%|███▊ | 607/1570 [1:22:37<1:29:18, 5.56s/it]
39%|███▊ | 608/1570 [1:22:43<1:29:21, 5.57s/it]
39%|███▉ | 609/1570 [1:22:48<1:29:25, 5.58s/it]
39%|███▉ | 610/1570 [1:22:54<1:29:15, 5.58s/it]
39%|███▉ | 611/1570 [1:23:00<1:29:09, 5.58s/it]
39%|███▉ | 612/1570 [1:23:05<1:29:06, 5.58s/it]
39%|███▉ | 613/1570 [1:23:11<1:29:10, 5.59s/it]
39%|███▉ | 614/1570 [1:23:16<1:28:49, 5.57s/it]
39%|███▉ | 615/1570 [1:23:22<1:28:52, 5.58s/it]
39%|███▉ | 616/1570 [1:23:27<1:28:52, 5.59s/it]
39%|███▉ | 617/1570 [1:23:33<1:28:48, 5.59s/it]
39%|███▉ | 618/1570 [1:23:39<1:28:34, 5.58s/it]
39%|███▉ | 619/1570 [1:23:44<1:28:25, 5.58s/it]
39%|███▉ | 620/1570 [1:23:50<1:28:21, 5.58s/it]
{'loss': '0.007295', 'grad_norm': '0.009172', 'learning_rate': '6.634e-05', 'epoch': '1.977'}
+
39%|███▉ | 620/1570 [1:23:50<1:28:21, 5.58s/it]
40%|███▉ | 621/1570 [1:23:55<1:27:40, 5.54s/it]
40%|███▉ | 622/1570 [1:24:01<1:27:31, 5.54s/it]
40%|███▉ | 623/1570 [1:24:06<1:27:10, 5.52s/it]
40%|███▉ | 624/1570 [1:24:12<1:27:13, 5.53s/it]
40%|███▉ | 625/1570 [1:24:17<1:27:01, 5.53s/it]
40%|███▉ | 626/1570 [1:24:23<1:27:01, 5.53s/it]
40%|███▉ | 627/1570 [1:24:28<1:27:01, 5.54s/it]
40%|████ | 628/1570 [1:24:30<1:09:40, 4.44s/it][INFO|trainer.py:2668] 2026-08-22 04:40:39,986 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 04:40:39,986 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 04:40:39,986 >> Batch size = 8
+
+
0%| | 0/214 [00:00> Saving model checkpoint to results/qwen3/teacher_lora/checkpoint-628
+[INFO|configuration_utils.py:780] 2026-08-22 05:05:27,612 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 05:05:27,616 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 05:05:27,712 >> chat template saved in results/qwen3/teacher_lora/checkpoint-628/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 05:05:27,712 >> tokenizer config file saved in results/qwen3/teacher_lora/checkpoint-628/tokenizer_config.json
+
40%|████ | 629/1570 [1:49:25<118:03:27, 451.66s/it]
40%|████ | 630/1570 [1:49:31<82:59:36, 317.85s/it]
40%|████ | 631/1570 [1:49:37<58:28:26, 224.18s/it]
40%|████ | 632/1570 [1:49:42<41:19:32, 158.61s/it]
40%|████ | 633/1570 [1:49:48<29:19:42, 112.68s/it]
40%|████ | 634/1570 [1:49:53<20:56:30, 80.55s/it]
40%|████ | 635/1570 [1:49:59<15:04:42, 58.06s/it]
41%|████ | 636/1570 [1:50:05<10:58:28, 42.30s/it]
41%|████ | 637/1570 [1:50:10<8:06:25, 31.28s/it]
41%|████ | 638/1570 [1:50:16<6:06:06, 23.57s/it]
41%|████ | 639/1570 [1:50:21<4:41:50, 18.16s/it]
41%|████ | 640/1570 [1:50:27<3:42:55, 14.38s/it]
{'loss': '0.003085', 'grad_norm': '0.05195', 'learning_rate': '6.444e-05', 'epoch': '2.038'}
+
41%|████ | 640/1570 [1:50:27<3:42:55, 14.38s/it]
41%|████ | 641/1570 [1:50:32<3:01:38, 11.73s/it]
41%|████ | 642/1570 [1:50:38<2:32:47, 9.88s/it]
41%|████ | 643/1570 [1:50:43<2:12:32, 8.58s/it]
41%|████ | 644/1570 [1:50:49<1:58:23, 7.67s/it]
41%|████ | 645/1570 [1:50:55<1:48:34, 7.04s/it]
41%|████ | 646/1570 [1:51:00<1:41:40, 6.60s/it]
41%|████ | 647/1570 [1:51:06<1:36:55, 6.30s/it]
41%|████▏ | 648/1570 [1:51:11<1:33:16, 6.07s/it]
41%|████▏ | 649/1570 [1:51:17<1:30:46, 5.91s/it]
41%|████▏ | 650/1570 [1:51:22<1:29:02, 5.81s/it]
41%|████▏ | 651/1570 [1:51:28<1:27:44, 5.73s/it]
42%|████▏ | 652/1570 [1:51:33<1:26:46, 5.67s/it]
42%|████▏ | 653/1570 [1:51:39<1:26:02, 5.63s/it]
42%|████▏ | 654/1570 [1:51:45<1:25:29, 5.60s/it]
42%|████▏ | 655/1570 [1:51:50<1:25:08, 5.58s/it]
42%|████▏ | 656/1570 [1:51:56<1:24:51, 5.57s/it]
42%|████▏ | 657/1570 [1:52:01<1:24:38, 5.56s/it]
42%|████▏ | 658/1570 [1:52:07<1:24:35, 5.57s/it]
42%|████▏ | 659/1570 [1:52:12<1:24:20, 5.56s/it]
42%|████▏ | 660/1570 [1:52:18<1:24:15, 5.56s/it]
{'loss': '0.002657', 'grad_norm': '0.0511', 'learning_rate': '6.251e-05', 'epoch': '2.102'}
+
42%|████▏ | 660/1570 [1:52:18<1:24:15, 5.56s/it]
42%|████▏ | 661/1570 [1:52:23<1:24:09, 5.56s/it]
42%|████▏ | 662/1570 [1:52:29<1:24:11, 5.56s/it]
42%|████▏ | 663/1570 [1:52:34<1:24:03, 5.56s/it]
42%|████▏ | 664/1570 [1:52:40<1:23:51, 5.55s/it]
42%|████▏ | 665/1570 [1:52:46<1:23:44, 5.55s/it]
42%|████▏ | 666/1570 [1:52:51<1:23:33, 5.55s/it]
42%|████▏ | 667/1570 [1:52:57<1:23:23, 5.54s/it]
43%|████▎ | 668/1570 [1:53:02<1:23:19, 5.54s/it]
43%|████▎ | 669/1570 [1:53:08<1:23:13, 5.54s/it]
43%|████▎ | 670/1570 [1:53:13<1:23:02, 5.54s/it]
43%|████▎ | 671/1570 [1:53:19<1:23:02, 5.54s/it]
43%|████▎ | 672/1570 [1:53:24<1:22:59, 5.55s/it]
43%|████▎ | 673/1570 [1:53:30<1:23:02, 5.55s/it]
43%|████▎ | 674/1570 [1:53:36<1:23:04, 5.56s/it]
43%|████▎ | 675/1570 [1:53:41<1:23:35, 5.60s/it]
43%|████▎ | 676/1570 [1:53:47<1:23:16, 5.59s/it]
43%|████▎ | 677/1570 [1:53:52<1:23:08, 5.59s/it]
43%|████▎ | 678/1570 [1:53:58<1:22:47, 5.57s/it]
43%|████▎ | 679/1570 [1:54:03<1:22:21, 5.55s/it]
43%|████▎ | 680/1570 [1:54:09<1:22:05, 5.53s/it]
{'loss': '0.001602', 'grad_norm': '0.01101', 'learning_rate': '6.057e-05', 'epoch': '2.166'}
+
43%|████▎ | 680/1570 [1:54:09<1:22:05, 5.53s/it]
43%|████▎ | 681/1570 [1:54:14<1:21:53, 5.53s/it]
43%|████▎ | 682/1570 [1:54:20<1:21:37, 5.52s/it]
44%|████▎ | 683/1570 [1:54:25<1:21:23, 5.51s/it]
44%|████▎ | 684/1570 [1:54:31<1:21:21, 5.51s/it]
44%|████▎ | 685/1570 [1:54:36<1:21:20, 5.51s/it]
44%|████▎ | 686/1570 [1:54:42<1:21:13, 5.51s/it]
44%|████▍ | 687/1570 [1:54:47<1:21:03, 5.51s/it]
44%|████▍ | 688/1570 [1:54:53<1:21:00, 5.51s/it]
44%|████▍ | 689/1570 [1:54:58<1:20:51, 5.51s/it]
44%|████▍ | 690/1570 [1:55:04<1:20:49, 5.51s/it]
44%|████▍ | 691/1570 [1:55:09<1:20:40, 5.51s/it]
44%|████▍ | 692/1570 [1:55:15<1:20:35, 5.51s/it]
44%|████▍ | 693/1570 [1:55:20<1:20:36, 5.51s/it]
44%|████▍ | 694/1570 [1:55:26<1:20:39, 5.52s/it]
44%|████▍ | 695/1570 [1:55:32<1:20:43, 5.54s/it]
44%|████▍ | 696/1570 [1:55:37<1:20:31, 5.53s/it]
44%|████▍ | 697/1570 [1:55:43<1:20:29, 5.53s/it]
44%|████▍ | 698/1570 [1:55:48<1:20:30, 5.54s/it]
45%|████▍ | 699/1570 [1:55:54<1:20:16, 5.53s/it]
45%|████▍ | 700/1570 [1:55:59<1:20:16, 5.54s/it]
{'loss': '0.003167', 'grad_norm': '0.02784', 'learning_rate': '5.86e-05', 'epoch': '2.23'}
+
45%|████▍ | 700/1570 [1:55:59<1:20:16, 5.54s/it]
45%|████▍ | 701/1570 [1:56:05<1:20:06, 5.53s/it]
45%|████▍ | 702/1570 [1:56:10<1:19:57, 5.53s/it]
45%|████▍ | 703/1570 [1:56:16<1:19:52, 5.53s/it]
45%|████▍ | 704/1570 [1:56:21<1:19:47, 5.53s/it]
45%|████▍ | 705/1570 [1:56:27<1:19:40, 5.53s/it]
45%|████▍ | 706/1570 [1:56:32<1:19:32, 5.52s/it]
45%|████▌ | 707/1570 [1:56:38<1:19:25, 5.52s/it]
45%|████▌ | 708/1570 [1:56:43<1:19:14, 5.52s/it]
45%|████▌ | 709/1570 [1:56:49<1:19:12, 5.52s/it]
45%|████▌ | 710/1570 [1:56:54<1:19:15, 5.53s/it]
45%|████▌ | 711/1570 [1:57:00<1:19:09, 5.53s/it]
45%|████▌ | 712/1570 [1:57:06<1:19:09, 5.54s/it]
45%|████▌ | 713/1570 [1:57:12<1:20:57, 5.67s/it]
45%|████▌ | 714/1570 [1:57:17<1:20:14, 5.62s/it]
46%|████▌ | 715/1570 [1:57:23<1:19:50, 5.60s/it]
46%|████▌ | 716/1570 [1:57:29<1:21:13, 5.71s/it]
46%|████▌ | 717/1570 [1:57:34<1:20:38, 5.67s/it]
46%|████▌ | 718/1570 [1:57:40<1:19:59, 5.63s/it]
46%|████▌ | 719/1570 [1:57:45<1:19:36, 5.61s/it]
46%|████▌ | 720/1570 [1:57:51<1:19:12, 5.59s/it]
{'loss': '0.002939', 'grad_norm': '0.008262', 'learning_rate': '5.663e-05', 'epoch': '2.294'}
+
46%|████▌ | 720/1570 [1:57:51<1:19:12, 5.59s/it]
46%|████▌ | 721/1570 [1:57:56<1:18:53, 5.58s/it]
46%|████▌ | 722/1570 [1:58:02<1:18:35, 5.56s/it]
46%|████▌ | 723/1570 [1:58:07<1:18:23, 5.55s/it]
46%|████▌ | 724/1570 [1:58:13<1:18:07, 5.54s/it]
46%|████▌ | 725/1570 [1:58:18<1:17:57, 5.54s/it]
46%|████▌ | 726/1570 [1:58:24<1:17:53, 5.54s/it]
46%|████▋ | 727/1570 [1:58:30<1:17:51, 5.54s/it]
46%|████▋ | 728/1570 [1:58:35<1:17:50, 5.55s/it]
46%|████▋ | 729/1570 [1:58:41<1:17:49, 5.55s/it]
46%|████▋ | 730/1570 [1:58:46<1:17:45, 5.55s/it]
47%|████▋ | 731/1570 [1:58:52<1:17:30, 5.54s/it]
47%|████▋ | 732/1570 [1:58:57<1:17:02, 5.52s/it]
47%|████▋ | 733/1570 [1:59:03<1:16:48, 5.51s/it]
47%|████▋ | 734/1570 [1:59:08<1:16:45, 5.51s/it]
47%|████▋ | 735/1570 [1:59:14<1:16:34, 5.50s/it]
47%|████▋ | 736/1570 [1:59:19<1:16:20, 5.49s/it]
47%|████▋ | 737/1570 [1:59:25<1:16:18, 5.50s/it]
47%|████▋ | 738/1570 [1:59:30<1:16:18, 5.50s/it]
47%|████▋ | 739/1570 [1:59:36<1:16:03, 5.49s/it]
47%|████▋ | 740/1570 [1:59:41<1:16:07, 5.50s/it]
{'loss': '0.005156', 'grad_norm': '0.03108', 'learning_rate': '5.464e-05', 'epoch': '2.357'}
+
47%|████▋ | 740/1570 [1:59:41<1:16:07, 5.50s/it]
47%|████▋ | 741/1570 [1:59:47<1:16:11, 5.51s/it]
47%|████▋ | 742/1570 [1:59:52<1:16:09, 5.52s/it]
47%|████▋ | 743/1570 [1:59:58<1:16:04, 5.52s/it]
47%|████▋ | 744/1570 [2:00:03<1:16:00, 5.52s/it]
47%|████▋ | 745/1570 [2:00:09<1:15:43, 5.51s/it]
48%|████▊ | 746/1570 [2:00:14<1:15:48, 5.52s/it]
48%|████▊ | 747/1570 [2:00:20<1:15:40, 5.52s/it]
48%|████▊ | 748/1570 [2:00:25<1:15:24, 5.50s/it]
48%|████▊ | 749/1570 [2:00:31<1:15:20, 5.51s/it]
48%|████▊ | 750/1570 [2:00:36<1:15:24, 5.52s/it]
48%|████▊ | 751/1570 [2:00:42<1:15:30, 5.53s/it]
48%|████▊ | 752/1570 [2:00:47<1:15:23, 5.53s/it]
48%|████▊ | 753/1570 [2:00:53<1:15:23, 5.54s/it]
48%|████▊ | 754/1570 [2:00:59<1:15:22, 5.54s/it]
48%|████▊ | 755/1570 [2:01:04<1:15:14, 5.54s/it]
48%|████▊ | 756/1570 [2:01:10<1:15:20, 5.55s/it]
48%|████▊ | 757/1570 [2:01:15<1:15:12, 5.55s/it]
48%|████▊ | 758/1570 [2:01:21<1:15:03, 5.55s/it]
48%|████▊ | 759/1570 [2:01:26<1:14:56, 5.54s/it]
48%|████▊ | 760/1570 [2:01:32<1:14:43, 5.53s/it]
{'loss': '0.001113', 'grad_norm': '0.04322', 'learning_rate': '5.265e-05', 'epoch': '2.421'}
+
48%|████▊ | 760/1570 [2:01:32<1:14:43, 5.53s/it]
48%|████▊ | 761/1570 [2:01:37<1:14:37, 5.53s/it]
49%|████▊ | 762/1570 [2:01:43<1:14:36, 5.54s/it]
49%|████▊ | 763/1570 [2:01:48<1:14:35, 5.55s/it]
49%|████▊ | 764/1570 [2:01:54<1:14:26, 5.54s/it]
49%|████▊ | 765/1570 [2:02:00<1:14:24, 5.55s/it]
49%|████▉ | 766/1570 [2:02:05<1:14:24, 5.55s/it]
49%|████▉ | 767/1570 [2:02:11<1:14:31, 5.57s/it]
49%|████▉ | 768/1570 [2:02:16<1:14:36, 5.58s/it]
49%|████▉ | 769/1570 [2:02:22<1:14:26, 5.58s/it]
49%|████▉ | 770/1570 [2:02:27<1:14:24, 5.58s/it]
49%|████▉ | 771/1570 [2:02:33<1:14:13, 5.57s/it]
49%|████▉ | 772/1570 [2:02:39<1:14:08, 5.57s/it]
49%|████▉ | 773/1570 [2:02:44<1:13:59, 5.57s/it]
49%|████▉ | 774/1570 [2:02:50<1:13:47, 5.56s/it]
49%|████▉ | 775/1570 [2:02:55<1:13:42, 5.56s/it]
49%|████▉ | 776/1570 [2:03:01<1:13:36, 5.56s/it]
49%|████▉ | 777/1570 [2:03:06<1:13:25, 5.56s/it]
50%|████▉ | 778/1570 [2:03:12<1:13:28, 5.57s/it]
50%|████▉ | 779/1570 [2:03:18<1:13:12, 5.55s/it]
50%|████▉ | 780/1570 [2:03:23<1:13:09, 5.56s/it]
{'loss': '0.002006', 'grad_norm': '0.02208', 'learning_rate': '5.065e-05', 'epoch': '2.485'}
+
50%|████▉ | 780/1570 [2:03:23<1:13:09, 5.56s/it]
50%|████▉ | 781/1570 [2:03:29<1:13:12, 5.57s/it]
50%|████▉ | 782/1570 [2:03:34<1:13:14, 5.58s/it]
50%|████▉ | 783/1570 [2:03:40<1:13:12, 5.58s/it]
50%|████▉ | 784/1570 [2:03:45<1:13:07, 5.58s/it]
50%|█████ | 785/1570 [2:03:51<1:13:13, 5.60s/it]
50%|█████ | 786/1570 [2:03:57<1:13:03, 5.59s/it]
50%|█████ | 787/1570 [2:04:02<1:12:59, 5.59s/it]
50%|█████ | 788/1570 [2:04:08<1:12:46, 5.58s/it]
50%|█████ | 789/1570 [2:04:13<1:12:29, 5.57s/it]
50%|█████ | 790/1570 [2:04:19<1:12:29, 5.58s/it]
50%|█████ | 791/1570 [2:04:25<1:12:23, 5.58s/it]
50%|█████ | 792/1570 [2:04:30<1:12:16, 5.57s/it]
51%|█████ | 793/1570 [2:04:36<1:12:10, 5.57s/it]
51%|█████ | 794/1570 [2:04:41<1:12:06, 5.58s/it]
51%|█████ | 795/1570 [2:04:47<1:12:00, 5.57s/it]
51%|█████ | 796/1570 [2:04:52<1:11:45, 5.56s/it]
51%|█████ | 797/1570 [2:04:58<1:11:42, 5.57s/it]
51%|█████ | 798/1570 [2:05:03<1:11:30, 5.56s/it]
51%|█████ | 799/1570 [2:05:09<1:11:21, 5.55s/it]
51%|█████ | 800/1570 [2:05:15<1:11:25, 5.57s/it]
{'loss': '0.002107', 'grad_norm': '0.04726', 'learning_rate': '4.865e-05', 'epoch': '2.549'}
+
51%|█████ | 800/1570 [2:05:15<1:11:25, 5.57s/it]
51%|█████ | 801/1570 [2:05:20<1:11:14, 5.56s/it]
51%|█████ | 802/1570 [2:05:26<1:11:10, 5.56s/it]
51%|█████ | 803/1570 [2:05:31<1:11:15, 5.57s/it]
51%|█████ | 804/1570 [2:05:37<1:11:09, 5.57s/it]
51%|█████▏ | 805/1570 [2:05:42<1:11:02, 5.57s/it]
51%|█████▏ | 806/1570 [2:05:48<1:11:04, 5.58s/it]
51%|█████▏ | 807/1570 [2:05:54<1:10:55, 5.58s/it]
51%|█████▏ | 808/1570 [2:05:59<1:10:47, 5.57s/it]
52%|█████▏ | 809/1570 [2:06:05<1:10:49, 5.58s/it]
52%|█████▏ | 810/1570 [2:06:10<1:10:36, 5.57s/it]
52%|█████▏ | 811/1570 [2:06:16<1:10:25, 5.57s/it]
52%|█████▏ | 812/1570 [2:06:21<1:10:17, 5.56s/it]
52%|█████▏ | 813/1570 [2:06:27<1:10:08, 5.56s/it]
52%|█████▏ | 814/1570 [2:06:33<1:10:04, 5.56s/it]
52%|█████▏ | 815/1570 [2:06:38<1:10:03, 5.57s/it]
52%|█████▏ | 816/1570 [2:06:44<1:09:58, 5.57s/it]
52%|█████▏ | 817/1570 [2:06:49<1:09:54, 5.57s/it]
52%|█████▏ | 818/1570 [2:06:55<1:09:51, 5.57s/it]
52%|█████▏ | 819/1570 [2:07:00<1:09:45, 5.57s/it]
52%|█████▏ | 820/1570 [2:07:06<1:09:36, 5.57s/it]
{'loss': '0.001612', 'grad_norm': '0.2759', 'learning_rate': '4.665e-05', 'epoch': '2.613'}
+
52%|█████▏ | 820/1570 [2:07:06<1:09:36, 5.57s/it]
52%|█████▏ | 821/1570 [2:07:12<1:09:32, 5.57s/it]
52%|█████▏ | 822/1570 [2:07:17<1:09:31, 5.58s/it]
52%|█████▏ | 823/1570 [2:07:23<1:09:26, 5.58s/it]
52%|█████▏ | 824/1570 [2:07:28<1:09:20, 5.58s/it]
53%|█████▎ | 825/1570 [2:07:34<1:09:19, 5.58s/it]
53%|█████▎ | 826/1570 [2:07:39<1:09:13, 5.58s/it]
53%|█████▎ | 827/1570 [2:07:45<1:09:06, 5.58s/it]
53%|█████▎ | 828/1570 [2:07:51<1:08:54, 5.57s/it]
53%|█████▎ | 829/1570 [2:07:56<1:08:48, 5.57s/it]
53%|█████▎ | 830/1570 [2:08:02<1:08:45, 5.57s/it]
53%|█████▎ | 831/1570 [2:08:07<1:08:37, 5.57s/it]
53%|█████▎ | 832/1570 [2:08:13<1:08:37, 5.58s/it]
53%|█████▎ | 833/1570 [2:08:18<1:08:21, 5.57s/it]
53%|█████▎ | 834/1570 [2:08:24<1:08:23, 5.58s/it]
53%|█████▎ | 835/1570 [2:08:30<1:08:23, 5.58s/it]
53%|█████▎ | 836/1570 [2:08:35<1:08:21, 5.59s/it]
53%|█████▎ | 837/1570 [2:08:41<1:08:17, 5.59s/it]
53%|█████▎ | 838/1570 [2:08:46<1:08:05, 5.58s/it]
53%|█████▎ | 839/1570 [2:08:52<1:07:59, 5.58s/it]
54%|█████▎ | 840/1570 [2:08:58<1:07:38, 5.56s/it]
{'loss': '0.000449', 'grad_norm': '0.002856', 'learning_rate': '4.466e-05', 'epoch': '2.677'}
+
54%|█████▎ | 840/1570 [2:08:58<1:07:38, 5.56s/it]
54%|█████▎ | 841/1570 [2:09:03<1:07:25, 5.55s/it]
54%|█████▎ | 842/1570 [2:09:09<1:07:07, 5.53s/it]
54%|█████▎ | 843/1570 [2:09:14<1:06:57, 5.53s/it]
54%|█████▍ | 844/1570 [2:09:20<1:06:42, 5.51s/it]
54%|█████▍ | 845/1570 [2:09:25<1:06:31, 5.51s/it]
54%|█████▍ | 846/1570 [2:09:31<1:06:23, 5.50s/it]
54%|█████▍ | 847/1570 [2:09:36<1:06:20, 5.50s/it]
54%|█████▍ | 848/1570 [2:09:42<1:06:15, 5.51s/it]
54%|█████▍ | 849/1570 [2:09:47<1:06:15, 5.51s/it]
54%|█████▍ | 850/1570 [2:09:53<1:06:10, 5.52s/it]
54%|█████▍ | 851/1570 [2:09:58<1:06:03, 5.51s/it]
54%|█████▍ | 852/1570 [2:10:04<1:06:02, 5.52s/it]
54%|█████▍ | 853/1570 [2:10:09<1:06:03, 5.53s/it]
54%|█████▍ | 854/1570 [2:10:15<1:06:03, 5.54s/it]
54%|█████▍ | 855/1570 [2:10:20<1:06:05, 5.55s/it]
55%|█████▍ | 856/1570 [2:10:26<1:05:59, 5.55s/it]
55%|█████▍ | 857/1570 [2:10:31<1:06:02, 5.56s/it]
55%|█████▍ | 858/1570 [2:10:37<1:05:58, 5.56s/it]
55%|█████▍ | 859/1570 [2:10:43<1:05:48, 5.55s/it]
55%|█████▍ | 860/1570 [2:10:48<1:05:47, 5.56s/it]
{'loss': '0.001641', 'grad_norm': '0.03328', 'learning_rate': '4.268e-05', 'epoch': '2.74'}
+
55%|█████▍ | 860/1570 [2:10:48<1:05:47, 5.56s/it]
55%|█████▍ | 861/1570 [2:10:54<1:05:44, 5.56s/it]
55%|█████▍ | 862/1570 [2:10:59<1:05:37, 5.56s/it]
55%|█████▍ | 863/1570 [2:11:05<1:05:35, 5.57s/it]
55%|█████▌ | 864/1570 [2:11:10<1:05:28, 5.56s/it]
55%|█████▌ | 865/1570 [2:11:16<1:05:23, 5.57s/it]
55%|█████▌ | 866/1570 [2:11:21<1:05:15, 5.56s/it]
55%|█████▌ | 867/1570 [2:11:27<1:05:12, 5.57s/it]
55%|█████▌ | 868/1570 [2:11:33<1:05:04, 5.56s/it]
55%|█████▌ | 869/1570 [2:11:38<1:04:58, 5.56s/it]
55%|█████▌ | 870/1570 [2:11:44<1:04:59, 5.57s/it]
55%|█████▌ | 871/1570 [2:11:49<1:04:58, 5.58s/it]
56%|█████▌ | 872/1570 [2:11:55<1:04:57, 5.58s/it]
56%|█████▌ | 873/1570 [2:12:01<1:04:45, 5.57s/it]
56%|█████▌ | 874/1570 [2:12:06<1:04:34, 5.57s/it]
56%|█████▌ | 875/1570 [2:12:12<1:04:20, 5.56s/it]
56%|█████▌ | 876/1570 [2:12:17<1:04:23, 5.57s/it]
56%|█████▌ | 877/1570 [2:12:23<1:04:20, 5.57s/it]
56%|█████▌ | 878/1570 [2:12:28<1:04:18, 5.58s/it]
56%|█████▌ | 879/1570 [2:12:34<1:04:02, 5.56s/it]
56%|█████▌ | 880/1570 [2:12:39<1:04:01, 5.57s/it]
{'loss': '0.0008663', 'grad_norm': '0.03642', 'learning_rate': '4.071e-05', 'epoch': '2.804'}
+
56%|█████▌ | 880/1570 [2:12:39<1:04:01, 5.57s/it]
56%|█████▌ | 881/1570 [2:12:45<1:04:03, 5.58s/it]
56%|█████▌ | 882/1570 [2:12:51<1:03:57, 5.58s/it]
56%|█████▌ | 883/1570 [2:12:56<1:03:48, 5.57s/it]
56%|█████▋ | 884/1570 [2:13:02<1:03:42, 5.57s/it]
56%|█████▋ | 885/1570 [2:13:07<1:03:34, 5.57s/it]
56%|█████▋ | 886/1570 [2:13:13<1:03:33, 5.58s/it]
56%|█████▋ | 887/1570 [2:13:18<1:03:28, 5.58s/it]
57%|█████▋ | 888/1570 [2:13:24<1:03:21, 5.57s/it]
57%|█████▋ | 889/1570 [2:13:30<1:03:18, 5.58s/it]
57%|█████▋ | 890/1570 [2:13:35<1:03:14, 5.58s/it]
57%|█████▋ | 891/1570 [2:13:41<1:03:04, 5.57s/it]
57%|█████▋ | 892/1570 [2:13:46<1:02:53, 5.56s/it]
57%|█████▋ | 893/1570 [2:13:52<1:02:40, 5.55s/it]
57%|█████▋ | 894/1570 [2:13:57<1:02:19, 5.53s/it]
57%|█████▋ | 895/1570 [2:14:03<1:02:14, 5.53s/it]
57%|█████▋ | 896/1570 [2:14:08<1:02:06, 5.53s/it]
57%|█████▋ | 897/1570 [2:14:14<1:02:01, 5.53s/it]
57%|█████▋ | 898/1570 [2:14:19<1:01:51, 5.52s/it]
57%|█████▋ | 899/1570 [2:14:25<1:01:40, 5.51s/it]
57%|█████▋ | 900/1570 [2:14:30<1:01:37, 5.52s/it]
{'loss': '0.0006273', 'grad_norm': '0.06211', 'learning_rate': '3.875e-05', 'epoch': '2.868'}
+
57%|█████▋ | 900/1570 [2:14:30<1:01:37, 5.52s/it]
57%|█████▋ | 901/1570 [2:14:36<1:01:29, 5.52s/it]
57%|█████▋ | 902/1570 [2:14:42<1:01:31, 5.53s/it]
58%|█████▊ | 903/1570 [2:14:47<1:01:22, 5.52s/it]
58%|█████▊ | 904/1570 [2:14:53<1:01:11, 5.51s/it]
58%|█████▊ | 905/1570 [2:14:58<1:01:12, 5.52s/it]
58%|█████▊ | 906/1570 [2:15:04<1:01:03, 5.52s/it]
58%|█████▊ | 907/1570 [2:15:09<1:01:05, 5.53s/it]
58%|█████▊ | 908/1570 [2:15:15<1:01:01, 5.53s/it]
58%|█████▊ | 909/1570 [2:15:20<1:00:55, 5.53s/it]
58%|█████▊ | 910/1570 [2:15:26<1:00:53, 5.54s/it]
58%|█████▊ | 911/1570 [2:15:31<1:00:52, 5.54s/it]
58%|█████▊ | 912/1570 [2:15:37<1:00:44, 5.54s/it]
58%|█████▊ | 913/1570 [2:15:42<1:00:33, 5.53s/it]
58%|█████▊ | 914/1570 [2:15:48<1:00:24, 5.53s/it]
58%|█████▊ | 915/1570 [2:15:53<1:00:22, 5.53s/it]
58%|█████▊ | 916/1570 [2:15:59<1:00:12, 5.52s/it]
58%|█████▊ | 917/1570 [2:16:04<1:00:07, 5.52s/it]
58%|█████▊ | 918/1570 [2:16:10<1:00:05, 5.53s/it]
59%|█████▊ | 919/1570 [2:16:16<1:00:02, 5.53s/it]
59%|█████▊ | 920/1570 [2:16:21<59:56, 5.53s/it]
{'loss': '0.0009967', 'grad_norm': '0.002508', 'learning_rate': '3.682e-05', 'epoch': '2.932'}
+
59%|█████▊ | 920/1570 [2:16:21<59:56, 5.53s/it]
59%|█████▊ | 921/1570 [2:16:27<59:48, 5.53s/it]
59%|█████▊ | 922/1570 [2:16:32<59:44, 5.53s/it]
59%|█████▉ | 923/1570 [2:16:38<59:41, 5.54s/it]
59%|█████▉ | 924/1570 [2:16:43<59:41, 5.54s/it]
59%|█████▉ | 925/1570 [2:16:49<59:37, 5.55s/it]
59%|█████▉ | 926/1570 [2:16:54<59:30, 5.54s/it]
59%|█████▉ | 927/1570 [2:17:00<59:25, 5.55s/it]
59%|█████▉ | 928/1570 [2:17:05<59:19, 5.54s/it]
59%|█████▉ | 929/1570 [2:17:11<59:11, 5.54s/it]
59%|█████▉ | 930/1570 [2:17:16<59:09, 5.55s/it]
59%|█████▉ | 931/1570 [2:17:22<59:12, 5.56s/it]
59%|█████▉ | 932/1570 [2:17:28<59:03, 5.55s/it]
59%|█████▉ | 933/1570 [2:17:33<59:01, 5.56s/it]
59%|█████▉ | 934/1570 [2:17:39<58:49, 5.55s/it]
60%|█████▉ | 935/1570 [2:17:44<58:41, 5.54s/it]
60%|█████▉ | 936/1570 [2:17:50<58:34, 5.54s/it]
60%|█████▉ | 937/1570 [2:17:55<58:25, 5.54s/it]
60%|█████▉ | 938/1570 [2:18:01<58:25, 5.55s/it]
60%|█████▉ | 939/1570 [2:18:06<58:25, 5.56s/it]
60%|█████▉ | 940/1570 [2:18:12<58:26, 5.57s/it]
{'loss': '0.0005241', 'grad_norm': '0.08202', 'learning_rate': '3.49e-05', 'epoch': '2.996'}
+
60%|█████▉ | 940/1570 [2:18:12<58:26, 5.57s/it]
60%|█████▉ | 941/1570 [2:18:18<58:18, 5.56s/it]
60%|██████ | 942/1570 [2:18:19<46:39, 4.46s/it][INFO|trainer.py:2668] 2026-08-22 05:34:29,171 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 05:34:29,171 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 05:34:29,171 >> Batch size = 8
+
+
0%| | 0/214 [00:00> Saving model checkpoint to results/qwen3/teacher_lora/checkpoint-942
+[INFO|configuration_utils.py:780] 2026-08-22 05:59:25,263 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 05:59:25,266 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 05:59:25,354 >> chat template saved in results/qwen3/teacher_lora/checkpoint-942/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 05:59:25,354 >> tokenizer config file saved in results/qwen3/teacher_lora/checkpoint-942/tokenizer_config.json
+
60%|██████ | 943/1570 [2:43:23<79:07:12, 454.28s/it]
60%|██████ | 944/1570 [2:43:29<55:35:20, 319.68s/it]
60%|██████ | 945/1570 [2:43:35<39:08:29, 225.46s/it]
60%|██████ | 946/1570 [2:43:40<27:38:39, 159.49s/it]
60%|██████ | 947/1570 [2:43:46<19:36:34, 113.31s/it]
60%|██████ | 948/1570 [2:43:51<13:59:38, 80.99s/it]
60%|██████ | 949/1570 [2:43:57<10:03:59, 58.36s/it]
61%|██████ | 950/1570 [2:44:02<7:19:10, 42.50s/it]
61%|██████ | 951/1570 [2:44:08<5:24:09, 31.42s/it]
61%|██████ | 952/1570 [2:44:13<4:03:51, 23.68s/it]
61%|██████ | 953/1570 [2:44:19<3:07:31, 18.24s/it]
61%|██████ | 954/1570 [2:44:25<2:28:15, 14.44s/it]
61%|██████ | 955/1570 [2:44:30<2:00:50, 11.79s/it]
61%|██████ | 956/1570 [2:44:36<1:41:28, 9.92s/it]
61%|██████ | 957/1570 [2:44:41<1:27:59, 8.61s/it]
61%|██████ | 958/1570 [2:44:47<1:20:05, 7.85s/it]
61%|██████ | 959/1570 [2:44:53<1:12:59, 7.17s/it]
61%|██████ | 960/1570 [2:44:59<1:08:03, 6.69s/it]
{'loss': '0.003412', 'grad_norm': '0.05989', 'learning_rate': '3.301e-05', 'epoch': '3.057'}
+
61%|██████ | 960/1570 [2:44:59<1:08:03, 6.69s/it]
61%|██████ | 961/1570 [2:45:04<1:04:31, 6.36s/it]
61%|██████▏ | 962/1570 [2:45:10<1:03:14, 6.24s/it]
61%|██████▏ | 963/1570 [2:45:16<1:01:09, 6.05s/it]
61%|██████▏ | 964/1570 [2:45:21<59:43, 5.91s/it]
61%|██████▏ | 965/1570 [2:45:27<58:35, 5.81s/it]
62%|██████▏ | 966/1570 [2:45:32<57:52, 5.75s/it]
62%|██████▏ | 967/1570 [2:45:38<57:19, 5.70s/it]
62%|██████▏ | 968/1570 [2:45:44<56:55, 5.67s/it]
62%|██████▏ | 969/1570 [2:45:49<56:34, 5.65s/it]
62%|██████▏ | 970/1570 [2:45:55<56:14, 5.62s/it]
62%|██████▏ | 971/1570 [2:46:00<56:03, 5.62s/it]
62%|██████▏ | 972/1570 [2:46:06<55:47, 5.60s/it]
62%|██████▏ | 973/1570 [2:46:12<55:40, 5.59s/it]
62%|██████▏ | 974/1570 [2:46:17<55:35, 5.60s/it]
62%|██████▏ | 975/1570 [2:46:23<55:25, 5.59s/it]
62%|██████▏ | 976/1570 [2:46:28<55:25, 5.60s/it]
62%|██████▏ | 977/1570 [2:46:34<55:24, 5.61s/it]
62%|██████▏ | 978/1570 [2:46:40<55:21, 5.61s/it]
62%|██████▏ | 979/1570 [2:46:45<55:12, 5.60s/it]
62%|██████▏ | 980/1570 [2:46:51<55:04, 5.60s/it]
{'loss': '0.000734', 'grad_norm': '0.002755', 'learning_rate': '3.114e-05', 'epoch': '3.121'}
+
62%|██████▏ | 980/1570 [2:46:51<55:04, 5.60s/it]
62%|██████▏ | 981/1570 [2:46:56<54:58, 5.60s/it]
63%|██████▎ | 982/1570 [2:47:02<54:52, 5.60s/it]
63%|██████▎ | 983/1570 [2:47:08<54:55, 5.61s/it]
63%|██████▎ | 984/1570 [2:47:13<54:48, 5.61s/it]
63%|██████▎ | 985/1570 [2:47:19<54:33, 5.60s/it]
63%|██████▎ | 986/1570 [2:47:24<54:25, 5.59s/it]
63%|██████▎ | 987/1570 [2:47:30<54:21, 5.59s/it]
63%|██████▎ | 988/1570 [2:47:36<54:05, 5.58s/it]
63%|██████▎ | 989/1570 [2:47:41<54:01, 5.58s/it]
63%|██████▎ | 990/1570 [2:47:47<53:55, 5.58s/it]
63%|██████▎ | 991/1570 [2:47:52<53:56, 5.59s/it]
63%|██████▎ | 992/1570 [2:47:58<53:51, 5.59s/it]
63%|██████▎ | 993/1570 [2:48:04<53:48, 5.60s/it]
63%|██████▎ | 994/1570 [2:48:09<53:43, 5.60s/it]
63%|██████▎ | 995/1570 [2:48:15<53:35, 5.59s/it]
63%|██████▎ | 996/1570 [2:48:20<53:30, 5.59s/it]
64%|██████▎ | 997/1570 [2:48:26<53:29, 5.60s/it]
64%|██████▎ | 998/1570 [2:48:32<53:25, 5.60s/it]
64%|██████▎ | 999/1570 [2:48:37<53:18, 5.60s/it]
64%|██████▎ | 1000/1570 [2:48:43<53:13, 5.60s/it]
{'loss': '0.000787', 'grad_norm': '0.004057', 'learning_rate': '2.931e-05', 'epoch': '3.185'}
+
64%|██████▎ | 1000/1570 [2:48:43<53:13, 5.60s/it]
64%|██████▍ | 1001/1570 [2:48:48<53:08, 5.60s/it]
64%|██████▍ | 1002/1570 [2:48:54<52:58, 5.60s/it]
64%|██████▍ | 1003/1570 [2:48:59<52:35, 5.57s/it]
64%|██████▍ | 1004/1570 [2:49:05<52:22, 5.55s/it]
64%|██████▍ | 1005/1570 [2:49:10<52:15, 5.55s/it]
64%|██████▍ | 1006/1570 [2:49:16<52:16, 5.56s/it]
64%|██████▍ | 1007/1570 [2:49:22<52:12, 5.56s/it]
64%|██████▍ | 1008/1570 [2:49:27<52:14, 5.58s/it]
64%|██████▍ | 1009/1570 [2:49:33<52:09, 5.58s/it]
64%|██████▍ | 1010/1570 [2:49:38<52:05, 5.58s/it]
64%|██████▍ | 1011/1570 [2:49:44<52:00, 5.58s/it]
64%|██████▍ | 1012/1570 [2:49:50<51:56, 5.59s/it]
65%|██████▍ | 1013/1570 [2:49:55<51:43, 5.57s/it]
65%|██████▍ | 1014/1570 [2:50:01<51:39, 5.58s/it]
65%|██████▍ | 1015/1570 [2:50:06<51:34, 5.58s/it]
65%|██████▍ | 1016/1570 [2:50:12<51:24, 5.57s/it]
65%|██████▍ | 1017/1570 [2:50:17<51:18, 5.57s/it]
65%|██████▍ | 1018/1570 [2:50:23<51:17, 5.58s/it]
65%|██████▍ | 1019/1570 [2:50:29<51:14, 5.58s/it]
65%|██████▍ | 1020/1570 [2:50:34<51:05, 5.57s/it]
{'loss': '0.0004239', 'grad_norm': '0.02156', 'learning_rate': '2.751e-05', 'epoch': '3.249'}
+
65%|██████▍ | 1020/1570 [2:50:34<51:05, 5.57s/it]
65%|██████▌ | 1021/1570 [2:50:40<51:02, 5.58s/it]
65%|██████▌ | 1022/1570 [2:50:45<50:51, 5.57s/it]
65%|██████▌ | 1023/1570 [2:50:51<50:45, 5.57s/it]
65%|██████▌ | 1024/1570 [2:50:56<50:41, 5.57s/it]
65%|██████▌ | 1025/1570 [2:51:02<50:41, 5.58s/it]
65%|██████▌ | 1026/1570 [2:51:08<50:40, 5.59s/it]
65%|██████▌ | 1027/1570 [2:51:13<50:37, 5.59s/it]
65%|██████▌ | 1028/1570 [2:51:19<50:28, 5.59s/it]
66%|██████▌ | 1029/1570 [2:51:24<50:17, 5.58s/it]
66%|██████▌ | 1030/1570 [2:51:30<50:15, 5.58s/it]
66%|██████▌ | 1031/1570 [2:51:36<50:09, 5.58s/it]
66%|██████▌ | 1032/1570 [2:51:41<49:59, 5.58s/it]
66%|██████▌ | 1033/1570 [2:51:47<49:57, 5.58s/it]
66%|██████▌ | 1034/1570 [2:51:52<49:50, 5.58s/it]
66%|██████▌ | 1035/1570 [2:51:58<49:43, 5.58s/it]
66%|██████▌ | 1036/1570 [2:52:03<49:42, 5.59s/it]
66%|██████▌ | 1037/1570 [2:52:09<49:37, 5.59s/it]
66%|██████▌ | 1038/1570 [2:52:15<49:32, 5.59s/it]
66%|██████▌ | 1039/1570 [2:52:20<49:26, 5.59s/it]
66%|██████▌ | 1040/1570 [2:52:26<49:23, 5.59s/it]
{'loss': '0.0004422', 'grad_norm': '0.01963', 'learning_rate': '2.574e-05', 'epoch': '3.313'}
+
66%|██████▌ | 1040/1570 [2:52:26<49:23, 5.59s/it]
66%|██████▋ | 1041/1570 [2:52:31<49:15, 5.59s/it]
66%|██████▋ | 1042/1570 [2:52:37<49:12, 5.59s/it]
66%|██████▋ | 1043/1570 [2:52:43<49:01, 5.58s/it]
66%|██████▋ | 1044/1570 [2:52:48<48:51, 5.57s/it]
67%|██████▋ | 1045/1570 [2:52:54<48:43, 5.57s/it]
67%|██████▋ | 1046/1570 [2:52:59<48:43, 5.58s/it]
67%|██████▋ | 1047/1570 [2:53:05<48:39, 5.58s/it]
67%|██████▋ | 1048/1570 [2:53:10<48:34, 5.58s/it]
67%|██████▋ | 1049/1570 [2:53:16<48:31, 5.59s/it]
67%|██████▋ | 1050/1570 [2:53:22<48:28, 5.59s/it]
67%|██████▋ | 1051/1570 [2:53:27<48:23, 5.59s/it]
67%|██████▋ | 1052/1570 [2:53:33<48:16, 5.59s/it]
67%|██████▋ | 1053/1570 [2:53:38<48:12, 5.59s/it]
67%|██████▋ | 1054/1570 [2:53:44<48:12, 5.61s/it]
67%|██████▋ | 1055/1570 [2:53:50<48:08, 5.61s/it]
67%|██████▋ | 1056/1570 [2:53:55<47:52, 5.59s/it]
67%|██████▋ | 1057/1570 [2:54:01<47:35, 5.57s/it]
67%|██████▋ | 1058/1570 [2:54:06<47:21, 5.55s/it]
67%|██████▋ | 1059/1570 [2:54:12<47:10, 5.54s/it]
68%|██████▊ | 1060/1570 [2:54:17<47:07, 5.54s/it]
{'loss': '0.001177', 'grad_norm': '0.001062', 'learning_rate': '2.402e-05', 'epoch': '3.377'}
+
68%|██████▊ | 1060/1570 [2:54:17<47:07, 5.54s/it]
68%|██████▊ | 1061/1570 [2:54:23<47:03, 5.55s/it]
68%|██████▊ | 1062/1570 [2:54:28<46:53, 5.54s/it]
68%|██████▊ | 1063/1570 [2:54:34<46:48, 5.54s/it]
68%|██████▊ | 1064/1570 [2:54:40<46:51, 5.56s/it]
68%|██████▊ | 1065/1570 [2:54:45<46:51, 5.57s/it]
68%|██████▊ | 1066/1570 [2:54:51<46:52, 5.58s/it]
68%|██████▊ | 1067/1570 [2:54:56<47:02, 5.61s/it]
68%|██████▊ | 1068/1570 [2:55:02<46:54, 5.61s/it]
68%|██████▊ | 1069/1570 [2:55:08<46:46, 5.60s/it]
68%|██████▊ | 1070/1570 [2:55:13<46:32, 5.59s/it]
68%|██████▊ | 1071/1570 [2:55:19<46:21, 5.57s/it]
68%|██████▊ | 1072/1570 [2:55:24<46:19, 5.58s/it]
68%|██████▊ | 1073/1570 [2:55:30<46:13, 5.58s/it]
68%|██████▊ | 1074/1570 [2:55:35<46:08, 5.58s/it]
68%|██████▊ | 1075/1570 [2:55:41<46:04, 5.59s/it]
69%|██████▊ | 1076/1570 [2:55:47<45:57, 5.58s/it]
69%|██████▊ | 1077/1570 [2:55:52<45:56, 5.59s/it]
69%|██████▊ | 1078/1570 [2:55:58<45:46, 5.58s/it]
69%|██████▊ | 1079/1570 [2:56:03<45:47, 5.60s/it]
69%|██████▉ | 1080/1570 [2:56:09<45:47, 5.61s/it]
{'loss': '0.0004841', 'grad_norm': '0.04265', 'learning_rate': '2.233e-05', 'epoch': '3.44'}
+
69%|██████▉ | 1080/1570 [2:56:09<45:47, 5.61s/it]
69%|██████▉ | 1081/1570 [2:56:15<45:42, 5.61s/it]
69%|██████▉ | 1082/1570 [2:56:20<45:32, 5.60s/it]
69%|██████▉ | 1083/1570 [2:56:26<45:29, 5.61s/it]
69%|██████▉ | 1084/1570 [2:56:31<45:21, 5.60s/it]
69%|██████▉ | 1085/1570 [2:56:37<45:10, 5.59s/it]
69%|██████▉ | 1086/1570 [2:56:43<45:08, 5.60s/it]
69%|██████▉ | 1087/1570 [2:56:48<45:07, 5.60s/it]
69%|██████▉ | 1088/1570 [2:56:54<45:00, 5.60s/it]
69%|██████▉ | 1089/1570 [2:56:59<44:57, 5.61s/it]
69%|██████▉ | 1090/1570 [2:57:05<44:47, 5.60s/it]
69%|██████▉ | 1091/1570 [2:57:11<44:47, 5.61s/it]
70%|██████▉ | 1092/1570 [2:57:16<44:37, 5.60s/it]
70%|██████▉ | 1093/1570 [2:57:22<44:35, 5.61s/it]
70%|██████▉ | 1094/1570 [2:57:28<44:31, 5.61s/it]
70%|██████▉ | 1095/1570 [2:57:33<44:30, 5.62s/it]
70%|██████▉ | 1096/1570 [2:57:39<44:23, 5.62s/it]
70%|██████▉ | 1097/1570 [2:57:44<44:14, 5.61s/it]
70%|██████▉ | 1098/1570 [2:57:50<44:08, 5.61s/it]
70%|███████ | 1099/1570 [2:57:56<44:05, 5.62s/it]
70%|███████ | 1100/1570 [2:58:01<43:58, 5.61s/it]
{'loss': '0.000766', 'grad_norm': '0.002609', 'learning_rate': '2.069e-05', 'epoch': '3.504'}
+
70%|███████ | 1100/1570 [2:58:01<43:58, 5.61s/it]
70%|███████ | 1101/1570 [2:58:07<43:52, 5.61s/it]
70%|███████ | 1102/1570 [2:58:12<43:47, 5.62s/it]
70%|███████ | 1103/1570 [2:58:18<43:48, 5.63s/it]
70%|███████ | 1104/1570 [2:58:24<43:47, 5.64s/it]
70%|███████ | 1105/1570 [2:58:29<43:38, 5.63s/it]
70%|███████ | 1106/1570 [2:58:35<43:26, 5.62s/it]
71%|███████ | 1107/1570 [2:58:41<43:20, 5.62s/it]
71%|███████ | 1108/1570 [2:58:46<43:15, 5.62s/it]
71%|███████ | 1109/1570 [2:58:52<43:10, 5.62s/it]
71%|███████ | 1110/1570 [2:58:57<43:01, 5.61s/it]
71%|███████ | 1111/1570 [2:59:03<42:57, 5.62s/it]
71%|███████ | 1112/1570 [2:59:09<42:48, 5.61s/it]
71%|███████ | 1113/1570 [2:59:14<42:39, 5.60s/it]
71%|███████ | 1114/1570 [2:59:20<42:31, 5.60s/it]
71%|███████ | 1115/1570 [2:59:25<42:30, 5.61s/it]
71%|███████ | 1116/1570 [2:59:31<42:22, 5.60s/it]
71%|███████ | 1117/1570 [2:59:37<42:17, 5.60s/it]
71%|███████ | 1118/1570 [2:59:42<42:14, 5.61s/it]
71%|███████▏ | 1119/1570 [2:59:48<42:06, 5.60s/it]
71%|███████▏ | 1120/1570 [2:59:53<41:55, 5.59s/it]
{'loss': '0.0004727', 'grad_norm': '0.008791', 'learning_rate': '1.91e-05', 'epoch': '3.568'}
+
71%|███████▏ | 1120/1570 [2:59:53<41:55, 5.59s/it]
71%|███████▏ | 1121/1570 [2:59:59<41:52, 5.59s/it]
71%|███████▏ | 1122/1570 [3:00:05<41:45, 5.59s/it]
72%|███████▏ | 1123/1570 [3:00:10<41:42, 5.60s/it]
72%|███████▏ | 1124/1570 [3:00:16<41:36, 5.60s/it]
72%|███████▏ | 1125/1570 [3:00:21<41:30, 5.60s/it]
72%|███████▏ | 1126/1570 [3:00:27<41:21, 5.59s/it]
72%|███████▏ | 1127/1570 [3:00:33<41:15, 5.59s/it]
72%|███████▏ | 1128/1570 [3:00:38<41:13, 5.60s/it]
72%|███████▏ | 1129/1570 [3:00:44<41:14, 5.61s/it]
72%|███████▏ | 1130/1570 [3:00:49<41:07, 5.61s/it]
72%|███████▏ | 1131/1570 [3:00:55<40:54, 5.59s/it]
72%|███████▏ | 1132/1570 [3:01:01<40:50, 5.59s/it]
72%|███████▏ | 1133/1570 [3:01:06<40:49, 5.61s/it]
72%|███████▏ | 1134/1570 [3:01:12<40:39, 5.60s/it]
72%|███████▏ | 1135/1570 [3:01:17<40:32, 5.59s/it]
72%|███████▏ | 1136/1570 [3:01:23<40:26, 5.59s/it]
72%|███████▏ | 1137/1570 [3:01:29<40:24, 5.60s/it]
72%|███████▏ | 1138/1570 [3:01:34<40:19, 5.60s/it]
73%|███████▎ | 1139/1570 [3:01:40<40:09, 5.59s/it]
73%|███████▎ | 1140/1570 [3:01:45<40:08, 5.60s/it]
{'loss': '0.0004489', 'grad_norm': '0.01038', 'learning_rate': '1.755e-05', 'epoch': '3.632'}
+
73%|███████▎ | 1140/1570 [3:01:45<40:08, 5.60s/it]
73%|███████▎ | 1141/1570 [3:01:51<40:01, 5.60s/it]
73%|███████▎ | 1142/1570 [3:01:57<39:56, 5.60s/it]
73%|███████▎ | 1143/1570 [3:02:02<39:50, 5.60s/it]
73%|███████▎ | 1144/1570 [3:02:08<39:43, 5.60s/it]
73%|███████▎ | 1145/1570 [3:02:13<39:40, 5.60s/it]
73%|███████▎ | 1146/1570 [3:02:19<39:38, 5.61s/it]
73%|███████▎ | 1147/1570 [3:02:25<39:37, 5.62s/it]
73%|███████▎ | 1148/1570 [3:02:30<39:29, 5.61s/it]
73%|███████▎ | 1149/1570 [3:02:36<39:24, 5.62s/it]
73%|███████▎ | 1150/1570 [3:02:41<39:17, 5.61s/it]
73%|███████▎ | 1151/1570 [3:02:47<39:13, 5.62s/it]
73%|███████▎ | 1152/1570 [3:02:53<39:05, 5.61s/it]
73%|███████▎ | 1153/1570 [3:02:58<39:00, 5.61s/it]
74%|███████▎ | 1154/1570 [3:03:04<38:57, 5.62s/it]
74%|███████▎ | 1155/1570 [3:03:10<38:47, 5.61s/it]
74%|███████▎ | 1156/1570 [3:03:15<38:41, 5.61s/it]
74%|███████▎ | 1157/1570 [3:03:21<38:37, 5.61s/it]
74%|███████▍ | 1158/1570 [3:03:26<38:33, 5.61s/it]
74%|███████▍ | 1159/1570 [3:03:32<38:25, 5.61s/it]
74%|███████▍ | 1160/1570 [3:03:38<38:22, 5.61s/it]
{'loss': '0.0006688', 'grad_norm': '0.005611', 'learning_rate': '1.606e-05', 'epoch': '3.696'}
+
74%|███████▍ | 1160/1570 [3:03:38<38:22, 5.61s/it]
74%|███████▍ | 1161/1570 [3:03:43<38:17, 5.62s/it]
74%|███████▍ | 1162/1570 [3:03:49<38:10, 5.61s/it]
74%|███████▍ | 1163/1570 [3:03:54<37:54, 5.59s/it]
74%|███████▍ | 1164/1570 [3:04:00<37:37, 5.56s/it]
74%|███████▍ | 1165/1570 [3:04:05<37:28, 5.55s/it]
74%|███████▍ | 1166/1570 [3:04:11<37:20, 5.54s/it]
74%|███████▍ | 1167/1570 [3:04:16<37:13, 5.54s/it]
74%|███████▍ | 1168/1570 [3:04:22<37:06, 5.54s/it]
74%|███████▍ | 1169/1570 [3:04:28<37:03, 5.54s/it]
75%|███████▍ | 1170/1570 [3:04:33<36:55, 5.54s/it]
75%|███████▍ | 1171/1570 [3:04:39<36:56, 5.55s/it]
75%|███████▍ | 1172/1570 [3:04:44<36:50, 5.55s/it]
75%|███████▍ | 1173/1570 [3:04:50<36:45, 5.56s/it]
75%|███████▍ | 1174/1570 [3:04:55<36:41, 5.56s/it]
75%|███████▍ | 1175/1570 [3:05:01<36:31, 5.55s/it]
75%|███████▍ | 1176/1570 [3:05:06<36:27, 5.55s/it]
75%|███████▍ | 1177/1570 [3:05:12<36:22, 5.55s/it]
75%|███████▌ | 1178/1570 [3:05:18<36:15, 5.55s/it]
75%|███████▌ | 1179/1570 [3:05:23<36:09, 5.55s/it]
75%|███████▌ | 1180/1570 [3:05:29<36:07, 5.56s/it]
{'loss': '0.0008441', 'grad_norm': '0.01078', 'learning_rate': '1.462e-05', 'epoch': '3.76'}
+
75%|███████▌ | 1180/1570 [3:05:29<36:07, 5.56s/it]
75%|███████▌ | 1181/1570 [3:05:34<36:07, 5.57s/it]
75%|███████▌ | 1182/1570 [3:05:40<36:04, 5.58s/it]
75%|███████▌ | 1183/1570 [3:05:45<36:03, 5.59s/it]
75%|███████▌ | 1184/1570 [3:05:51<35:58, 5.59s/it]
75%|███████▌ | 1185/1570 [3:05:57<35:50, 5.59s/it]
76%|███████▌ | 1186/1570 [3:06:02<35:44, 5.59s/it]
76%|███████▌ | 1187/1570 [3:06:08<35:40, 5.59s/it]
76%|███████▌ | 1188/1570 [3:06:13<35:31, 5.58s/it]
76%|███████▌ | 1189/1570 [3:06:19<35:20, 5.57s/it]
76%|███████▌ | 1190/1570 [3:06:24<35:17, 5.57s/it]
76%|███████▌ | 1191/1570 [3:06:30<35:14, 5.58s/it]
76%|███████▌ | 1192/1570 [3:06:36<35:02, 5.56s/it]
76%|███████▌ | 1193/1570 [3:06:41<34:58, 5.57s/it]
76%|███████▌ | 1194/1570 [3:06:47<34:55, 5.57s/it]
76%|███████▌ | 1195/1570 [3:06:52<34:43, 5.56s/it]
76%|███████▌ | 1196/1570 [3:06:58<34:37, 5.55s/it]
76%|███████▌ | 1197/1570 [3:07:03<34:38, 5.57s/it]
76%|███████▋ | 1198/1570 [3:07:09<34:36, 5.58s/it]
76%|███████▋ | 1199/1570 [3:07:15<34:29, 5.58s/it]
76%|███████▋ | 1200/1570 [3:07:20<34:22, 5.57s/it]
{'loss': '0.0008272', 'grad_norm': '0.0097', 'learning_rate': '1.324e-05', 'epoch': '3.823'}
+
76%|███████▋ | 1200/1570 [3:07:20<34:22, 5.57s/it]
76%|███████▋ | 1201/1570 [3:07:26<34:19, 5.58s/it]
77%|███████▋ | 1202/1570 [3:07:31<34:14, 5.58s/it]
77%|███████▋ | 1203/1570 [3:07:37<34:44, 5.68s/it]
77%|███████▋ | 1204/1570 [3:07:43<34:29, 5.66s/it]
77%|███████▋ | 1205/1570 [3:07:48<34:15, 5.63s/it]
77%|███████▋ | 1206/1570 [3:07:54<34:06, 5.62s/it]
77%|███████▋ | 1207/1570 [3:08:00<34:20, 5.68s/it]
77%|███████▋ | 1208/1570 [3:08:05<34:03, 5.65s/it]
77%|███████▋ | 1209/1570 [3:08:11<33:46, 5.61s/it]
77%|███████▋ | 1210/1570 [3:08:17<33:35, 5.60s/it]
77%|███████▋ | 1211/1570 [3:08:22<33:27, 5.59s/it]
77%|███████▋ | 1212/1570 [3:08:28<33:19, 5.58s/it]
77%|███████▋ | 1213/1570 [3:08:33<33:15, 5.59s/it]
77%|███████▋ | 1214/1570 [3:08:39<33:06, 5.58s/it]
77%|███████▋ | 1215/1570 [3:08:44<33:03, 5.59s/it]
77%|███████▋ | 1216/1570 [3:08:50<32:55, 5.58s/it]
78%|███████▊ | 1217/1570 [3:08:55<32:42, 5.56s/it]
78%|███████▊ | 1218/1570 [3:09:01<32:32, 5.55s/it]
78%|███████▊ | 1219/1570 [3:09:07<32:34, 5.57s/it]
78%|███████▊ | 1220/1570 [3:09:12<32:31, 5.57s/it]
{'loss': '0.0003761', 'grad_norm': '0.01817', 'learning_rate': '1.192e-05', 'epoch': '3.887'}
+
78%|███████▊ | 1220/1570 [3:09:12<32:31, 5.57s/it]
78%|███████▊ | 1221/1570 [3:09:18<32:28, 5.58s/it]
78%|███████▊ | 1222/1570 [3:09:23<32:27, 5.60s/it]
78%|███████▊ | 1223/1570 [3:09:29<32:24, 5.60s/it]
78%|███████▊ | 1224/1570 [3:09:35<32:21, 5.61s/it]
78%|███████▊ | 1225/1570 [3:09:40<32:20, 5.62s/it]
78%|███████▊ | 1226/1570 [3:09:46<32:18, 5.63s/it]
78%|███████▊ | 1227/1570 [3:09:52<32:04, 5.61s/it]
78%|███████▊ | 1228/1570 [3:09:57<31:52, 5.59s/it]
78%|███████▊ | 1229/1570 [3:10:03<31:43, 5.58s/it]
78%|███████▊ | 1230/1570 [3:10:08<31:39, 5.59s/it]
78%|███████▊ | 1231/1570 [3:10:14<31:33, 5.59s/it]
78%|███████▊ | 1232/1570 [3:10:19<31:26, 5.58s/it]
79%|███████▊ | 1233/1570 [3:10:25<31:19, 5.58s/it]
79%|███████▊ | 1234/1570 [3:10:31<31:12, 5.57s/it]
79%|███████▊ | 1235/1570 [3:10:36<31:05, 5.57s/it]
79%|███████▊ | 1236/1570 [3:10:42<30:59, 5.57s/it]
79%|███████▉ | 1237/1570 [3:10:47<30:55, 5.57s/it]
79%|███████▉ | 1238/1570 [3:10:53<30:47, 5.57s/it]
79%|███████▉ | 1239/1570 [3:10:58<30:41, 5.56s/it]
79%|███████▉ | 1240/1570 [3:11:04<30:34, 5.56s/it]
{'loss': '0.0003943', 'grad_norm': '0.01076', 'learning_rate': '1.066e-05', 'epoch': '3.951'}
+
79%|███████▉ | 1240/1570 [3:11:04<30:34, 5.56s/it]
79%|███████▉ | 1241/1570 [3:11:09<30:28, 5.56s/it]
79%|███████▉ | 1242/1570 [3:11:15<30:22, 5.56s/it]
79%|███████▉ | 1243/1570 [3:11:21<30:12, 5.54s/it]
79%|███████▉ | 1244/1570 [3:11:26<30:09, 5.55s/it]
79%|███████▉ | 1245/1570 [3:11:32<30:02, 5.55s/it]
79%|███████▉ | 1246/1570 [3:11:37<30:02, 5.56s/it]
79%|███████▉ | 1247/1570 [3:11:43<30:11, 5.61s/it]
79%|███████▉ | 1248/1570 [3:11:49<30:36, 5.70s/it]
80%|███████▉ | 1249/1570 [3:11:56<32:14, 6.03s/it]
80%|███████▉ | 1250/1570 [3:12:01<31:42, 5.95s/it]
80%|███████▉ | 1251/1570 [3:12:07<31:09, 5.86s/it]
80%|███████▉ | 1252/1570 [3:12:13<30:40, 5.79s/it]
80%|███████▉ | 1253/1570 [3:12:18<30:18, 5.74s/it]
80%|███████▉ | 1254/1570 [3:12:24<29:59, 5.70s/it]
80%|███████▉ | 1255/1570 [3:12:29<29:40, 5.65s/it]
80%|████████ | 1256/1570 [3:12:31<23:44, 4.54s/it][INFO|trainer.py:2668] 2026-08-22 06:28:41,069 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 06:28:41,069 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 06:28:41,069 >> Batch size = 8
+
+
0%| | 0/214 [00:00> Saving model checkpoint to results/qwen3/teacher_lora/checkpoint-1256
+[INFO|configuration_utils.py:780] 2026-08-22 06:53:37,753 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 06:53:37,755 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 06:53:37,846 >> chat template saved in results/qwen3/teacher_lora/checkpoint-1256/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 06:53:37,846 >> tokenizer config file saved in results/qwen3/teacher_lora/checkpoint-1256/tokenizer_config.json
+
80%|████████ | 1257/1570 [3:37:36<39:31:38, 454.63s/it]
80%|████████ | 1258/1570 [3:37:42<27:43:36, 319.92s/it]
80%|████████ | 1259/1570 [3:37:47<19:29:33, 225.64s/it]
80%|████████ | 1260/1570 [3:37:53<13:44:46, 159.63s/it]
{'loss': '0.0008496', 'grad_norm': '0.02127', 'learning_rate': '9.464e-06', 'epoch': '4.013'}
+
80%|████████ | 1260/1570 [3:37:53<13:44:46, 159.63s/it]
80%|████████ | 1261/1570 [3:37:59<9:44:08, 113.43s/it]
80%|████████ | 1262/1570 [3:38:04<6:56:12, 81.08s/it]
80%|████████ | 1263/1570 [3:38:10<4:58:58, 58.43s/it]
81%|████████ | 1264/1570 [3:38:16<3:37:12, 42.59s/it]
81%|████████ | 1265/1570 [3:38:21<2:40:09, 31.51s/it]
81%|████████ | 1266/1570 [3:38:27<2:00:15, 23.73s/it]
81%|████████ | 1267/1570 [3:38:32<1:32:22, 18.29s/it]
81%|████████ | 1268/1570 [3:38:38<1:12:51, 14.48s/it]
81%|████████ | 1269/1570 [3:38:44<59:11, 11.80s/it]
81%|████████ | 1270/1570 [3:38:49<49:41, 9.94s/it]
81%|████████ | 1271/1570 [3:38:55<42:52, 8.60s/it]
81%|████████ | 1272/1570 [3:39:00<38:06, 7.67s/it]
81%|████████ | 1273/1570 [3:39:06<34:44, 7.02s/it]
81%|████████ | 1274/1570 [3:39:11<32:27, 6.58s/it]
81%|████████ | 1275/1570 [3:39:17<30:48, 6.26s/it]
81%|████████▏ | 1276/1570 [3:39:22<29:37, 6.05s/it]
81%|████████▏ | 1277/1570 [3:39:28<28:44, 5.88s/it]
81%|████████▏ | 1278/1570 [3:39:33<28:09, 5.79s/it]
81%|████████▏ | 1279/1570 [3:39:39<27:45, 5.72s/it]
82%|████████▏ | 1280/1570 [3:39:44<27:30, 5.69s/it]
{'loss': '0.0002365', 'grad_norm': '0.04594', 'learning_rate': '8.332e-06', 'epoch': '4.077'}
+
82%|████████▏ | 1280/1570 [3:39:44<27:30, 5.69s/it]
82%|████████▏ | 1281/1570 [3:39:50<27:16, 5.66s/it]
82%|████████▏ | 1282/1570 [3:39:56<27:02, 5.63s/it]
82%|████████▏ | 1283/1570 [3:40:01<26:51, 5.62s/it]
82%|████████▏ | 1284/1570 [3:40:07<26:39, 5.59s/it]
82%|████████▏ | 1285/1570 [3:40:12<26:32, 5.59s/it]
82%|████████▏ | 1286/1570 [3:40:18<26:25, 5.58s/it]
82%|████████▏ | 1287/1570 [3:40:23<26:19, 5.58s/it]
82%|████████▏ | 1288/1570 [3:40:29<26:15, 5.59s/it]
82%|████████▏ | 1289/1570 [3:40:35<26:08, 5.58s/it]
82%|████████▏ | 1290/1570 [3:40:40<26:00, 5.57s/it]
82%|████████▏ | 1291/1570 [3:40:46<25:55, 5.57s/it]
82%|████████▏ | 1292/1570 [3:40:51<25:50, 5.58s/it]
82%|████████▏ | 1293/1570 [3:40:57<25:46, 5.58s/it]
82%|████████▏ | 1294/1570 [3:41:03<25:41, 5.59s/it]
82%|████████▏ | 1295/1570 [3:41:08<25:35, 5.58s/it]
83%|████████▎ | 1296/1570 [3:41:14<25:30, 5.58s/it]
83%|████████▎ | 1297/1570 [3:41:19<25:24, 5.58s/it]
83%|████████▎ | 1298/1570 [3:41:25<25:19, 5.58s/it]
83%|████████▎ | 1299/1570 [3:41:30<25:10, 5.57s/it]
83%|████████▎ | 1300/1570 [3:41:36<25:05, 5.58s/it]
{'loss': '0.0002794', 'grad_norm': '0.000374', 'learning_rate': '7.266e-06', 'epoch': '4.14'}
+
83%|████████▎ | 1300/1570 [3:41:36<25:05, 5.58s/it]
83%|████████▎ | 1301/1570 [3:41:42<24:59, 5.58s/it]
83%|████████▎ | 1302/1570 [3:41:47<24:54, 5.58s/it]
83%|████████▎ | 1303/1570 [3:41:53<24:50, 5.58s/it]
83%|████████▎ | 1304/1570 [3:41:58<24:46, 5.59s/it]
83%|████████▎ | 1305/1570 [3:42:04<24:40, 5.59s/it]
83%|████████▎ | 1306/1570 [3:42:10<24:35, 5.59s/it]
83%|████████▎ | 1307/1570 [3:42:15<24:31, 5.59s/it]
83%|████████▎ | 1308/1570 [3:42:21<24:27, 5.60s/it]
83%|████████▎ | 1309/1570 [3:42:26<24:16, 5.58s/it]
83%|████████▎ | 1310/1570 [3:42:32<24:09, 5.57s/it]
84%|████████▎ | 1311/1570 [3:42:37<24:00, 5.56s/it]
84%|████████▎ | 1312/1570 [3:42:43<23:57, 5.57s/it]
84%|████████▎ | 1313/1570 [3:42:49<23:50, 5.57s/it]
84%|████████▎ | 1314/1570 [3:42:54<23:49, 5.58s/it]
84%|████████▍ | 1315/1570 [3:43:00<23:44, 5.59s/it]
84%|████████▍ | 1316/1570 [3:43:05<23:42, 5.60s/it]
84%|████████▍ | 1317/1570 [3:43:11<23:35, 5.60s/it]
84%|████████▍ | 1318/1570 [3:43:17<23:27, 5.59s/it]
84%|████████▍ | 1319/1570 [3:43:22<23:21, 5.58s/it]
84%|████████▍ | 1320/1570 [3:43:28<23:14, 5.58s/it]
{'loss': '0.0003295', 'grad_norm': '0.002863', 'learning_rate': '6.269e-06', 'epoch': '4.204'}
+
84%|████████▍ | 1320/1570 [3:43:28<23:14, 5.58s/it]
84%|████████▍ | 1321/1570 [3:43:33<23:10, 5.58s/it]
84%|████████▍ | 1322/1570 [3:43:39<23:05, 5.59s/it]
84%|████████▍ | 1323/1570 [3:43:44<22:58, 5.58s/it]
84%|████████▍ | 1324/1570 [3:43:50<22:52, 5.58s/it]
84%|████████▍ | 1325/1570 [3:43:55<22:41, 5.56s/it]
84%|████████▍ | 1326/1570 [3:44:01<22:31, 5.54s/it]
85%|████████▍ | 1327/1570 [3:44:06<22:20, 5.52s/it]
85%|████████▍ | 1328/1570 [3:44:12<22:16, 5.52s/it]
85%|████████▍ | 1329/1570 [3:44:18<22:11, 5.52s/it]
85%|████████▍ | 1330/1570 [3:44:23<22:08, 5.54s/it]
85%|████████▍ | 1331/1570 [3:44:29<22:03, 5.54s/it]
85%|████████▍ | 1332/1570 [3:44:34<21:59, 5.54s/it]
85%|████████▍ | 1333/1570 [3:44:40<21:52, 5.54s/it]
85%|████████▍ | 1334/1570 [3:44:45<21:43, 5.52s/it]
85%|████████▌ | 1335/1570 [3:44:51<21:39, 5.53s/it]
85%|████████▌ | 1336/1570 [3:44:56<21:35, 5.54s/it]
85%|████████▌ | 1337/1570 [3:45:02<21:32, 5.55s/it]
85%|████████▌ | 1338/1570 [3:45:07<21:29, 5.56s/it]
85%|████████▌ | 1339/1570 [3:45:13<21:25, 5.56s/it]
85%|████████▌ | 1340/1570 [3:45:19<21:18, 5.56s/it]
{'loss': '0.0003575', 'grad_norm': '0.03912', 'learning_rate': '5.342e-06', 'epoch': '4.268'}
+
85%|████████▌ | 1340/1570 [3:45:19<21:18, 5.56s/it]
85%|████████▌ | 1341/1570 [3:45:24<21:12, 5.56s/it]
85%|████████▌ | 1342/1570 [3:45:30<21:08, 5.57s/it]
86%|████████▌ | 1343/1570 [3:45:35<21:04, 5.57s/it]
86%|████████▌ | 1344/1570 [3:45:41<20:56, 5.56s/it]
86%|████████▌ | 1345/1570 [3:45:46<20:49, 5.55s/it]
86%|████████▌ | 1346/1570 [3:45:52<20:43, 5.55s/it]
86%|████████▌ | 1347/1570 [3:45:57<20:39, 5.56s/it]
86%|████████▌ | 1348/1570 [3:46:03<20:34, 5.56s/it]
86%|████████▌ | 1349/1570 [3:46:09<20:28, 5.56s/it]
86%|████████▌ | 1350/1570 [3:46:14<20:19, 5.54s/it]
86%|████████▌ | 1351/1570 [3:46:20<20:12, 5.54s/it]
86%|████████▌ | 1352/1570 [3:46:25<20:11, 5.56s/it]
86%|████████▌ | 1353/1570 [3:46:31<20:06, 5.56s/it]
86%|████████▌ | 1354/1570 [3:46:36<19:59, 5.56s/it]
86%|████████▋ | 1355/1570 [3:46:42<19:55, 5.56s/it]
86%|████████▋ | 1356/1570 [3:46:47<19:49, 5.56s/it]
86%|████████▋ | 1357/1570 [3:46:53<19:45, 5.57s/it]
86%|████████▋ | 1358/1570 [3:46:59<19:40, 5.57s/it]
87%|████████▋ | 1359/1570 [3:47:04<19:37, 5.58s/it]
87%|████████▋ | 1360/1570 [3:47:10<19:30, 5.57s/it]
{'loss': '0.0003375', 'grad_norm': '0.001067', 'learning_rate': '4.486e-06', 'epoch': '4.332'}
+
87%|████████▋ | 1360/1570 [3:47:10<19:30, 5.57s/it]
87%|████████▋ | 1361/1570 [3:47:15<19:24, 5.57s/it]
87%|████████▋ | 1362/1570 [3:47:21<19:20, 5.58s/it]
87%|████████▋ | 1363/1570 [3:47:27<19:13, 5.57s/it]
87%|████████▋ | 1364/1570 [3:47:32<19:09, 5.58s/it]
87%|████████▋ | 1365/1570 [3:47:38<19:03, 5.58s/it]
87%|████████▋ | 1366/1570 [3:47:43<18:58, 5.58s/it]
87%|████████▋ | 1367/1570 [3:47:49<18:53, 5.59s/it]
87%|████████▋ | 1368/1570 [3:47:54<18:48, 5.58s/it]
87%|████████▋ | 1369/1570 [3:48:00<18:42, 5.58s/it]
87%|████████▋ | 1370/1570 [3:48:06<18:36, 5.58s/it]
87%|████████▋ | 1371/1570 [3:48:11<18:31, 5.58s/it]
87%|████████▋ | 1372/1570 [3:48:17<18:26, 5.59s/it]
87%|████████▋ | 1373/1570 [3:48:22<18:21, 5.59s/it]
88%|████████▊ | 1374/1570 [3:48:28<18:17, 5.60s/it]
88%|████████▊ | 1375/1570 [3:48:34<18:09, 5.59s/it]
88%|████████▊ | 1376/1570 [3:48:39<18:01, 5.58s/it]
88%|████████▊ | 1377/1570 [3:48:45<17:57, 5.58s/it]
88%|████████▊ | 1378/1570 [3:48:50<17:49, 5.57s/it]
88%|████████▊ | 1379/1570 [3:48:56<17:42, 5.56s/it]
88%|████████▊ | 1380/1570 [3:49:01<17:35, 5.56s/it]
{'loss': '0.0008517', 'grad_norm': '0.002243', 'learning_rate': '3.704e-06', 'epoch': '4.396'}
+
88%|████████▊ | 1380/1570 [3:49:01<17:35, 5.56s/it]
88%|████████▊ | 1381/1570 [3:49:07<17:27, 5.54s/it]
88%|████████▊ | 1382/1570 [3:49:12<17:22, 5.54s/it]
88%|████████▊ | 1383/1570 [3:49:18<17:13, 5.53s/it]
88%|████████▊ | 1384/1570 [3:49:23<17:10, 5.54s/it]
88%|████████▊ | 1385/1570 [3:49:29<17:04, 5.54s/it]
88%|████████▊ | 1386/1570 [3:49:35<16:59, 5.54s/it]
88%|████████▊ | 1387/1570 [3:49:40<16:54, 5.54s/it]
88%|████████▊ | 1388/1570 [3:49:46<16:49, 5.55s/it]
88%|████████▊ | 1389/1570 [3:49:51<16:44, 5.55s/it]
89%|████████▊ | 1390/1570 [3:49:57<16:41, 5.56s/it]
89%|████████▊ | 1391/1570 [3:50:02<16:36, 5.57s/it]
89%|████████▊ | 1392/1570 [3:50:08<16:31, 5.57s/it]
89%|████████▊ | 1393/1570 [3:50:14<16:26, 5.57s/it]
89%|████████▉ | 1394/1570 [3:50:19<16:22, 5.58s/it]
89%|████████▉ | 1395/1570 [3:50:25<16:16, 5.58s/it]
89%|████████▉ | 1396/1570 [3:50:30<16:11, 5.58s/it]
89%|████████▉ | 1397/1570 [3:50:36<16:06, 5.58s/it]
89%|████████▉ | 1398/1570 [3:50:42<16:01, 5.59s/it]
89%|████████▉ | 1399/1570 [3:50:47<15:55, 5.59s/it]
89%|████████▉ | 1400/1570 [3:50:53<15:49, 5.59s/it]
{'loss': '0.0009319', 'grad_norm': '0.01994', 'learning_rate': '2.996e-06', 'epoch': '4.46'}
+
89%|████████▉ | 1400/1570 [3:50:53<15:49, 5.59s/it]
89%|████████▉ | 1401/1570 [3:50:58<15:45, 5.60s/it]
89%|████████▉ | 1402/1570 [3:51:04<15:40, 5.60s/it]
89%|████████▉ | 1403/1570 [3:51:09<15:34, 5.60s/it]
89%|████████▉ | 1404/1570 [3:51:15<15:29, 5.60s/it]
89%|████████▉ | 1405/1570 [3:51:21<15:22, 5.59s/it]
90%|████████▉ | 1406/1570 [3:51:26<15:16, 5.59s/it]
90%|████████▉ | 1407/1570 [3:51:32<15:09, 5.58s/it]
90%|████████▉ | 1408/1570 [3:51:37<15:04, 5.58s/it]
90%|████████▉ | 1409/1570 [3:51:43<15:00, 5.59s/it]
90%|████████▉ | 1410/1570 [3:51:49<14:55, 5.60s/it]
90%|████████▉ | 1411/1570 [3:51:54<14:50, 5.60s/it]
90%|████████▉ | 1412/1570 [3:52:00<14:44, 5.60s/it]
90%|█████████ | 1413/1570 [3:52:05<14:38, 5.59s/it]
90%|█████████ | 1414/1570 [3:52:11<14:31, 5.59s/it]
90%|█████████ | 1415/1570 [3:52:17<14:26, 5.59s/it]
90%|█████████ | 1416/1570 [3:52:22<14:21, 5.59s/it]
90%|█████████ | 1417/1570 [3:52:28<14:13, 5.58s/it]
90%|█████████ | 1418/1570 [3:52:33<14:08, 5.59s/it]
90%|█████████ | 1419/1570 [3:52:39<14:02, 5.58s/it]
90%|█████████ | 1420/1570 [3:52:44<13:55, 5.57s/it]
{'loss': '0.0003644', 'grad_norm': '0.02732', 'learning_rate': '2.363e-06', 'epoch': '4.523'}
+
90%|█████████ | 1420/1570 [3:52:44<13:55, 5.57s/it]
91%|█████████ | 1421/1570 [3:52:50<13:51, 5.58s/it]
91%|█████████ | 1422/1570 [3:52:56<13:45, 5.58s/it]
91%|█████████ | 1423/1570 [3:53:01<13:38, 5.57s/it]
91%|█████████ | 1424/1570 [3:53:07<13:34, 5.58s/it]
91%|█████████ | 1425/1570 [3:53:12<13:26, 5.56s/it]
91%|█████████ | 1426/1570 [3:53:18<13:21, 5.56s/it]
91%|█████████ | 1427/1570 [3:53:23<13:16, 5.57s/it]
91%|█████████ | 1428/1570 [3:53:29<13:10, 5.56s/it]
91%|█████████ | 1429/1570 [3:53:34<13:02, 5.55s/it]
91%|█████████ | 1430/1570 [3:53:40<12:55, 5.54s/it]
91%|█████████ | 1431/1570 [3:53:46<12:50, 5.54s/it]
91%|█████████ | 1432/1570 [3:53:51<12:44, 5.54s/it]
91%|█████████▏| 1433/1570 [3:53:57<12:38, 5.54s/it]
91%|█████████▏| 1434/1570 [3:54:02<12:35, 5.56s/it]
91%|█████████▏| 1435/1570 [3:54:08<12:30, 5.56s/it]
91%|█████████▏| 1436/1570 [3:54:13<12:25, 5.56s/it]
92%|█████████▏| 1437/1570 [3:54:19<12:20, 5.57s/it]
92%|█████████▏| 1438/1570 [3:54:25<12:14, 5.57s/it]
92%|█████████▏| 1439/1570 [3:54:30<12:09, 5.57s/it]
92%|█████████▏| 1440/1570 [3:54:36<12:03, 5.57s/it]
{'loss': '0.0002116', 'grad_norm': '0.007133', 'learning_rate': '1.806e-06', 'epoch': '4.587'}
+
92%|█████████▏| 1440/1570 [3:54:36<12:03, 5.57s/it]
92%|█████████▏| 1441/1570 [3:54:41<11:58, 5.57s/it]
92%|█████████▏| 1442/1570 [3:54:47<11:54, 5.58s/it]
92%|█████████▏| 1443/1570 [3:54:52<11:48, 5.58s/it]
92%|█████████▏| 1444/1570 [3:54:58<11:42, 5.57s/it]
92%|█████████▏| 1445/1570 [3:55:04<11:36, 5.57s/it]
92%|█████████▏| 1446/1570 [3:55:09<11:29, 5.56s/it]
92%|█████████▏| 1447/1570 [3:55:15<11:24, 5.57s/it]
92%|█████████▏| 1448/1570 [3:55:20<11:19, 5.57s/it]
92%|█████████▏| 1449/1570 [3:55:26<11:14, 5.57s/it]
92%|█████████▏| 1450/1570 [3:55:32<11:25, 5.72s/it]
92%|█████████▏| 1451/1570 [3:55:37<11:15, 5.68s/it]
92%|█████████▏| 1452/1570 [3:55:43<11:06, 5.65s/it]
93%|█████████▎| 1453/1570 [3:55:49<11:10, 5.73s/it]
93%|█████████▎| 1454/1570 [3:55:55<10:59, 5.69s/it]
93%|█████████▎| 1455/1570 [3:56:00<10:50, 5.65s/it]
93%|█████████▎| 1456/1570 [3:56:06<10:43, 5.65s/it]
93%|█████████▎| 1457/1570 [3:56:11<10:37, 5.64s/it]
93%|█████████▎| 1458/1570 [3:56:17<10:28, 5.61s/it]
93%|█████████▎| 1459/1570 [3:56:23<10:22, 5.61s/it]
93%|█████████▎| 1460/1570 [3:56:28<10:16, 5.61s/it]
{'loss': '0.000227', 'grad_norm': '0.02018', 'learning_rate': '1.327e-06', 'epoch': '4.651'}
+
93%|█████████▎| 1460/1570 [3:56:28<10:16, 5.61s/it]
93%|█████████▎| 1461/1570 [3:56:34<10:11, 5.61s/it]
93%|█████████▎| 1462/1570 [3:56:39<10:06, 5.61s/it]
93%|█████████▎| 1463/1570 [3:56:45<10:00, 5.61s/it]
93%|█████████▎| 1464/1570 [3:56:51<09:54, 5.61s/it]
93%|█████████▎| 1465/1570 [3:56:56<09:50, 5.62s/it]
93%|█████████▎| 1466/1570 [3:57:02<09:44, 5.62s/it]
93%|█████████▎| 1467/1570 [3:57:07<09:38, 5.62s/it]
94%|█████████▎| 1468/1570 [3:57:13<09:32, 5.62s/it]
94%|█████████▎| 1469/1570 [3:57:19<09:26, 5.61s/it]
94%|█████████▎| 1470/1570 [3:57:24<09:20, 5.61s/it]
94%|█████████▎| 1471/1570 [3:57:30<09:15, 5.61s/it]
94%|█████████▍| 1472/1570 [3:57:36<09:10, 5.62s/it]
94%|█████████▍| 1473/1570 [3:57:41<09:04, 5.61s/it]
94%|█████████▍| 1474/1570 [3:57:47<08:59, 5.61s/it]
94%|█████████▍| 1475/1570 [3:57:52<08:53, 5.62s/it]
94%|█████████▍| 1476/1570 [3:57:58<08:47, 5.62s/it]
94%|█████████▍| 1477/1570 [3:58:04<08:41, 5.61s/it]
94%|█████████▍| 1478/1570 [3:58:09<08:35, 5.61s/it]
94%|█████████▍| 1479/1570 [3:58:15<08:30, 5.61s/it]
94%|█████████▍| 1480/1570 [3:58:20<08:24, 5.60s/it]
{'loss': '0.0002237', 'grad_norm': '0.0143', 'learning_rate': '9.258e-07', 'epoch': '4.715'}
+
94%|█████████▍| 1480/1570 [3:58:20<08:24, 5.60s/it]
94%|█████████▍| 1481/1570 [3:58:26<08:20, 5.62s/it]
94%|█████████▍| 1482/1570 [3:58:32<08:15, 5.63s/it]
94%|█████████▍| 1483/1570 [3:58:37<08:09, 5.63s/it]
95%|█████████▍| 1484/1570 [3:58:43<08:04, 5.64s/it]
95%|█████████▍| 1485/1570 [3:58:49<07:59, 5.64s/it]
95%|█████████▍| 1486/1570 [3:58:54<07:52, 5.62s/it]
95%|█████████▍| 1487/1570 [3:59:00<07:45, 5.61s/it]
95%|█████████▍| 1488/1570 [3:59:05<07:38, 5.59s/it]
95%|█████████▍| 1489/1570 [3:59:11<07:32, 5.59s/it]
95%|█████████▍| 1490/1570 [3:59:16<07:27, 5.59s/it]
95%|█████████▍| 1491/1570 [3:59:22<07:20, 5.58s/it]
95%|█████████▌| 1492/1570 [3:59:28<07:14, 5.58s/it]
95%|█████████▌| 1493/1570 [3:59:33<07:09, 5.58s/it]
95%|█████████▌| 1494/1570 [3:59:39<07:03, 5.57s/it]
95%|█████████▌| 1495/1570 [3:59:44<06:58, 5.58s/it]
95%|█████████▌| 1496/1570 [3:59:50<06:52, 5.58s/it]
95%|█████████▌| 1497/1570 [3:59:55<06:46, 5.57s/it]
95%|█████████▌| 1498/1570 [4:00:01<06:41, 5.57s/it]
95%|█████████▌| 1499/1570 [4:00:07<06:34, 5.56s/it]
96%|█████████▌| 1500/1570 [4:00:12<06:29, 5.56s/it]
{'loss': '0.0004318', 'grad_norm': '0.00789', 'learning_rate': '6.033e-07', 'epoch': '4.779'}
+
96%|█████████▌| 1500/1570 [4:00:12<06:29, 5.56s/it]
96%|█████████▌| 1501/1570 [4:00:18<06:23, 5.56s/it]
96%|█████████▌| 1502/1570 [4:00:23<06:19, 5.58s/it]
96%|█████████▌| 1503/1570 [4:00:29<06:13, 5.58s/it]
96%|█████████▌| 1504/1570 [4:00:35<06:08, 5.58s/it]
96%|█████████▌| 1505/1570 [4:00:40<06:02, 5.58s/it]
96%|█████████▌| 1506/1570 [4:00:46<05:56, 5.58s/it]
96%|█████████▌| 1507/1570 [4:00:51<05:51, 5.58s/it]
96%|█████████▌| 1508/1570 [4:00:57<05:46, 5.59s/it]
96%|█████████▌| 1509/1570 [4:01:02<05:40, 5.58s/it]
96%|█████████▌| 1510/1570 [4:01:08<05:35, 5.59s/it]
96%|█████████▌| 1511/1570 [4:01:14<05:29, 5.58s/it]
96%|█████████▋| 1512/1570 [4:01:19<05:23, 5.58s/it]
96%|█████████▋| 1513/1570 [4:01:25<05:18, 5.58s/it]
96%|█████████▋| 1514/1570 [4:01:30<05:12, 5.58s/it]
96%|█████████▋| 1515/1570 [4:01:36<05:05, 5.56s/it]
97%|█████████▋| 1516/1570 [4:01:41<05:00, 5.57s/it]
97%|█████████▋| 1517/1570 [4:01:47<04:56, 5.59s/it]
97%|█████████▋| 1518/1570 [4:01:53<04:50, 5.58s/it]
97%|█████████▋| 1519/1570 [4:01:58<04:46, 5.61s/it]
97%|█████████▋| 1520/1570 [4:02:04<04:40, 5.61s/it]
{'loss': '0.0002423', 'grad_norm': '0.02257', 'learning_rate': '3.599e-07', 'epoch': '4.843'}
+
97%|█████████▋| 1520/1570 [4:02:04<04:40, 5.61s/it]
97%|█████████▋| 1521/1570 [4:02:09<04:33, 5.59s/it]
97%|█████████▋| 1522/1570 [4:02:15<04:28, 5.59s/it]
97%|█████████▋| 1523/1570 [4:02:21<04:22, 5.59s/it]
97%|█████████▋| 1524/1570 [4:02:26<04:17, 5.60s/it]
97%|█████████▋| 1525/1570 [4:02:32<04:11, 5.59s/it]
97%|█████████▋| 1526/1570 [4:02:37<04:05, 5.59s/it]
97%|█████████▋| 1527/1570 [4:02:43<04:00, 5.60s/it]
97%|█████████▋| 1528/1570 [4:02:49<03:55, 5.60s/it]
97%|█████████▋| 1529/1570 [4:02:54<03:49, 5.60s/it]
97%|█████████▋| 1530/1570 [4:03:00<03:43, 5.59s/it]
98%|█████████▊| 1531/1570 [4:03:05<03:38, 5.59s/it]
98%|█████████▊| 1532/1570 [4:03:11<03:32, 5.60s/it]
98%|█████████▊| 1533/1570 [4:03:17<03:27, 5.60s/it]
98%|█████████▊| 1534/1570 [4:03:22<03:21, 5.59s/it]
98%|█████████▊| 1535/1570 [4:03:28<03:15, 5.59s/it]
98%|█████████▊| 1536/1570 [4:03:33<03:09, 5.58s/it]
98%|█████████▊| 1537/1570 [4:03:39<03:04, 5.59s/it]
98%|█████████▊| 1538/1570 [4:03:45<02:58, 5.59s/it]
98%|█████████▊| 1539/1570 [4:03:50<02:53, 5.59s/it]
98%|█████████▊| 1540/1570 [4:03:56<02:47, 5.57s/it]
{'loss': '0.0002711', 'grad_norm': '0.02837', 'learning_rate': '1.961e-07', 'epoch': '4.906'}
+
98%|█████████▊| 1540/1570 [4:03:56<02:47, 5.57s/it]
98%|█████████▊| 1541/1570 [4:04:01<02:41, 5.56s/it]
98%|█████████▊| 1542/1570 [4:04:07<02:35, 5.56s/it]
98%|█████████▊| 1543/1570 [4:04:12<02:29, 5.54s/it]
98%|█████████▊| 1544/1570 [4:04:18<02:23, 5.54s/it]
98%|█████████▊| 1545/1570 [4:04:23<02:18, 5.55s/it]
98%|█████████▊| 1546/1570 [4:04:29<02:13, 5.56s/it]
99%|█████████▊| 1547/1570 [4:04:35<02:07, 5.56s/it]
99%|█████████▊| 1548/1570 [4:04:40<02:02, 5.56s/it]
99%|█████████▊| 1549/1570 [4:04:46<01:56, 5.55s/it]
99%|█████████▊| 1550/1570 [4:04:51<01:51, 5.55s/it]
99%|█████████▉| 1551/1570 [4:04:57<01:45, 5.56s/it]
99%|█████████▉| 1552/1570 [4:05:02<01:40, 5.56s/it]
99%|█████████▉| 1553/1570 [4:05:08<01:34, 5.57s/it]
99%|█████████▉| 1554/1570 [4:05:13<01:29, 5.57s/it]
99%|█████████▉| 1555/1570 [4:05:19<01:23, 5.57s/it]
99%|█████████▉| 1556/1570 [4:05:25<01:17, 5.56s/it]
99%|█████████▉| 1557/1570 [4:05:30<01:12, 5.58s/it]
99%|█████████▉| 1558/1570 [4:05:36<01:07, 5.59s/it]
99%|█████████▉| 1559/1570 [4:05:41<01:01, 5.59s/it]
99%|█████████▉| 1560/1570 [4:05:47<00:55, 5.59s/it]
{'loss': '0.0003345', 'grad_norm': '0.01945', 'learning_rate': '1.121e-07', 'epoch': '4.97'}
+
99%|█████████▉| 1560/1570 [4:05:47<00:55, 5.59s/it]
99%|█████████▉| 1561/1570 [4:05:53<00:50, 5.58s/it]
99%|█████████▉| 1562/1570 [4:05:58<00:44, 5.58s/it]
100%|█████████▉| 1563/1570 [4:06:04<00:38, 5.57s/it]
100%|█████████▉| 1564/1570 [4:06:09<00:33, 5.56s/it]
100%|█████████▉| 1565/1570 [4:06:15<00:27, 5.57s/it]
100%|█████████▉| 1566/1570 [4:06:20<00:22, 5.58s/it]
100%|█████████▉| 1567/1570 [4:06:26<00:16, 5.59s/it]
100%|█████████▉| 1568/1570 [4:06:32<00:11, 5.58s/it]
100%|█████████▉| 1569/1570 [4:06:37<00:05, 5.59s/it]
100%|██████████| 1570/1570 [4:06:39<00:00, 4.49s/it][INFO|trainer.py:2668] 2026-08-22 07:22:48,781 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 07:22:48,781 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 07:22:48,781 >> Batch size = 8
+
+
0%| | 0/214 [00:00> Saving model checkpoint to results/qwen3/teacher_lora/checkpoint-1570
+[INFO|configuration_utils.py:780] 2026-08-22 07:47:47,507 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 07:47:47,508 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 07:47:47,596 >> chat template saved in results/qwen3/teacher_lora/checkpoint-1570/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 07:47:47,596 >> tokenizer config file saved in results/qwen3/teacher_lora/checkpoint-1570/tokenizer_config.json
+[INFO|trainer.py:1823] 2026-08-22 07:47:50,053 >>
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
{'train_runtime': '1.63e+04', 'train_samples_per_second': '3.076', 'train_steps_per_second': '0.096', 'train_loss': '0.009031', 'epoch': '5'}
+
100%|██████████| 1570/1570 [4:31:40<00:00, 4.49s/it]
100%|██████████| 1570/1570 [4:31:40<00:00, 10.38s/it]
+[INFO|trainer.py:3815] 2026-08-22 07:47:52,061 >> Saving model checkpoint to results/qwen3/teacher_lora
+[INFO|configuration_utils.py:780] 2026-08-22 07:47:52,555 >> loading configuration file config.json from cache at /home/hungpv/.cache/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/config.json
+[INFO|configuration_utils.py:856] 2026-08-22 07:47:52,558 >> Model config Qwen3Config {
+ "architectures": [
+ "Qwen3ForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "dtype": "bfloat16",
+ "eos_token_id": 151645,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 2560,
+ "initializer_range": 0.02,
+ "intermediate_size": 9728,
+ "layer_types": [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ "full_attention"
+ ],
+ "max_position_embeddings": 262144,
+ "max_window_layers": 36,
+ "model_type": "qwen3",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "rope_theta": 5000000,
+ "rope_type": "default"
+ },
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "transformers_version": "5.8.0",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+}
+
+[INFO|tokenization_utils_base.py:3281] 2026-08-22 07:47:52,644 >> chat template saved in results/qwen3/teacher_lora/chat_template.jinja
+[INFO|tokenization_utils_base.py:2114] 2026-08-22 07:47:52,644 >> tokenizer config file saved in results/qwen3/teacher_lora/tokenizer_config.json
+***** train metrics *****
+ epoch = 5.0
+ total_flos = 370781970GF
+ train_loss = 0.009
+ train_runtime = 4:31:40.88
+ train_samples_per_second = 3.076
+ train_steps_per_second = 0.096
+[INFO|trainer.py:2668] 2026-08-22 07:47:52,846 >>
+***** Running Evaluation *****
+[INFO|trainer.py:2670] 2026-08-22 07:47:52,846 >> Num examples = 3414
+[INFO|trainer.py:2673] 2026-08-22 07:47:52,846 >> Batch size = 8
+[WARNING|logging.py:340] 2026-08-22 07:47:53,011 >> The following generation flags are not valid and may be ignored: ['temperature', 'top_p'].
+[INFO|logging.py:355] 2026-08-22 07:47:53,011 >> - `temperature`: `do_sample` is not set to `True`. However, `temperature` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.
+- `top_p`: `do_sample` is not set to `True`. However, `top_p` is set to `0.7` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.
+If you're using a pretrained model, note that some of these attributes may be set through the model's `generation_config.json` file.
+[WARNING|utils.py:1642] 2026-08-22 07:47:53,014 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] The following generation flags are not valid and may be ignored: ['temperature', 'top_p']. Set `TRANSFORMERS_VERBOSITY=info` for more details.
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
0%| | 0/214 [00:00> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
1%| | 2/214 [00:06<11:47, 3.34s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:48:07,767 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
1%|▏ | 3/214 [00:12<15:44, 4.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:48:13,842 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
2%|▏ | 4/214 [00:19<19:11, 5.48s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:48:21,018 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
2%|▏ | 5/214 [00:26<20:54, 6.00s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:48:27,994 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
3%|▎ | 6/214 [00:34<22:20, 6.44s/it][WARNING|utils.py:1642] 2026-08-22 07:48:35,323 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
3%|▎ | 7/214 [00:41<23:22, 6.77s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:48:42,795 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
4%|▎ | 8/214 [00:48<22:55, 6.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:48:49,261 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
4%|▍ | 9/214 [00:54<22:18, 6.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:48:55,468 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
5%|▍ | 10/214 [01:01<23:03, 6.78s/it][WARNING|utils.py:1642] 2026-08-22 07:49:02,813 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
5%|▌ | 11/214 [01:08<22:32, 6.66s/it][WARNING|utils.py:1642] 2026-08-22 07:49:09,207 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
6%|▌ | 12/214 [01:14<21:50, 6.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:49:15,299 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
6%|▌ | 13/214 [01:21<22:31, 6.72s/it][WARNING|utils.py:1642] 2026-08-22 07:49:22,561 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 14/214 [01:28<22:46, 6.83s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:49:29,642 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 15/214 [01:35<23:01, 6.94s/it][WARNING|utils.py:1642] 2026-08-22 07:49:36,843 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
7%|▋ | 16/214 [01:42<22:52, 6.93s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:49:43,755 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
8%|▊ | 17/214 [01:50<23:49, 7.26s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:49:51,768 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
8%|▊ | 18/214 [01:56<22:36, 6.92s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:49:57,898 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
9%|▉ | 19/214 [02:03<22:32, 6.94s/it][WARNING|utils.py:1642] 2026-08-22 07:50:04,874 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
9%|▉ | 20/214 [02:11<22:46, 7.04s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:50:12,169 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
10%|▉ | 21/214 [02:17<21:44, 6.76s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:50:18,262 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
10%|█ | 22/214 [02:23<21:02, 6.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:50:24,416 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
11%|█ | 23/214 [02:30<21:28, 6.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:50:31,559 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
11%|█ | 24/214 [02:38<22:11, 7.01s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:50:39,178 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
12%|█▏ | 25/214 [02:45<22:30, 7.15s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:50:46,651 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
12%|█▏ | 26/214 [02:52<22:25, 7.16s/it][WARNING|utils.py:1642] 2026-08-22 07:50:53,831 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
13%|█▎ | 27/214 [02:59<22:17, 7.15s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:51:00,974 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
13%|█▎ | 28/214 [03:06<21:16, 6.87s/it][WARNING|utils.py:1642] 2026-08-22 07:51:07,167 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
14%|█▎ | 29/214 [03:13<21:33, 6.99s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:51:14,446 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
14%|█▍ | 30/214 [03:19<20:41, 6.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:51:20,628 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
14%|█▍ | 31/214 [03:25<19:49, 6.50s/it][WARNING|utils.py:1642] 2026-08-22 07:51:26,554 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
15%|█▍ | 32/214 [03:32<20:26, 6.74s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:51:33,849 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
15%|█▌ | 33/214 [03:39<20:34, 6.82s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:51:40,851 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
16%|█▌ | 34/214 [03:47<21:08, 7.05s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:51:48,439 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
16%|█▋ | 35/214 [03:54<21:17, 7.14s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:51:55,785 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
17%|█▋ | 36/214 [04:02<21:23, 7.21s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:52:03,167 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
17%|█▋ | 37/214 [04:09<21:04, 7.15s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:52:10,161 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
18%|█▊ | 38/214 [04:16<20:50, 7.10s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:52:17,169 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
18%|█▊ | 39/214 [04:23<20:44, 7.11s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:52:24,299 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
19%|█▊ | 40/214 [04:29<20:07, 6.94s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:52:30,835 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
19%|█▉ | 41/214 [04:35<19:18, 6.70s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:52:36,968 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
20%|█▉ | 42/214 [04:42<18:56, 6.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:52:43,358 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
20%|██ | 43/214 [04:49<18:56, 6.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:52:50,101 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██ | 44/214 [04:56<19:37, 6.93s/it][WARNING|utils.py:1642] 2026-08-22 07:52:57,682 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██ | 45/214 [05:03<19:50, 7.05s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:53:05,006 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
21%|██▏ | 46/214 [05:11<19:46, 7.06s/it][WARNING|utils.py:1642] 2026-08-22 07:53:12,114 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
22%|██▏ | 47/214 [05:18<19:44, 7.09s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:53:19,269 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
22%|██▏ | 48/214 [05:24<19:05, 6.90s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:53:25,726 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
23%|██▎ | 49/214 [05:33<20:23, 7.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:53:34,341 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
23%|██▎ | 50/214 [05:40<20:21, 7.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:53:41,858 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
24%|██▍ | 51/214 [05:47<20:01, 7.37s/it][WARNING|utils.py:1642] 2026-08-22 07:53:49,064 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
24%|██▍ | 52/214 [05:55<20:13, 7.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:53:56,837 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
25%|██▍ | 53/214 [06:03<20:06, 7.49s/it][WARNING|utils.py:1642] 2026-08-22 07:54:04,321 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
25%|██▌ | 54/214 [06:09<19:16, 7.23s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:54:10,931 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
26%|██▌ | 55/214 [06:16<18:58, 7.16s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:54:17,943 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
26%|██▌ | 56/214 [06:24<19:14, 7.30s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:54:25,578 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
27%|██▋ | 57/214 [06:32<19:18, 7.38s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:54:33,129 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
27%|██▋ | 58/214 [06:40<19:45, 7.60s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:54:41,252 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
28%|██▊ | 59/214 [06:47<19:22, 7.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:54:48,521 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
28%|██▊ | 60/214 [06:54<18:57, 7.38s/it][WARNING|utils.py:1642] 2026-08-22 07:54:55,630 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
29%|██▊ | 61/214 [07:00<17:46, 6.97s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:55:01,638 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
29%|██▉ | 62/214 [07:06<16:45, 6.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:55:07,416 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
29%|██▉ | 63/214 [07:12<16:20, 6.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:55:13,629 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
30%|██▉ | 64/214 [07:19<16:37, 6.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:55:20,650 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
30%|███ | 65/214 [07:26<16:51, 6.79s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:55:27,756 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
31%|███ | 66/214 [07:34<17:09, 6.96s/it][WARNING|utils.py:1642] 2026-08-22 07:55:35,105 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
31%|███▏ | 67/214 [07:42<18:06, 7.39s/it][WARNING|utils.py:1642] 2026-08-22 07:55:43,516 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
32%|███▏ | 68/214 [07:49<17:33, 7.22s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:55:50,329 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
32%|███▏ | 69/214 [07:56<17:25, 7.21s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:55:57,515 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
33%|███▎ | 70/214 [08:02<16:47, 6.99s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:56:04,008 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
33%|███▎ | 71/214 [08:09<16:43, 7.02s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:56:11,073 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
34%|███▎ | 72/214 [08:16<16:31, 6.98s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:56:17,977 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
34%|███▍ | 73/214 [08:23<15:51, 6.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:56:24,179 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
35%|███▍ | 74/214 [08:29<15:26, 6.61s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:56:30,481 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
35%|███▌ | 75/214 [08:36<16:01, 6.91s/it][WARNING|utils.py:1642] 2026-08-22 07:56:38,094 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▌ | 76/214 [08:44<16:14, 7.06s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:56:45,495 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▌ | 77/214 [08:50<15:47, 6.91s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:56:52,064 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
36%|███▋ | 78/214 [08:56<14:43, 6.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:56:57,589 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
37%|███▋ | 79/214 [09:03<14:57, 6.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:57:04,594 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
37%|███▋ | 80/214 [09:09<14:31, 6.50s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:57:10,749 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
38%|███▊ | 81/214 [09:16<14:26, 6.51s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:57:17,295 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
38%|███▊ | 82/214 [09:23<14:58, 6.81s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:57:24,779 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
39%|███▉ | 83/214 [09:31<15:40, 7.18s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:57:32,832 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
39%|███▉ | 84/214 [09:39<15:40, 7.23s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:57:40,190 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
40%|███▉ | 85/214 [09:47<16:02, 7.46s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:57:48,183 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
40%|████ | 86/214 [09:54<16:01, 7.51s/it][WARNING|utils.py:1642] 2026-08-22 07:57:55,820 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
41%|████ | 87/214 [10:01<15:16, 7.22s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:58:02,338 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
41%|████ | 88/214 [10:08<15:03, 7.17s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:58:09,410 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
42%|████▏ | 89/214 [10:14<14:17, 6.86s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:58:15,537 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
42%|████▏ | 90/214 [10:21<14:02, 6.79s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:58:22,182 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
43%|████▎ | 91/214 [10:27<13:42, 6.69s/it][WARNING|utils.py:1642] 2026-08-22 07:58:28,625 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
43%|████▎ | 92/214 [10:34<13:53, 6.83s/it][WARNING|utils.py:1642] 2026-08-22 07:58:35,791 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
43%|████▎ | 93/214 [10:41<13:49, 6.86s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:58:42,704 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
44%|████▍ | 94/214 [10:49<14:13, 7.11s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:58:50,414 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
44%|████▍ | 95/214 [10:56<14:25, 7.27s/it][WARNING|utils.py:1642] 2026-08-22 07:58:58,060 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
45%|████▍ | 96/214 [11:03<13:44, 6.98s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:59:04,372 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
45%|████▌ | 97/214 [11:10<13:44, 7.05s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:59:11,572 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
46%|████▌ | 98/214 [11:16<13:06, 6.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:59:17,719 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
46%|████▋ | 99/214 [11:22<12:30, 6.53s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:59:23,653 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
47%|████▋ | 100/214 [11:29<12:46, 6.73s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:59:30,849 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
47%|████▋ | 101/214 [11:35<12:17, 6.52s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:59:36,901 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
48%|████▊ | 102/214 [11:43<12:39, 6.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:59:44,289 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
48%|████▊ | 103/214 [11:50<12:40, 6.85s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:59:51,308 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
49%|████▊ | 104/214 [11:56<12:14, 6.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 07:59:57,585 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
49%|████▉ | 105/214 [12:04<12:48, 7.05s/it][WARNING|utils.py:1642] 2026-08-22 08:00:05,504 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|████▉ | 106/214 [12:11<12:39, 7.03s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:00:12,498 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|█████ | 107/214 [12:17<12:00, 6.73s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:00:18,522 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
50%|█████ | 108/214 [12:24<11:59, 6.79s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:00:25,445 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
51%|█████ | 109/214 [12:31<12:00, 6.87s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:00:32,491 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
51%|█████▏ | 110/214 [12:38<12:15, 7.07s/it][WARNING|utils.py:1642] 2026-08-22 08:00:40,053 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
52%|█████▏ | 111/214 [12:45<12:01, 7.00s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:00:46,895 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
52%|█████▏ | 112/214 [12:53<12:18, 7.24s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:00:54,673 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
53%|█████▎ | 113/214 [13:01<12:19, 7.32s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:01:02,192 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
53%|█████▎ | 114/214 [13:07<11:37, 6.98s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:01:08,361 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
54%|█████▎ | 115/214 [13:13<11:04, 6.71s/it][WARNING|utils.py:1642] 2026-08-22 08:01:14,444 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
54%|█████▍ | 116/214 [13:20<11:15, 6.89s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:01:21,765 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
55%|█████▍ | 117/214 [13:27<11:16, 6.98s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:01:28,934 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
55%|█████▌ | 118/214 [13:34<11:13, 7.02s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:01:36,049 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
56%|█████▌ | 119/214 [13:41<11:00, 6.95s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:01:42,842 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
56%|█████▌ | 120/214 [13:49<11:02, 7.05s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:01:50,117 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
57%|█████▋ | 121/214 [13:55<10:36, 6.85s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:01:56,492 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
57%|█████▋ | 122/214 [14:01<10:14, 6.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:02:02,774 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
57%|█████▋ | 123/214 [14:08<10:15, 6.77s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:02:09,747 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
58%|█████▊ | 124/214 [14:14<09:49, 6.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:02:15,811 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
58%|█████▊ | 125/214 [14:22<10:06, 6.82s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:02:23,246 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
59%|█████▉ | 126/214 [14:29<10:05, 6.88s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:02:30,272 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
59%|█████▉ | 127/214 [14:35<09:42, 6.69s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:02:36,532 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
60%|█████▉ | 128/214 [14:42<09:46, 6.82s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:02:43,638 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
60%|██████ | 129/214 [14:49<09:41, 6.84s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:02:50,539 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
61%|██████ | 130/214 [14:56<09:32, 6.82s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:02:57,309 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
61%|██████ | 131/214 [15:03<09:32, 6.90s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:03:04,385 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
62%|██████▏ | 132/214 [15:10<09:22, 6.86s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:03:11,170 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
62%|██████▏ | 133/214 [15:17<09:23, 6.96s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:03:18,361 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
63%|██████▎ | 134/214 [15:23<08:51, 6.65s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:03:24,280 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
63%|██████▎ | 135/214 [15:29<08:39, 6.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:03:30,692 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▎ | 136/214 [15:35<08:26, 6.49s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:03:36,981 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▍ | 137/214 [15:42<08:26, 6.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:03:43,781 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
64%|██████▍ | 138/214 [15:48<07:52, 6.22s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:03:49,141 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
65%|██████▍ | 139/214 [15:55<08:23, 6.71s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:03:56,999 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
65%|██████▌ | 140/214 [16:02<08:05, 6.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:04:03,218 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
66%|██████▌ | 141/214 [16:09<08:11, 6.74s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:04:10,363 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
66%|██████▋ | 142/214 [16:16<08:21, 6.97s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:04:17,865 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
67%|██████▋ | 143/214 [16:23<08:02, 6.80s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:04:24,267 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
67%|██████▋ | 144/214 [16:30<07:59, 6.85s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:04:31,255 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
68%|██████▊ | 145/214 [16:36<07:38, 6.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:04:37,399 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
68%|██████▊ | 146/214 [16:43<07:47, 6.88s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:04:44,837 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
69%|██████▊ | 147/214 [16:50<07:29, 6.70s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:04:51,131 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
69%|██████▉ | 148/214 [16:57<07:31, 6.84s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:04:58,298 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
70%|██████▉ | 149/214 [17:04<07:29, 6.92s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:05:05,388 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
70%|███████ | 150/214 [17:10<07:09, 6.72s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:05:11,638 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
71%|███████ | 151/214 [17:17<07:14, 6.89s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:05:18,944 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
71%|███████ | 152/214 [17:24<07:06, 6.88s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:05:25,781 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
71%|███████▏ | 153/214 [17:31<07:02, 6.93s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:05:32,829 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
72%|███████▏ | 154/214 [17:37<06:27, 6.45s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:05:38,166 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
72%|███████▏ | 155/214 [17:44<06:34, 6.68s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:05:45,393 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
73%|███████▎ | 156/214 [17:51<06:37, 6.86s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:05:52,657 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
73%|███████▎ | 157/214 [17:58<06:38, 6.99s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:05:59,964 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
74%|███████▍ | 158/214 [18:05<06:22, 6.84s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:06:06,444 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
74%|███████▍ | 159/214 [18:13<06:32, 7.14s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:06:14,299 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
75%|███████▍ | 160/214 [18:19<06:09, 6.85s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:06:20,459 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
75%|███████▌ | 161/214 [18:25<05:52, 6.64s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:06:26,621 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
76%|███████▌ | 162/214 [18:32<05:51, 6.76s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:06:33,667 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
76%|███████▌ | 163/214 [18:38<05:36, 6.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:06:39,854 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
77%|███████▋ | 164/214 [18:45<05:34, 6.69s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:06:46,779 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
77%|███████▋ | 165/214 [18:52<05:32, 6.79s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:06:53,816 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
78%|███████▊ | 166/214 [18:58<05:15, 6.58s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:06:59,890 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
78%|███████▊ | 167/214 [19:05<05:14, 6.70s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:07:06,866 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▊ | 168/214 [19:12<05:06, 6.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:07:13,453 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▉ | 169/214 [19:17<04:43, 6.30s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:07:18,909 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
79%|███████▉ | 170/214 [19:23<04:29, 6.12s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:07:24,620 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
80%|███████▉ | 171/214 [19:29<04:24, 6.15s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:07:30,841 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
80%|████████ | 172/214 [19:37<04:35, 6.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:07:38,363 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
81%|████████ | 173/214 [19:42<04:14, 6.20s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:07:43,717 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
81%|████████▏ | 174/214 [19:48<04:05, 6.15s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:07:49,740 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
82%|████████▏ | 175/214 [19:55<04:10, 6.42s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:07:56,808 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
82%|████████▏ | 176/214 [20:01<04:00, 6.32s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:08:02,901 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
83%|████████▎ | 177/214 [20:09<04:03, 6.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:08:10,116 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
83%|████████▎ | 178/214 [20:17<04:15, 7.09s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:08:18,360 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
84%|████████▎ | 179/214 [20:24<04:09, 7.13s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:08:25,600 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
84%|████████▍ | 180/214 [20:30<03:50, 6.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:08:31,565 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
85%|████████▍ | 181/214 [20:37<03:44, 6.81s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:08:38,453 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
85%|████████▌ | 182/214 [20:43<03:33, 6.66s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:08:44,759 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
86%|████████▌ | 183/214 [20:50<03:29, 6.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:08:51,717 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
86%|████████▌ | 184/214 [20:57<03:20, 6.69s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:08:58,250 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
86%|████████▋ | 185/214 [21:04<03:15, 6.76s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:09:05,174 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
87%|████████▋ | 186/214 [21:11<03:10, 6.82s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:09:12,145 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
87%|████████▋ | 187/214 [21:18<03:05, 6.88s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:09:19,149 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
88%|████████▊ | 188/214 [21:25<03:01, 6.99s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:09:26,408 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
88%|████████▊ | 189/214 [21:32<02:56, 7.05s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:09:33,607 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
89%|████████▉ | 190/214 [21:38<02:41, 6.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:09:39,634 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
89%|████████▉ | 191/214 [21:45<02:35, 6.76s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:09:46,425 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
90%|████████▉ | 192/214 [21:51<02:24, 6.56s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:09:52,537 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
90%|█████████ | 193/214 [21:57<02:17, 6.55s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:09:59,052 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
91%|█████████ | 194/214 [22:05<02:14, 6.72s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:10:06,164 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
91%|█████████ | 195/214 [22:12<02:10, 6.85s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:10:13,306 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
92%|█████████▏| 196/214 [22:18<02:01, 6.77s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:10:19,907 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
92%|█████████▏| 197/214 [22:25<01:56, 6.84s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:10:26,919 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 198/214 [22:32<01:50, 6.94s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:10:34,073 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 199/214 [22:39<01:43, 6.90s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:10:40,901 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
93%|█████████▎| 200/214 [22:46<01:36, 6.87s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:10:47,683 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
94%|█████████▍| 201/214 [22:53<01:28, 6.82s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:10:54,402 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
94%|█████████▍| 202/214 [23:01<01:25, 7.09s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:11:02,125 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
95%|█████████▍| 203/214 [23:07<01:14, 6.78s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:11:08,191 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
95%|█████████▌| 204/214 [23:14<01:09, 6.97s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:11:15,586 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
96%|█████████▌| 205/214 [23:20<01:00, 6.67s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:11:21,567 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
96%|█████████▋| 206/214 [23:27<00:53, 6.75s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:11:28,487 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
97%|█████████▋| 207/214 [23:34<00:48, 6.90s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:11:35,748 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
97%|█████████▋| 208/214 [23:41<00:41, 6.99s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:11:42,954 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
98%|█████████▊| 209/214 [23:49<00:36, 7.21s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:11:50,667 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
98%|█████████▊| 210/214 [23:56<00:27, 6.99s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:11:57,133 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
99%|█████████▊| 211/214 [24:03<00:21, 7.15s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:12:04,651 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
99%|█████████▉| 212/214 [24:09<00:13, 6.93s/it][WARNING|utils.py:1642] 2026-08-22 08:12:11,089 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
100%|█████████▉| 213/214 [24:15<00:06, 6.59s/it][transformers] Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+[WARNING|utils.py:1642] 2026-08-22 08:12:17,039 >> Both `max_new_tokens` (=256) and `max_length`(=20) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)
+
100%|██████████| 214/214 [24:23<00:00, 6.93s/it]
100%|██████████| 214/214 [24:24<00:00, 6.84s/it]
+***** eval metrics *****
+ epoch = 5.0
+ eval_generator_count = 1712.0
+ eval_generator_exact_match = 97.0794
+ eval_generator_rouge1 = 99.9517
+ eval_generator_rouge2 = 99.914
+ eval_generator_rougeL = 99.7744
+ eval_loss = 0.0029
+ eval_runtime = 0:24:32.90
+ eval_samples_per_second = 2.318
+ eval_selector_accuracy = 99.2407
+ eval_selector_count = 1712.0
+ eval_steps_per_second = 0.145
+[INFO|modelcard.py:263] 2026-08-22 08:12:25,748 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
diff --git a/qwen3/run_all_logs/20260823_150131/teacher_sft.log b/qwen3/run_all_logs/20260823_150131/teacher_sft.log
new file mode 100644
index 0000000000000000000000000000000000000000..f26374546069d79bb535ba9c49119c38e7c0e8c2
--- /dev/null
+++ b/qwen3/run_all_logs/20260823_150131/teacher_sft.log
@@ -0,0 +1,4 @@
+W0823 15:01:32.644000 1429241 torch/distributed/run.py:803]
+W0823 15:01:32.644000 1429241 torch/distributed/run.py:803] *****************************************
+W0823 15:01:32.644000 1429241 torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+W0823 15:01:32.644000 1429241 torch/distributed/run.py:803] *****************************************
diff --git a/qwen3/run_all_logs/20260823_150138/teacher_sft.log b/qwen3/run_all_logs/20260823_150138/teacher_sft.log
new file mode 100644
index 0000000000000000000000000000000000000000..5f7126a884efa4437c993bb6fceafb7da8acf080
--- /dev/null
+++ b/qwen3/run_all_logs/20260823_150138/teacher_sft.log
@@ -0,0 +1,4 @@
+W0823 15:01:39.593000 1429319 torch/distributed/run.py:803]
+W0823 15:01:39.593000 1429319 torch/distributed/run.py:803] *****************************************
+W0823 15:01:39.593000 1429319 torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+W0823 15:01:39.593000 1429319 torch/distributed/run.py:803] *****************************************
diff --git a/qwen3/sfkl/README.md b/qwen3/sfkl/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..bf7f21b8bcadddec9bec41d9d300d7d84d913f31
--- /dev/null
+++ b/qwen3/sfkl/README.md
@@ -0,0 +1,79 @@
+---
+library_name: peft
+license: other
+base_model: Qwen/Qwen3-0.6B
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+pipeline_tag: text-generation
+model-index:
+- name: sfkl
+ results: []
+---
+
+
+
+# sfkl
+
+This model is a fine-tuned version of [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) on the cypher_prepared_train dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0032
+- Selector Count: 1712.0
+- Generator Count: 1712.0
+- Selector Accuracy: 98.715
+- Generator Exact Match: 96.729
+- Generator Rouge1: 99.9412
+- Generator Rouge2: 99.8856
+- Generator Rougel: 99.7768
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 2
+- eval_batch_size: 16
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine_with_min_lr
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Selector Count | Generator Count | Selector Accuracy | Generator Exact Match | Generator Rouge1 | Generator Rouge2 | Generator Rougel |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:---------------:|:-----------------:|:---------------------:|:----------------:|:----------------:|:----------------:|
+| 0.0089 | 1.0 | 314 | 0.0076 | 1712.0 | 1712.0 | 96.6121 | 91.2967 | 99.5496 | 99.3396 | 99.2032 |
+| 0.0030 | 2.0 | 628 | 0.0056 | 1712.0 | 1712.0 | 97.6051 | 93.4579 | 99.8096 | 99.6454 | 99.6441 |
+| 0.0014 | 3.0 | 942 | 0.0041 | 1712.0 | 1712.0 | 98.0724 | 95.4439 | 99.8682 | 99.7679 | 99.6703 |
+| 0.0009 | 4.0 | 1256 | 0.0032 | 1712.0 | 1712.0 | 98.5397 | 97.021 | 99.95 | 99.9065 | 99.7835 |
+| 0.0015 | 5.0 | 1570 | 0.0032 | 1712.0 | 1712.0 | 98.7734 | 96.6121 | 99.9495 | 99.9016 | 99.764 |
+
+
+### Framework versions
+
+- PEFT 0.18.1
+- Transformers 5.8.0
+- Pytorch 2.9.1+cu128
+- Datasets 4.0.0
+- Tokenizers 0.22.2
\ No newline at end of file
diff --git a/qwen3/sfkl/adapter_config.json b/qwen3/sfkl/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..df150dce63edb909460ef4d599512e21d2dc45da
--- /dev/null
+++ b/qwen3/sfkl/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "up_proj",
+ "q_proj",
+ "o_proj",
+ "gate_proj",
+ "k_proj",
+ "v_proj",
+ "down_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/sfkl/adapter_model.safetensors b/qwen3/sfkl/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fca8349ddf820c4db47d66ac50693a5a8dd1c0b8
--- /dev/null
+++ b/qwen3/sfkl/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36e440497f4b55289300e8706e17c35558f9c70b70d99a174d53731222bb2fe1
+size 40422560
diff --git a/qwen3/sfkl/all_results.json b/qwen3/sfkl/all_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..b982dda91c9402fecd642a0b18c620b6c9092f19
--- /dev/null
+++ b/qwen3/sfkl/all_results.json
@@ -0,0 +1,19 @@
+{
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.729,
+ "eval_generator_rouge1": 99.9412,
+ "eval_generator_rouge2": 99.8856,
+ "eval_generator_rougeL": 99.7768,
+ "eval_loss": 0.003214908065274358,
+ "eval_runtime": 598.5838,
+ "eval_samples_per_second": 5.703,
+ "eval_selector_accuracy": 98.715,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.179,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.012116143728493695,
+ "train_runtime": 7600.5401,
+ "train_samples_per_second": 6.596,
+ "train_steps_per_second": 0.207
+}
\ No newline at end of file
diff --git a/qwen3/sfkl/chat_template.jinja b/qwen3/sfkl/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/sfkl/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-1256/README.md b/qwen3/sfkl/checkpoint-1256/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-1256/adapter_config.json b/qwen3/sfkl/checkpoint-1256/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..df150dce63edb909460ef4d599512e21d2dc45da
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "up_proj",
+ "q_proj",
+ "o_proj",
+ "gate_proj",
+ "k_proj",
+ "v_proj",
+ "down_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-1256/adapter_model.safetensors b/qwen3/sfkl/checkpoint-1256/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9d7855a4de548872b0798191b8d410ac959b3be2
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42c7093fa44593c4b9af64edc44ce38802c7a9f106cbb7e823b197241f813a36
+size 40422560
diff --git a/qwen3/sfkl/checkpoint-1256/chat_template.jinja b/qwen3/sfkl/checkpoint-1256/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/sfkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f8fe8572d23c86dfa06ed0c5a2ee0b996eb0e12f
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39a421103d3a87df61b1460d63456d4a1b1802f9174ccee785578e700458bfd7
+size 121135557
diff --git a/qwen3/sfkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/sfkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9424788f7a22541fcd79a6f5de450bc9ade2886c
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b79174e357608ff74cc54ac813b859a4a7808484d468940c5b08dfe0229320b
+size 121135749
diff --git a/qwen3/sfkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt b/qwen3/sfkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8b7ce482af61132a0cdfd81e72a0ca2609bd8d38
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24148b76f1df50802e5ef8cbeb100c87d00a644092275852dbacdfc84c8ed674
+size 351841765
diff --git a/qwen3/sfkl/checkpoint-1256/latest b/qwen3/sfkl/checkpoint-1256/latest
new file mode 100644
index 0000000000000000000000000000000000000000..74abb3d23ea83ebdac59918752db5d1782834ca7
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/latest
@@ -0,0 +1 @@
+global_step1256
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-1256/rng_state_0.pth b/qwen3/sfkl/checkpoint-1256/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5953bad23aab84b3ae9f9488ea315d311b31de6b
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e50bd6493972c0363d8d68878cfc849df66928333ce565a918fd1a93e9abe626
+size 14917
diff --git a/qwen3/sfkl/checkpoint-1256/rng_state_1.pth b/qwen3/sfkl/checkpoint-1256/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4bd3a727adf58684fad2ca63d69b940201917711
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:868aaf2c3a1fed5e1452ea6e11155ed0c7f8aa410d7f20481887dd854e264aae
+size 14917
diff --git a/qwen3/sfkl/checkpoint-1256/scheduler.pt b/qwen3/sfkl/checkpoint-1256/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40e863d07cfed22cd912014eb3ba3fd2b16b173f
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1775297f14e75f7463ae51983a66296f81f4b2e2e5d7234f1a6d295abfa4cf23
+size 1465
diff --git a/qwen3/sfkl/checkpoint-1256/tokenizer.json b/qwen3/sfkl/checkpoint-1256/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/sfkl/checkpoint-1256/tokenizer_config.json b/qwen3/sfkl/checkpoint-1256/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/sfkl/checkpoint-1256/trainer_state.json b/qwen3/sfkl/checkpoint-1256/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..44ac1bacd4ba2b8fa27eb060ff3e68812d50629e
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/trainer_state.json
@@ -0,0 +1,528 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 4.0,
+ "eval_steps": 500,
+ "global_step": 1256,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.2921956777572632,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.36003756523132324,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 1.6740401983261108,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.09387726187705994,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 1.1545500755310059,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.06687320470809936,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.42471569776535034,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.04825929999351501,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.34235191345214844,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.02988826632499695,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.2986029088497162,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.021525751054286956,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.27962905168533325,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.02630552351474762,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.173891082406044,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.010886980593204499,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 0.4828740358352661,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.012579935789108276,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.3599541485309601,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.018575572967529298,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.21371620893478394,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.012867593765258789,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.2939048111438751,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.011218706518411637,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.2609497010707855,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.0136981800198555,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.17881983518600464,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.008669006079435349,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.24963699281215668,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.008861982077360154,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 91.2967,
+ "eval_generator_rouge1": 99.5496,
+ "eval_generator_rouge2": 99.3396,
+ "eval_generator_rougeL": 99.2032,
+ "eval_loss": 0.007557725068181753,
+ "eval_runtime": 597.4488,
+ "eval_samples_per_second": 5.714,
+ "eval_selector_accuracy": 96.6121,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.179,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 1.7057307958602905,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.008970273286104202,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.1011928841471672,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.003804096579551697,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.33115050196647644,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.005674988031387329,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.21229985356330872,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.0043993469327688215,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.11725866794586182,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.006616537272930145,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.1508084237575531,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.0059788327664136885,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.034210484474897385,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.004834429547190666,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.36105096340179443,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.006534397602081299,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.34886789321899414,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.004452808201313019,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.07787973433732986,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.008046584576368332,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.05649268627166748,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.006670549511909485,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.16277413070201874,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.007858693599700928,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.016449013724923134,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.0019811514765024187,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.4171461760997772,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.009944957494735718,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.2326679229736328,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.008132340759038926,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.047354914247989655,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.0029657069593667985,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.4579,
+ "eval_generator_rouge1": 99.8096,
+ "eval_generator_rouge2": 99.6454,
+ "eval_generator_rougeL": 99.6441,
+ "eval_loss": 0.005621648393571377,
+ "eval_runtime": 595.9265,
+ "eval_samples_per_second": 5.729,
+ "eval_selector_accuracy": 97.6051,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.18,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.11734319478273392,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.0028536681085824965,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.20789793133735657,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.002620901167392731,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.12088560312986374,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0017223741859197616,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.0175655297935009,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0019577957689762117,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.1677847057580948,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.005092801898717881,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.027272474020719528,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.0032966256141662596,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.030720148235559464,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0015653561800718307,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.028172848746180534,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0025819912552833557,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.1057867631316185,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.005924087017774582,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.23805274069309235,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.004035309329628945,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.055235955864191055,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.0022109072655439378,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.07275807857513428,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.005412191897630691,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.036749616265296936,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0019010215997695922,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.11888282001018524,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0013746129348874091,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.004523377399891615,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.00334613136947155,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.08859030157327652,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0014281794428825378,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.4439,
+ "eval_generator_rouge1": 99.8682,
+ "eval_generator_rouge2": 99.7679,
+ "eval_generator_rougeL": 99.6703,
+ "eval_loss": 0.004075318109244108,
+ "eval_runtime": 595.0587,
+ "eval_samples_per_second": 5.737,
+ "eval_selector_accuracy": 98.0724,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.18,
+ "step": 942
+ },
+ {
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.033190421760082245,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.003326282650232315,
+ "step": 960
+ },
+ {
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.038682322949171066,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.0027384284883737566,
+ "step": 980
+ },
+ {
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.024536851793527603,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.001315690018236637,
+ "step": 1000
+ },
+ {
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.02027392014861107,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.003594046086072922,
+ "step": 1020
+ },
+ {
+ "epoch": 3.3127243717590744,
+ "grad_norm": 0.26362860202789307,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.0022306183353066446,
+ "step": 1040
+ },
+ {
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.025666173547506332,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.002582985907793045,
+ "step": 1060
+ },
+ {
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.06274232268333435,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.003804165869951248,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.0051957834511995316,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0013254371471703053,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.08129020780324936,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.0011303042992949486,
+ "step": 1120
+ },
+ {
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.08928140252828598,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0009621990844607353,
+ "step": 1140
+ },
+ {
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.5374053716659546,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.004591869562864304,
+ "step": 1160
+ },
+ {
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.032263629138469696,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0023022303357720375,
+ "step": 1180
+ },
+ {
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.005962996743619442,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.002387513592839241,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8871160749900278,
+ "grad_norm": 0.07597373425960541,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.002432684227824211,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.025445526465773582,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0009138435125350952,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 97.021,
+ "eval_generator_rouge1": 99.95,
+ "eval_generator_rouge2": 99.9065,
+ "eval_generator_rougeL": 99.7835,
+ "eval_loss": 0.00319857569411397,
+ "eval_runtime": 604.8868,
+ "eval_samples_per_second": 5.644,
+ "eval_selector_accuracy": 98.5397,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.177,
+ "step": 1256
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 4.697643190059008e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/sfkl/checkpoint-1256/training_args.bin b/qwen3/sfkl/checkpoint-1256/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..019105eaa7de05515b126bf422de20d1df4e85f5
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2d955bcf260cad259a2bda62ea641cce8e39b7cf407995d5f86c2c02b6e6f3e
+size 7185
diff --git a/qwen3/sfkl/checkpoint-1256/zero_to_fp32.py b/qwen3/sfkl/checkpoint-1256/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1256/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/sfkl/checkpoint-1570/README.md b/qwen3/sfkl/checkpoint-1570/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-1570/adapter_config.json b/qwen3/sfkl/checkpoint-1570/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..df150dce63edb909460ef4d599512e21d2dc45da
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "up_proj",
+ "q_proj",
+ "o_proj",
+ "gate_proj",
+ "k_proj",
+ "v_proj",
+ "down_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-1570/adapter_model.safetensors b/qwen3/sfkl/checkpoint-1570/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fca8349ddf820c4db47d66ac50693a5a8dd1c0b8
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36e440497f4b55289300e8706e17c35558f9c70b70d99a174d53731222bb2fe1
+size 40422560
diff --git a/qwen3/sfkl/checkpoint-1570/chat_template.jinja b/qwen3/sfkl/checkpoint-1570/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/sfkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a25a11ed8865a795c9b854dcefa723df32125b5a
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3aefd650bb15d695a7407876949f35ae82da0073db027f7c045f34be8ca31a2e
+size 121135557
diff --git a/qwen3/sfkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/sfkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1854742317ae79714ec7ae92011f601e849ad3d1
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1552338ae54981624c0e5327348e0ed6cf26634358cbc2f64844ca28f2489f4b
+size 121135749
diff --git a/qwen3/sfkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt b/qwen3/sfkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f02b2247d012d7a5f4260e37cb6e7a3a77f5e7c2
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4bc61d892efdb4fb48bae247a7971e3bb37f8767e818fb41833af4c6f73c3d3
+size 351841765
diff --git a/qwen3/sfkl/checkpoint-1570/latest b/qwen3/sfkl/checkpoint-1570/latest
new file mode 100644
index 0000000000000000000000000000000000000000..18c2fcff58fa6daa0a2e7c3a015552a69085778b
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/latest
@@ -0,0 +1 @@
+global_step1570
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-1570/rng_state_0.pth b/qwen3/sfkl/checkpoint-1570/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..30760a89ada2f12a49f76b347f60a03375b7c90e
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e34f640593c20df12bb1ef6a085ff075fae9e4f2e80091757641256f75567a2
+size 14917
diff --git a/qwen3/sfkl/checkpoint-1570/rng_state_1.pth b/qwen3/sfkl/checkpoint-1570/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..c5c47b551feed7cdc16d6e6767c7c34b023ff38b
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfa8fcc86db225c9766fc0b8e4a00006b99d4a8f9929ecbc8189828057d657c0
+size 14917
diff --git a/qwen3/sfkl/checkpoint-1570/scheduler.pt b/qwen3/sfkl/checkpoint-1570/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e22954463492d8a070ae427c95418df61893c54c
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6332fac17ae99a07597512c705658da2fd8429d1c39bb776b30e107c2cb7aae
+size 1465
diff --git a/qwen3/sfkl/checkpoint-1570/tokenizer.json b/qwen3/sfkl/checkpoint-1570/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/sfkl/checkpoint-1570/tokenizer_config.json b/qwen3/sfkl/checkpoint-1570/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/sfkl/checkpoint-1570/trainer_state.json b/qwen3/sfkl/checkpoint-1570/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..95050c20f213caa2a2722ff2e3153044c024839b
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/trainer_state.json
@@ -0,0 +1,655 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 5.0,
+ "eval_steps": 500,
+ "global_step": 1570,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.2921956777572632,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.36003756523132324,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 1.6740401983261108,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.09387726187705994,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 1.1545500755310059,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.06687320470809936,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.42471569776535034,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.04825929999351501,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.34235191345214844,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.02988826632499695,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.2986029088497162,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.021525751054286956,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.27962905168533325,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.02630552351474762,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.173891082406044,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.010886980593204499,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 0.4828740358352661,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.012579935789108276,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.3599541485309601,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.018575572967529298,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.21371620893478394,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.012867593765258789,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.2939048111438751,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.011218706518411637,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.2609497010707855,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.0136981800198555,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.17881983518600464,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.008669006079435349,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.24963699281215668,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.008861982077360154,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 91.2967,
+ "eval_generator_rouge1": 99.5496,
+ "eval_generator_rouge2": 99.3396,
+ "eval_generator_rougeL": 99.2032,
+ "eval_loss": 0.007557725068181753,
+ "eval_runtime": 597.4488,
+ "eval_samples_per_second": 5.714,
+ "eval_selector_accuracy": 96.6121,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.179,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 1.7057307958602905,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.008970273286104202,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.1011928841471672,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.003804096579551697,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.33115050196647644,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.005674988031387329,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.21229985356330872,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.0043993469327688215,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.11725866794586182,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.006616537272930145,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.1508084237575531,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.0059788327664136885,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.034210484474897385,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.004834429547190666,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.36105096340179443,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.006534397602081299,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.34886789321899414,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.004452808201313019,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.07787973433732986,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.008046584576368332,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.05649268627166748,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.006670549511909485,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.16277413070201874,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.007858693599700928,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.016449013724923134,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.0019811514765024187,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.4171461760997772,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.009944957494735718,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.2326679229736328,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.008132340759038926,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.047354914247989655,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.0029657069593667985,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.4579,
+ "eval_generator_rouge1": 99.8096,
+ "eval_generator_rouge2": 99.6454,
+ "eval_generator_rougeL": 99.6441,
+ "eval_loss": 0.005621648393571377,
+ "eval_runtime": 595.9265,
+ "eval_samples_per_second": 5.729,
+ "eval_selector_accuracy": 97.6051,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.18,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.11734319478273392,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.0028536681085824965,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.20789793133735657,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.002620901167392731,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.12088560312986374,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0017223741859197616,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.0175655297935009,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0019577957689762117,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.1677847057580948,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.005092801898717881,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.027272474020719528,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.0032966256141662596,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.030720148235559464,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0015653561800718307,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.028172848746180534,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0025819912552833557,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.1057867631316185,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.005924087017774582,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.23805274069309235,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.004035309329628945,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.055235955864191055,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.0022109072655439378,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.07275807857513428,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.005412191897630691,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.036749616265296936,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0019010215997695922,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.11888282001018524,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0013746129348874091,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.004523377399891615,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.00334613136947155,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.08859030157327652,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0014281794428825378,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.4439,
+ "eval_generator_rouge1": 99.8682,
+ "eval_generator_rouge2": 99.7679,
+ "eval_generator_rougeL": 99.6703,
+ "eval_loss": 0.004075318109244108,
+ "eval_runtime": 595.0587,
+ "eval_samples_per_second": 5.737,
+ "eval_selector_accuracy": 98.0724,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.18,
+ "step": 942
+ },
+ {
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.033190421760082245,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.003326282650232315,
+ "step": 960
+ },
+ {
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.038682322949171066,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.0027384284883737566,
+ "step": 980
+ },
+ {
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.024536851793527603,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.001315690018236637,
+ "step": 1000
+ },
+ {
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.02027392014861107,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.003594046086072922,
+ "step": 1020
+ },
+ {
+ "epoch": 3.3127243717590744,
+ "grad_norm": 0.26362860202789307,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.0022306183353066446,
+ "step": 1040
+ },
+ {
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.025666173547506332,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.002582985907793045,
+ "step": 1060
+ },
+ {
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.06274232268333435,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.003804165869951248,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.0051957834511995316,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0013254371471703053,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.08129020780324936,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.0011303042992949486,
+ "step": 1120
+ },
+ {
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.08928140252828598,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0009621990844607353,
+ "step": 1140
+ },
+ {
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.5374053716659546,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.004591869562864304,
+ "step": 1160
+ },
+ {
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.032263629138469696,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0023022303357720375,
+ "step": 1180
+ },
+ {
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.005962996743619442,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.002387513592839241,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8871160749900278,
+ "grad_norm": 0.07597373425960541,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.002432684227824211,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.025445526465773582,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0009138435125350952,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 97.021,
+ "eval_generator_rouge1": 99.95,
+ "eval_generator_rouge2": 99.9065,
+ "eval_generator_rougeL": 99.7835,
+ "eval_loss": 0.00319857569411397,
+ "eval_runtime": 604.8868,
+ "eval_samples_per_second": 5.644,
+ "eval_selector_accuracy": 98.5397,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.177,
+ "step": 1256
+ },
+ {
+ "epoch": 4.0127642600717985,
+ "grad_norm": 0.34829947352409363,
+ "learning_rate": 9.464081721960592e-06,
+ "loss": 0.0020675655454397202,
+ "step": 1260
+ },
+ {
+ "epoch": 4.076585560430794,
+ "grad_norm": 0.1925937831401825,
+ "learning_rate": 8.331628936475818e-06,
+ "loss": 0.0007737125270068645,
+ "step": 1280
+ },
+ {
+ "epoch": 4.140406860789788,
+ "grad_norm": 0.03017379343509674,
+ "learning_rate": 7.2659843568748385e-06,
+ "loss": 0.0011951016262173654,
+ "step": 1300
+ },
+ {
+ "epoch": 4.204228161148784,
+ "grad_norm": 0.007225691340863705,
+ "learning_rate": 6.268854516770008e-06,
+ "loss": 0.0011913912370800973,
+ "step": 1320
+ },
+ {
+ "epoch": 4.268049461507778,
+ "grad_norm": 0.012843134813010693,
+ "learning_rate": 5.341836229602331e-06,
+ "loss": 0.0011537913233041763,
+ "step": 1340
+ },
+ {
+ "epoch": 4.331870761866773,
+ "grad_norm": 0.6861193776130676,
+ "learning_rate": 4.4864140314888216e-06,
+ "loss": 0.0043139740824699405,
+ "step": 1360
+ },
+ {
+ "epoch": 4.395692062225768,
+ "grad_norm": 0.025450166314840317,
+ "learning_rate": 3.703957803871932e-06,
+ "loss": 0.001425867434591055,
+ "step": 1380
+ },
+ {
+ "epoch": 4.4595133625847625,
+ "grad_norm": 0.023695463314652443,
+ "learning_rate": 2.9957205797780495e-06,
+ "loss": 0.001892777904868126,
+ "step": 1400
+ },
+ {
+ "epoch": 4.523334662943758,
+ "grad_norm": 0.04371035099029541,
+ "learning_rate": 2.362836537198199e-06,
+ "loss": 0.0016116222366690635,
+ "step": 1420
+ },
+ {
+ "epoch": 4.587155963302752,
+ "grad_norm": 0.01301620528101921,
+ "learning_rate": 1.8063191828043703e-06,
+ "loss": 0.0010820812545716762,
+ "step": 1440
+ },
+ {
+ "epoch": 4.650977263661747,
+ "grad_norm": 0.03708479553461075,
+ "learning_rate": 1.3270597289100437e-06,
+ "loss": 0.000809190608561039,
+ "step": 1460
+ },
+ {
+ "epoch": 4.714798564020742,
+ "grad_norm": 0.0060765426605939865,
+ "learning_rate": 9.258256662741205e-07,
+ "loss": 0.0007033236790448428,
+ "step": 1480
+ },
+ {
+ "epoch": 4.778619864379737,
+ "grad_norm": 0.014266903512179852,
+ "learning_rate": 6.032595350337341e-07,
+ "loss": 0.0015078827738761902,
+ "step": 1500
+ },
+ {
+ "epoch": 4.842441164738732,
+ "grad_norm": 0.01251401286572218,
+ "learning_rate": 3.5987789573421356e-07,
+ "loss": 0.004571952670812607,
+ "step": 1520
+ },
+ {
+ "epoch": 4.9062624650977265,
+ "grad_norm": 0.1618906855583191,
+ "learning_rate": 1.9607050210401192e-07,
+ "loss": 0.0009092647582292557,
+ "step": 1540
+ },
+ {
+ "epoch": 4.970083765456721,
+ "grad_norm": 0.026854628697037697,
+ "learning_rate": 1.1209967689925832e-07,
+ "loss": 0.0015063435770571233,
+ "step": 1560
+ },
+ {
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.6121,
+ "eval_generator_rouge1": 99.9495,
+ "eval_generator_rouge2": 99.9016,
+ "eval_generator_rougeL": 99.764,
+ "eval_loss": 0.003214908065274358,
+ "eval_runtime": 612.4845,
+ "eval_samples_per_second": 5.574,
+ "eval_selector_accuracy": 98.7734,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 1570
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 5.871444840310374e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/sfkl/checkpoint-1570/training_args.bin b/qwen3/sfkl/checkpoint-1570/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..019105eaa7de05515b126bf422de20d1df4e85f5
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2d955bcf260cad259a2bda62ea641cce8e39b7cf407995d5f86c2c02b6e6f3e
+size 7185
diff --git a/qwen3/sfkl/checkpoint-1570/zero_to_fp32.py b/qwen3/sfkl/checkpoint-1570/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-1570/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/sfkl/checkpoint-314/README.md b/qwen3/sfkl/checkpoint-314/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-314/adapter_config.json b/qwen3/sfkl/checkpoint-314/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..df150dce63edb909460ef4d599512e21d2dc45da
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "up_proj",
+ "q_proj",
+ "o_proj",
+ "gate_proj",
+ "k_proj",
+ "v_proj",
+ "down_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-314/adapter_model.safetensors b/qwen3/sfkl/checkpoint-314/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fe2959449634b2549ab1c3922d33b9b56926f73e
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20dce96cdaccd5dabd3335357cf9897a646358de859d7d51875a236e9e5a0b88
+size 40422560
diff --git a/qwen3/sfkl/checkpoint-314/chat_template.jinja b/qwen3/sfkl/checkpoint-314/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/sfkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ade4e370a48de2b3cb29bc02daa044a09a167bdb
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74f2386f20cf3a31332789eda209cc6b9ffcba9e96832fa7cd09fa0d1cb08705
+size 121135557
diff --git a/qwen3/sfkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/sfkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fc059b8964122f9c10706f617bdba81ed9d95f42
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b86effa48bde9bd03b40b57d3f63412a46f25e5d07d7ee076b291cd160c0b62
+size 121135749
diff --git a/qwen3/sfkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt b/qwen3/sfkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..47fc02fddc98bf89d34d5ae094e646c48466f9cb
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b9c04519b6c90eeade63050c1f3069f19893398d6cf3d2882bf5410c0ebdcf5
+size 351841765
diff --git a/qwen3/sfkl/checkpoint-314/latest b/qwen3/sfkl/checkpoint-314/latest
new file mode 100644
index 0000000000000000000000000000000000000000..a9786513b1a514db2c3a9da7408844635663d55d
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/latest
@@ -0,0 +1 @@
+global_step314
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-314/rng_state_0.pth b/qwen3/sfkl/checkpoint-314/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..29b45d7f899e9eeaef060485b04a499402b7e7fd
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ed4aef0d4d68afdd75137898b82610e070c2dae5992614174875bb3508c2d66
+size 14917
diff --git a/qwen3/sfkl/checkpoint-314/rng_state_1.pth b/qwen3/sfkl/checkpoint-314/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..76f75c2ae94635d1eedc1587bcea11a7b14929df
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f05b8aadd0567e7816ea17524bc4fc746072725ac87fbc1612abe0c5d21ad7b0
+size 14917
diff --git a/qwen3/sfkl/checkpoint-314/scheduler.pt b/qwen3/sfkl/checkpoint-314/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3cb460802cda03bf0833a98a6f1428a3fd4f585
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17e254b5642bb75983fd5105bec79b35376cfbc99d29702287668b8767d46149
+size 1465
diff --git a/qwen3/sfkl/checkpoint-314/tokenizer.json b/qwen3/sfkl/checkpoint-314/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/sfkl/checkpoint-314/tokenizer_config.json b/qwen3/sfkl/checkpoint-314/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/sfkl/checkpoint-314/trainer_state.json b/qwen3/sfkl/checkpoint-314/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..09e197fd802b0a5ec78a5e691f6aeeeed3ab0f3d
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/trainer_state.json
@@ -0,0 +1,154 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 1.0,
+ "eval_steps": 500,
+ "global_step": 314,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.2921956777572632,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.36003756523132324,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 1.6740401983261108,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.09387726187705994,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 1.1545500755310059,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.06687320470809936,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.42471569776535034,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.04825929999351501,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.34235191345214844,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.02988826632499695,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.2986029088497162,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.021525751054286956,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.27962905168533325,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.02630552351474762,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.173891082406044,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.010886980593204499,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 0.4828740358352661,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.012579935789108276,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.3599541485309601,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.018575572967529298,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.21371620893478394,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.012867593765258789,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.2939048111438751,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.011218706518411637,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.2609497010707855,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.0136981800198555,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.17881983518600464,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.008669006079435349,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.24963699281215668,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.008861982077360154,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 91.2967,
+ "eval_generator_rouge1": 99.5496,
+ "eval_generator_rouge2": 99.3396,
+ "eval_generator_rougeL": 99.2032,
+ "eval_loss": 0.007557725068181753,
+ "eval_runtime": 597.4488,
+ "eval_samples_per_second": 5.714,
+ "eval_selector_accuracy": 96.6121,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.179,
+ "step": 314
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 1.172842003562496e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/sfkl/checkpoint-314/training_args.bin b/qwen3/sfkl/checkpoint-314/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..019105eaa7de05515b126bf422de20d1df4e85f5
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2d955bcf260cad259a2bda62ea641cce8e39b7cf407995d5f86c2c02b6e6f3e
+size 7185
diff --git a/qwen3/sfkl/checkpoint-314/zero_to_fp32.py b/qwen3/sfkl/checkpoint-314/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-314/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/sfkl/checkpoint-628/README.md b/qwen3/sfkl/checkpoint-628/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-628/adapter_config.json b/qwen3/sfkl/checkpoint-628/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..df150dce63edb909460ef4d599512e21d2dc45da
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "up_proj",
+ "q_proj",
+ "o_proj",
+ "gate_proj",
+ "k_proj",
+ "v_proj",
+ "down_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-628/adapter_model.safetensors b/qwen3/sfkl/checkpoint-628/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..53e93ab0b232972aa6ea56b9bb7040e43b404acf
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02bd10d83c98c20abb0cfd538db0c7a0fa453f58d6543196095236a494030c22
+size 40422560
diff --git a/qwen3/sfkl/checkpoint-628/chat_template.jinja b/qwen3/sfkl/checkpoint-628/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/sfkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..41c19e15e72c8619fc0658aec11b210c2310ecd4
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e61214f2b20bb7f7c8f1885c969143b33ed15fe4c0c11d9e1d71712c71790b97
+size 121135557
diff --git a/qwen3/sfkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/sfkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a82c43c3966c91ae1aad8a10713c1d562bfe79b6
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb78b477d14b3431389f4914b89c39d5365f72d1a6bbb28df37b85e385fd121c
+size 121135749
diff --git a/qwen3/sfkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt b/qwen3/sfkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76c87e52fbb03be8cb6ecd7688af344e860f164e
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a082bb93ba47dad54beda7f6f4a70de91f9f8c01010082573f891f9d9311b5fa
+size 351841765
diff --git a/qwen3/sfkl/checkpoint-628/latest b/qwen3/sfkl/checkpoint-628/latest
new file mode 100644
index 0000000000000000000000000000000000000000..af055c3db05dcb2af541005f22308dbd946aba65
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/latest
@@ -0,0 +1 @@
+global_step628
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-628/rng_state_0.pth b/qwen3/sfkl/checkpoint-628/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..8d1102282696497be1f51f429f4a6ec0401c9a0c
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f11f7b754ad74bebc1e21ae5e6c0e976d8c5e40825631821ce967d3092bcc7f
+size 14917
diff --git a/qwen3/sfkl/checkpoint-628/rng_state_1.pth b/qwen3/sfkl/checkpoint-628/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..142a7daaadc9f55a8ed7a39ff52dcd1e7fda8e46
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4935c417414ce18576dc32f8cabdb0b7674f72ff1173958c2eb8bde409696aec
+size 14917
diff --git a/qwen3/sfkl/checkpoint-628/scheduler.pt b/qwen3/sfkl/checkpoint-628/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96c0cabcd99be4945ffbd7897ecc5367ac4beb28
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb823a2c51d7fb42dfe3b693a7159784285dbf38c5a797e6e73292096123ab7a
+size 1465
diff --git a/qwen3/sfkl/checkpoint-628/tokenizer.json b/qwen3/sfkl/checkpoint-628/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/sfkl/checkpoint-628/tokenizer_config.json b/qwen3/sfkl/checkpoint-628/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/sfkl/checkpoint-628/trainer_state.json b/qwen3/sfkl/checkpoint-628/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..722d937e599346d1827cf5a208fbcd7cc671bdfa
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/trainer_state.json
@@ -0,0 +1,281 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 2.0,
+ "eval_steps": 500,
+ "global_step": 628,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.2921956777572632,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.36003756523132324,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 1.6740401983261108,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.09387726187705994,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 1.1545500755310059,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.06687320470809936,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.42471569776535034,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.04825929999351501,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.34235191345214844,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.02988826632499695,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.2986029088497162,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.021525751054286956,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.27962905168533325,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.02630552351474762,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.173891082406044,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.010886980593204499,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 0.4828740358352661,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.012579935789108276,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.3599541485309601,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.018575572967529298,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.21371620893478394,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.012867593765258789,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.2939048111438751,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.011218706518411637,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.2609497010707855,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.0136981800198555,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.17881983518600464,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.008669006079435349,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.24963699281215668,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.008861982077360154,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 91.2967,
+ "eval_generator_rouge1": 99.5496,
+ "eval_generator_rouge2": 99.3396,
+ "eval_generator_rougeL": 99.2032,
+ "eval_loss": 0.007557725068181753,
+ "eval_runtime": 597.4488,
+ "eval_samples_per_second": 5.714,
+ "eval_selector_accuracy": 96.6121,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.179,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 1.7057307958602905,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.008970273286104202,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.1011928841471672,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.003804096579551697,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.33115050196647644,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.005674988031387329,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.21229985356330872,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.0043993469327688215,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.11725866794586182,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.006616537272930145,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.1508084237575531,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.0059788327664136885,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.034210484474897385,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.004834429547190666,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.36105096340179443,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.006534397602081299,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.34886789321899414,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.004452808201313019,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.07787973433732986,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.008046584576368332,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.05649268627166748,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.006670549511909485,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.16277413070201874,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.007858693599700928,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.016449013724923134,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.0019811514765024187,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.4171461760997772,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.009944957494735718,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.2326679229736328,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.008132340759038926,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.047354914247989655,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.0029657069593667985,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.4579,
+ "eval_generator_rouge1": 99.8096,
+ "eval_generator_rouge2": 99.6454,
+ "eval_generator_rougeL": 99.6441,
+ "eval_loss": 0.005621648393571377,
+ "eval_runtime": 595.9265,
+ "eval_samples_per_second": 5.729,
+ "eval_selector_accuracy": 97.6051,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.18,
+ "step": 628
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 2.346351770520781e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/sfkl/checkpoint-628/training_args.bin b/qwen3/sfkl/checkpoint-628/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..019105eaa7de05515b126bf422de20d1df4e85f5
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2d955bcf260cad259a2bda62ea641cce8e39b7cf407995d5f86c2c02b6e6f3e
+size 7185
diff --git a/qwen3/sfkl/checkpoint-628/zero_to_fp32.py b/qwen3/sfkl/checkpoint-628/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-628/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/sfkl/checkpoint-942/README.md b/qwen3/sfkl/checkpoint-942/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-942/adapter_config.json b/qwen3/sfkl/checkpoint-942/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..df150dce63edb909460ef4d599512e21d2dc45da
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "up_proj",
+ "q_proj",
+ "o_proj",
+ "gate_proj",
+ "k_proj",
+ "v_proj",
+ "down_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-942/adapter_model.safetensors b/qwen3/sfkl/checkpoint-942/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6b7feb5a4c46e54641161c7c4a570161d47affb1
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:891574a5e6241ec81821c2951e7d961341194a51df492ce13365d2feed1e87d8
+size 40422560
diff --git a/qwen3/sfkl/checkpoint-942/chat_template.jinja b/qwen3/sfkl/checkpoint-942/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8f1229ed13052aec3adb0f87a81c6bc4dfca1c80
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63d6644e44f107d73019160bd8711bbbdbc443a24323d1abc44dfbe212ff1e43
+size 121135557
diff --git a/qwen3/sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dce9591e2f361bc0eea1d1fb9f24a20f9221afca
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc0e4c22397343692081a1d3de06a87fa786b7f34a23e4372310fc809e000f00
+size 121135749
diff --git a/qwen3/sfkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt b/qwen3/sfkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..51ec3e48cc3805661d32621608049e67a5d44358
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5d1d85c718d5c8a6b5bdf9be68cc35f76796c5d9736bfd1138bb8e7e7bab274
+size 351841765
diff --git a/qwen3/sfkl/checkpoint-942/latest b/qwen3/sfkl/checkpoint-942/latest
new file mode 100644
index 0000000000000000000000000000000000000000..4e9a68e4460aca593c480611a486d2a4cd73ef42
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/latest
@@ -0,0 +1 @@
+global_step942
\ No newline at end of file
diff --git a/qwen3/sfkl/checkpoint-942/rng_state_0.pth b/qwen3/sfkl/checkpoint-942/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..dbf622c87164a3c0558198f15fd15b6be4c1caae
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d487677d4c4ee699f209a4fed9319b5b7dacb24f57bdd8ad13ff66bbc499b0d
+size 14917
diff --git a/qwen3/sfkl/checkpoint-942/rng_state_1.pth b/qwen3/sfkl/checkpoint-942/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d31b1b0347a74ed9cb7e593cafcfc53ef1781109
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f59b2d30c979cf05d397e2e8cf014fbefc0753346a44be996db3d081b7ef201
+size 14917
diff --git a/qwen3/sfkl/checkpoint-942/scheduler.pt b/qwen3/sfkl/checkpoint-942/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b1f8af0480bf83ab728abfeff11bc3288214349
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8076da6f897951c5bdf8649e4fba596e8c49819927746017442ffd85f40b18a3
+size 1465
diff --git a/qwen3/sfkl/checkpoint-942/tokenizer.json b/qwen3/sfkl/checkpoint-942/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/sfkl/checkpoint-942/tokenizer_config.json b/qwen3/sfkl/checkpoint-942/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/sfkl/checkpoint-942/trainer_state.json b/qwen3/sfkl/checkpoint-942/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..4115a9ccf5f3f1dc909ad7fdc56cc2e123d3054f
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/trainer_state.json
@@ -0,0 +1,408 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 3.0,
+ "eval_steps": 500,
+ "global_step": 942,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.2921956777572632,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.36003756523132324,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 1.6740401983261108,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.09387726187705994,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 1.1545500755310059,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.06687320470809936,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.42471569776535034,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.04825929999351501,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.34235191345214844,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.02988826632499695,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.2986029088497162,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.021525751054286956,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.27962905168533325,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.02630552351474762,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.173891082406044,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.010886980593204499,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 0.4828740358352661,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.012579935789108276,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.3599541485309601,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.018575572967529298,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.21371620893478394,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.012867593765258789,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.2939048111438751,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.011218706518411637,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.2609497010707855,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.0136981800198555,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.17881983518600464,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.008669006079435349,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.24963699281215668,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.008861982077360154,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 91.2967,
+ "eval_generator_rouge1": 99.5496,
+ "eval_generator_rouge2": 99.3396,
+ "eval_generator_rougeL": 99.2032,
+ "eval_loss": 0.007557725068181753,
+ "eval_runtime": 597.4488,
+ "eval_samples_per_second": 5.714,
+ "eval_selector_accuracy": 96.6121,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.179,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 1.7057307958602905,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.008970273286104202,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.1011928841471672,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.003804096579551697,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.33115050196647644,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.005674988031387329,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.21229985356330872,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.0043993469327688215,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.11725866794586182,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.006616537272930145,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.1508084237575531,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.0059788327664136885,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.034210484474897385,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.004834429547190666,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.36105096340179443,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.006534397602081299,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.34886789321899414,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.004452808201313019,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.07787973433732986,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.008046584576368332,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.05649268627166748,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.006670549511909485,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.16277413070201874,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.007858693599700928,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.016449013724923134,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.0019811514765024187,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.4171461760997772,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.009944957494735718,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.2326679229736328,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.008132340759038926,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.047354914247989655,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.0029657069593667985,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.4579,
+ "eval_generator_rouge1": 99.8096,
+ "eval_generator_rouge2": 99.6454,
+ "eval_generator_rougeL": 99.6441,
+ "eval_loss": 0.005621648393571377,
+ "eval_runtime": 595.9265,
+ "eval_samples_per_second": 5.729,
+ "eval_selector_accuracy": 97.6051,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.18,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.11734319478273392,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.0028536681085824965,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.20789793133735657,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.002620901167392731,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.12088560312986374,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0017223741859197616,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.0175655297935009,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0019577957689762117,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.1677847057580948,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.005092801898717881,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.027272474020719528,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.0032966256141662596,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.030720148235559464,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0015653561800718307,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.028172848746180534,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0025819912552833557,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.1057867631316185,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.005924087017774582,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.23805274069309235,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.004035309329628945,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.055235955864191055,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.0022109072655439378,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.07275807857513428,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.005412191897630691,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.036749616265296936,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0019010215997695922,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.11888282001018524,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0013746129348874091,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.004523377399891615,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.00334613136947155,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.08859030157327652,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0014281794428825378,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.4439,
+ "eval_generator_rouge1": 99.8682,
+ "eval_generator_rouge2": 99.7679,
+ "eval_generator_rougeL": 99.6703,
+ "eval_loss": 0.004075318109244108,
+ "eval_runtime": 595.0587,
+ "eval_samples_per_second": 5.737,
+ "eval_selector_accuracy": 98.0724,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.18,
+ "step": 942
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 3.524157743353037e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/sfkl/checkpoint-942/training_args.bin b/qwen3/sfkl/checkpoint-942/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..019105eaa7de05515b126bf422de20d1df4e85f5
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2d955bcf260cad259a2bda62ea641cce8e39b7cf407995d5f86c2c02b6e6f3e
+size 7185
diff --git a/qwen3/sfkl/checkpoint-942/zero_to_fp32.py b/qwen3/sfkl/checkpoint-942/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/sfkl/checkpoint-942/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/sfkl/eval_results.json b/qwen3/sfkl/eval_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..4f82da981714b63becad31f2318847e8adf7d464
--- /dev/null
+++ b/qwen3/sfkl/eval_results.json
@@ -0,0 +1,14 @@
+{
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.729,
+ "eval_generator_rouge1": 99.9412,
+ "eval_generator_rouge2": 99.8856,
+ "eval_generator_rougeL": 99.7768,
+ "eval_loss": 0.003214908065274358,
+ "eval_runtime": 598.5838,
+ "eval_samples_per_second": 5.703,
+ "eval_selector_accuracy": 98.715,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.179
+}
\ No newline at end of file
diff --git a/qwen3/sfkl/tokenizer.json b/qwen3/sfkl/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/sfkl/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/sfkl/tokenizer_config.json b/qwen3/sfkl/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/sfkl/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/sfkl/train_results.json b/qwen3/sfkl/train_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..5aa0b9a676a7b705e804475b8a353be36f50a167
--- /dev/null
+++ b/qwen3/sfkl/train_results.json
@@ -0,0 +1,8 @@
+{
+ "epoch": 5.0,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.012116143728493695,
+ "train_runtime": 7600.5401,
+ "train_samples_per_second": 6.596,
+ "train_steps_per_second": 0.207
+}
\ No newline at end of file
diff --git a/qwen3/sfkl/trainer_state.json b/qwen3/sfkl/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..420d7b0c40eb7e95b4dc068f33863229dda60bbf
--- /dev/null
+++ b/qwen3/sfkl/trainer_state.json
@@ -0,0 +1,664 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 5.0,
+ "eval_steps": 500,
+ "global_step": 1570,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.2921956777572632,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.36003756523132324,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 1.6740401983261108,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.09387726187705994,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 1.1545500755310059,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.06687320470809936,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.42471569776535034,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.04825929999351501,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.34235191345214844,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.02988826632499695,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.2986029088497162,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.021525751054286956,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.27962905168533325,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.02630552351474762,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.173891082406044,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.010886980593204499,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 0.4828740358352661,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.012579935789108276,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.3599541485309601,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.018575572967529298,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.21371620893478394,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.012867593765258789,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.2939048111438751,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.011218706518411637,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.2609497010707855,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.0136981800198555,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.17881983518600464,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.008669006079435349,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.24963699281215668,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.008861982077360154,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 91.2967,
+ "eval_generator_rouge1": 99.5496,
+ "eval_generator_rouge2": 99.3396,
+ "eval_generator_rougeL": 99.2032,
+ "eval_loss": 0.007557725068181753,
+ "eval_runtime": 597.4488,
+ "eval_samples_per_second": 5.714,
+ "eval_selector_accuracy": 96.6121,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.179,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 1.7057307958602905,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.008970273286104202,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.1011928841471672,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.003804096579551697,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.33115050196647644,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.005674988031387329,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.21229985356330872,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.0043993469327688215,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.11725866794586182,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.006616537272930145,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.1508084237575531,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.0059788327664136885,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.034210484474897385,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.004834429547190666,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.36105096340179443,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.006534397602081299,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.34886789321899414,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.004452808201313019,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.07787973433732986,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.008046584576368332,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.05649268627166748,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.006670549511909485,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.16277413070201874,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.007858693599700928,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.016449013724923134,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.0019811514765024187,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.4171461760997772,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.009944957494735718,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.2326679229736328,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.008132340759038926,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.047354914247989655,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.0029657069593667985,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.4579,
+ "eval_generator_rouge1": 99.8096,
+ "eval_generator_rouge2": 99.6454,
+ "eval_generator_rougeL": 99.6441,
+ "eval_loss": 0.005621648393571377,
+ "eval_runtime": 595.9265,
+ "eval_samples_per_second": 5.729,
+ "eval_selector_accuracy": 97.6051,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.18,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.11734319478273392,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.0028536681085824965,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.20789793133735657,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.002620901167392731,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.12088560312986374,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0017223741859197616,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.0175655297935009,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0019577957689762117,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.1677847057580948,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.005092801898717881,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.027272474020719528,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.0032966256141662596,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.030720148235559464,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0015653561800718307,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.028172848746180534,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0025819912552833557,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.1057867631316185,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.005924087017774582,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.23805274069309235,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.004035309329628945,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.055235955864191055,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.0022109072655439378,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.07275807857513428,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.005412191897630691,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.036749616265296936,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0019010215997695922,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.11888282001018524,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0013746129348874091,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.004523377399891615,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.00334613136947155,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.08859030157327652,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0014281794428825378,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.4439,
+ "eval_generator_rouge1": 99.8682,
+ "eval_generator_rouge2": 99.7679,
+ "eval_generator_rougeL": 99.6703,
+ "eval_loss": 0.004075318109244108,
+ "eval_runtime": 595.0587,
+ "eval_samples_per_second": 5.737,
+ "eval_selector_accuracy": 98.0724,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.18,
+ "step": 942
+ },
+ {
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.033190421760082245,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.003326282650232315,
+ "step": 960
+ },
+ {
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.038682322949171066,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.0027384284883737566,
+ "step": 980
+ },
+ {
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.024536851793527603,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.001315690018236637,
+ "step": 1000
+ },
+ {
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.02027392014861107,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.003594046086072922,
+ "step": 1020
+ },
+ {
+ "epoch": 3.3127243717590744,
+ "grad_norm": 0.26362860202789307,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.0022306183353066446,
+ "step": 1040
+ },
+ {
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.025666173547506332,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.002582985907793045,
+ "step": 1060
+ },
+ {
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.06274232268333435,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.003804165869951248,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.0051957834511995316,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0013254371471703053,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.08129020780324936,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.0011303042992949486,
+ "step": 1120
+ },
+ {
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.08928140252828598,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0009621990844607353,
+ "step": 1140
+ },
+ {
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.5374053716659546,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.004591869562864304,
+ "step": 1160
+ },
+ {
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.032263629138469696,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0023022303357720375,
+ "step": 1180
+ },
+ {
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.005962996743619442,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.002387513592839241,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8871160749900278,
+ "grad_norm": 0.07597373425960541,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.002432684227824211,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.025445526465773582,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0009138435125350952,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 97.021,
+ "eval_generator_rouge1": 99.95,
+ "eval_generator_rouge2": 99.9065,
+ "eval_generator_rougeL": 99.7835,
+ "eval_loss": 0.00319857569411397,
+ "eval_runtime": 604.8868,
+ "eval_samples_per_second": 5.644,
+ "eval_selector_accuracy": 98.5397,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.177,
+ "step": 1256
+ },
+ {
+ "epoch": 4.0127642600717985,
+ "grad_norm": 0.34829947352409363,
+ "learning_rate": 9.464081721960592e-06,
+ "loss": 0.0020675655454397202,
+ "step": 1260
+ },
+ {
+ "epoch": 4.076585560430794,
+ "grad_norm": 0.1925937831401825,
+ "learning_rate": 8.331628936475818e-06,
+ "loss": 0.0007737125270068645,
+ "step": 1280
+ },
+ {
+ "epoch": 4.140406860789788,
+ "grad_norm": 0.03017379343509674,
+ "learning_rate": 7.2659843568748385e-06,
+ "loss": 0.0011951016262173654,
+ "step": 1300
+ },
+ {
+ "epoch": 4.204228161148784,
+ "grad_norm": 0.007225691340863705,
+ "learning_rate": 6.268854516770008e-06,
+ "loss": 0.0011913912370800973,
+ "step": 1320
+ },
+ {
+ "epoch": 4.268049461507778,
+ "grad_norm": 0.012843134813010693,
+ "learning_rate": 5.341836229602331e-06,
+ "loss": 0.0011537913233041763,
+ "step": 1340
+ },
+ {
+ "epoch": 4.331870761866773,
+ "grad_norm": 0.6861193776130676,
+ "learning_rate": 4.4864140314888216e-06,
+ "loss": 0.0043139740824699405,
+ "step": 1360
+ },
+ {
+ "epoch": 4.395692062225768,
+ "grad_norm": 0.025450166314840317,
+ "learning_rate": 3.703957803871932e-06,
+ "loss": 0.001425867434591055,
+ "step": 1380
+ },
+ {
+ "epoch": 4.4595133625847625,
+ "grad_norm": 0.023695463314652443,
+ "learning_rate": 2.9957205797780495e-06,
+ "loss": 0.001892777904868126,
+ "step": 1400
+ },
+ {
+ "epoch": 4.523334662943758,
+ "grad_norm": 0.04371035099029541,
+ "learning_rate": 2.362836537198199e-06,
+ "loss": 0.0016116222366690635,
+ "step": 1420
+ },
+ {
+ "epoch": 4.587155963302752,
+ "grad_norm": 0.01301620528101921,
+ "learning_rate": 1.8063191828043703e-06,
+ "loss": 0.0010820812545716762,
+ "step": 1440
+ },
+ {
+ "epoch": 4.650977263661747,
+ "grad_norm": 0.03708479553461075,
+ "learning_rate": 1.3270597289100437e-06,
+ "loss": 0.000809190608561039,
+ "step": 1460
+ },
+ {
+ "epoch": 4.714798564020742,
+ "grad_norm": 0.0060765426605939865,
+ "learning_rate": 9.258256662741205e-07,
+ "loss": 0.0007033236790448428,
+ "step": 1480
+ },
+ {
+ "epoch": 4.778619864379737,
+ "grad_norm": 0.014266903512179852,
+ "learning_rate": 6.032595350337341e-07,
+ "loss": 0.0015078827738761902,
+ "step": 1500
+ },
+ {
+ "epoch": 4.842441164738732,
+ "grad_norm": 0.01251401286572218,
+ "learning_rate": 3.5987789573421356e-07,
+ "loss": 0.004571952670812607,
+ "step": 1520
+ },
+ {
+ "epoch": 4.9062624650977265,
+ "grad_norm": 0.1618906855583191,
+ "learning_rate": 1.9607050210401192e-07,
+ "loss": 0.0009092647582292557,
+ "step": 1540
+ },
+ {
+ "epoch": 4.970083765456721,
+ "grad_norm": 0.026854628697037697,
+ "learning_rate": 1.1209967689925832e-07,
+ "loss": 0.0015063435770571233,
+ "step": 1560
+ },
+ {
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.6121,
+ "eval_generator_rouge1": 99.9495,
+ "eval_generator_rouge2": 99.9016,
+ "eval_generator_rougeL": 99.764,
+ "eval_loss": 0.003214908065274358,
+ "eval_runtime": 612.4845,
+ "eval_samples_per_second": 5.574,
+ "eval_selector_accuracy": 98.7734,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 1570
+ },
+ {
+ "epoch": 5.0,
+ "step": 1570,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.012116143728493695,
+ "train_runtime": 7600.5401,
+ "train_samples_per_second": 6.596,
+ "train_steps_per_second": 0.207
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 5.871444840310374e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/sfkl/training_args.bin b/qwen3/sfkl/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..019105eaa7de05515b126bf422de20d1df4e85f5
--- /dev/null
+++ b/qwen3/sfkl/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2d955bcf260cad259a2bda62ea641cce8e39b7cf407995d5f86c2c02b6e6f3e
+size 7185
diff --git a/qwen3/sft/README.md b/qwen3/sft/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..af0dcb32e2eef9762a6bab10c7bc5a256e3d50cc
--- /dev/null
+++ b/qwen3/sft/README.md
@@ -0,0 +1,79 @@
+---
+library_name: peft
+license: other
+base_model: Qwen/Qwen3-0.6B
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+pipeline_tag: text-generation
+model-index:
+- name: sft
+ results: []
+---
+
+
+
+# sft
+
+This model is a fine-tuned version of [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) on the cypher_prepared_train dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0033
+- Selector Count: 1712.0
+- Generator Count: 1712.0
+- Selector Accuracy: 99.2407
+- Generator Exact Match: 96.9626
+- Generator Rouge1: 99.8875
+- Generator Rouge2: 99.8292
+- Generator Rougel: 99.735
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 2
+- eval_batch_size: 16
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine_with_min_lr
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Selector Count | Generator Count | Selector Accuracy | Generator Exact Match | Generator Rouge1 | Generator Rouge2 | Generator Rougel |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:---------------:|:-----------------:|:---------------------:|:----------------:|:----------------:|:----------------:|
+| 0.0109 | 1.0 | 314 | 0.0083 | 1712.0 | 1712.0 | 96.7874 | 88.3178 | 99.4774 | 99.1631 | 99.0528 |
+| 0.0042 | 2.0 | 628 | 0.0055 | 1712.0 | 1712.0 | 97.7804 | 93.0491 | 99.6708 | 99.4934 | 99.3984 |
+| 0.0016 | 3.0 | 942 | 0.0044 | 1712.0 | 1712.0 | 96.6706 | 95.1519 | 99.8526 | 99.7382 | 99.647 |
+| 0.0006 | 4.0 | 1256 | 0.0034 | 1712.0 | 1712.0 | 98.7734 | 95.7944 | 99.8712 | 99.7877 | 99.6495 |
+| 0.0007 | 5.0 | 1570 | 0.0033 | 1712.0 | 1712.0 | 99.1238 | 96.0864 | 99.8772 | 99.805 | 99.679 |
+
+
+### Framework versions
+
+- PEFT 0.18.1
+- Transformers 5.8.0
+- Pytorch 2.9.1+cu128
+- Datasets 4.0.0
+- Tokenizers 0.22.2
\ No newline at end of file
diff --git a/qwen3/sft/adapter_config.json b/qwen3/sft/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9da5b578a766fe6816cb69301e1d55e3bea62cf
--- /dev/null
+++ b/qwen3/sft/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "o_proj",
+ "k_proj",
+ "up_proj",
+ "gate_proj",
+ "q_proj",
+ "down_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/sft/adapter_model.safetensors b/qwen3/sft/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..28ac5e768b6dc52841e4caa347ba9331aa28ae9f
--- /dev/null
+++ b/qwen3/sft/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:310de365315a8e6ffbe9c11f764e5907a0ef6b0d1ffdbcd3b0e19b428a60454a
+size 40422560
diff --git a/qwen3/sft/all_results.json b/qwen3/sft/all_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..240b857f3bb7bef6a0e80c32a79d568c1d96ab3a
--- /dev/null
+++ b/qwen3/sft/all_results.json
@@ -0,0 +1,19 @@
+{
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.9626,
+ "eval_generator_rouge1": 99.8875,
+ "eval_generator_rouge2": 99.8292,
+ "eval_generator_rougeL": 99.735,
+ "eval_loss": 0.003336928552016616,
+ "eval_runtime": 603.5681,
+ "eval_samples_per_second": 5.656,
+ "eval_selector_accuracy": 99.2407,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.177,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.016630506924513705,
+ "train_runtime": 6427.6125,
+ "train_samples_per_second": 7.8,
+ "train_steps_per_second": 0.244
+}
\ No newline at end of file
diff --git a/qwen3/sft/chat_template.jinja b/qwen3/sft/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/sft/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-1256/README.md b/qwen3/sft/checkpoint-1256/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-1256/adapter_config.json b/qwen3/sft/checkpoint-1256/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9da5b578a766fe6816cb69301e1d55e3bea62cf
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "o_proj",
+ "k_proj",
+ "up_proj",
+ "gate_proj",
+ "q_proj",
+ "down_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-1256/adapter_model.safetensors b/qwen3/sft/checkpoint-1256/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..29951b129cc187105bc91504cbcf10cc7360de46
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2008ea1109f6e615f474c9896f15255561f51da35e5cd1fc6dab3e0a33e1e85
+size 40422560
diff --git a/qwen3/sft/checkpoint-1256/chat_template.jinja b/qwen3/sft/checkpoint-1256/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/sft/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..031271deba52d27091c34af2b15e8d89547ee267
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:460e78f5cbee6a2aa3270152ab629d58aa42e8a29d3491df118304d53bdfd134
+size 121135557
diff --git a/qwen3/sft/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/sft/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94b02cf8b4f8074f93fdb0747ffdebba83ce48b9
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f597c347d82bdad704fa6f4f0e0868627d9e14e1d6c74d9a0b57455e9b629e4
+size 121135749
diff --git a/qwen3/sft/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt b/qwen3/sft/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1893f2ca60741830209602f7f9db064b68d1b49c
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f0bad4cf43db9a071345c1db274458c3ef8c4b517464bf35b6249df782dcd0e
+size 351841765
diff --git a/qwen3/sft/checkpoint-1256/latest b/qwen3/sft/checkpoint-1256/latest
new file mode 100644
index 0000000000000000000000000000000000000000..74abb3d23ea83ebdac59918752db5d1782834ca7
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/latest
@@ -0,0 +1 @@
+global_step1256
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-1256/rng_state_0.pth b/qwen3/sft/checkpoint-1256/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..f1e19fd5d528cca02b46816bc4dae31be2a5babc
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd58cff16a4b5917c2ebc4b32f742b57d69be86c508b39450aaedb1167d78473
+size 14917
diff --git a/qwen3/sft/checkpoint-1256/rng_state_1.pth b/qwen3/sft/checkpoint-1256/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..3f33cd9ffac1d85966a401c286f5a4436b18d95b
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86366ddf01699ca05ff173c8a1c8fc28b191a24a0253dbcfaf4fe894fcbc3776
+size 14917
diff --git a/qwen3/sft/checkpoint-1256/scheduler.pt b/qwen3/sft/checkpoint-1256/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40e863d07cfed22cd912014eb3ba3fd2b16b173f
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1775297f14e75f7463ae51983a66296f81f4b2e2e5d7234f1a6d295abfa4cf23
+size 1465
diff --git a/qwen3/sft/checkpoint-1256/tokenizer.json b/qwen3/sft/checkpoint-1256/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/sft/checkpoint-1256/tokenizer_config.json b/qwen3/sft/checkpoint-1256/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/sft/checkpoint-1256/trainer_state.json b/qwen3/sft/checkpoint-1256/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..e31333db7f67d66891e1b38aee35c510160b7ba5
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/trainer_state.json
@@ -0,0 +1,528 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 4.0,
+ "eval_steps": 500,
+ "global_step": 1256,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.8955494165420532,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.567183542251587,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 1.35407292842865,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.12746150493621827,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 0.9626659154891968,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.08457614183425903,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.4221804440021515,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.060509908199310306,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.4434971511363983,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.04329058527946472,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.7847144603729248,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.0370758205652237,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.30355238914489746,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.04995985627174378,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.598273754119873,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.03158450722694397,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 1.9779328107833862,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.0206314280629158,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.25729191303253174,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.021701553463935853,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.21868865191936493,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.02294553518295288,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 1.2186007499694824,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.013198558986186982,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.31176164746284485,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.013272044062614442,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.15429984033107758,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.011466830968856812,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.38197460770606995,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.010920770466327667,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 88.3178,
+ "eval_generator_rouge1": 99.4774,
+ "eval_generator_rouge2": 99.1631,
+ "eval_generator_rougeL": 99.0528,
+ "eval_loss": 0.008339357562363148,
+ "eval_runtime": 584.2228,
+ "eval_samples_per_second": 5.844,
+ "eval_selector_accuracy": 96.7874,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 0.4359477758407593,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.00993276685476303,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.13242340087890625,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.005089616030454635,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.13412831723690033,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.005343190580606461,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.2253202497959137,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.0047687780112028125,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.27512186765670776,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.0066933780908584595,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.1620800793170929,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.01029568463563919,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.153878852725029,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.005285752564668655,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.9735704064369202,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.006016410514712333,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.05531628057360649,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.0033036738634109495,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.08567451685667038,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.004241976141929627,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.07484884560108185,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.010447616130113602,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.38432225584983826,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.012001012265682221,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.2757236957550049,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.004458250850439072,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 1.082757830619812,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.011233760416507721,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.8778036236763,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.006709276884794235,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.1680862009525299,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.004196321591734886,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.0491,
+ "eval_generator_rouge1": 99.6708,
+ "eval_generator_rouge2": 99.4934,
+ "eval_generator_rougeL": 99.3984,
+ "eval_loss": 0.0054823532700538635,
+ "eval_runtime": 581.9182,
+ "eval_samples_per_second": 5.867,
+ "eval_selector_accuracy": 97.7804,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.184,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.2708258032798767,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.003280312567949295,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.1907150149345398,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.00262494757771492,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.0759034976363182,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0016721963882446289,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.8344013690948486,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0019289907068014145,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.08497262001037598,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.0021992195397615433,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.11710834503173828,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.00548185333609581,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.02737243101000786,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0029599376022815703,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.031406909227371216,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0012881334871053696,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.1769132763147354,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.004978828132152557,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.5065577030181885,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.0028983473777770994,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.13216915726661682,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.002082060277462006,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.08058275282382965,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.0047489568591117855,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.2788284122943878,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0014096658676862716,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.6501541137695312,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0017810370773077011,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.007601048331707716,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.002541198208928108,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.12641772627830505,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0016433246433734893,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.1519,
+ "eval_generator_rouge1": 99.8526,
+ "eval_generator_rouge2": 99.7382,
+ "eval_generator_rougeL": 99.647,
+ "eval_loss": 0.0044273375533521175,
+ "eval_runtime": 583.9099,
+ "eval_samples_per_second": 5.847,
+ "eval_selector_accuracy": 96.6706,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 942
+ },
+ {
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.06881710141897202,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.001966923289000988,
+ "step": 960
+ },
+ {
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.004793599247932434,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.001482640765607357,
+ "step": 980
+ },
+ {
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.005230629350990057,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.0005885613150894642,
+ "step": 1000
+ },
+ {
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.0802706778049469,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.0016000797972083093,
+ "step": 1020
+ },
+ {
+ "epoch": 3.3127243717590744,
+ "grad_norm": 0.04865531995892525,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.0007603847421705723,
+ "step": 1040
+ },
+ {
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.002228009281679988,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.0016863197088241577,
+ "step": 1060
+ },
+ {
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.12053167074918747,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.0008015411905944348,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.004856751766055822,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0011467586271464824,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.03997187688946724,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.001309672836214304,
+ "step": 1120
+ },
+ {
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.04420069605112076,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0008654085919260979,
+ "step": 1140
+ },
+ {
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.011205613613128662,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.004080542176961899,
+ "step": 1160
+ },
+ {
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.015621992759406567,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0017238393425941466,
+ "step": 1180
+ },
+ {
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.028632311150431633,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.0011840559542179107,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8871160749900278,
+ "grad_norm": 0.19821636378765106,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.001461276412010193,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.026029646396636963,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0006165837403386831,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.7944,
+ "eval_generator_rouge1": 99.8712,
+ "eval_generator_rouge2": 99.7877,
+ "eval_generator_rougeL": 99.6495,
+ "eval_loss": 0.003426080336794257,
+ "eval_runtime": 585.1791,
+ "eval_samples_per_second": 5.834,
+ "eval_selector_accuracy": 98.7734,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 1256
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 4.697643190059008e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/sft/checkpoint-1256/training_args.bin b/qwen3/sft/checkpoint-1256/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3903357a542bc3a9ccd098cf573248a02728c6e9
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:601ad253ebc37cfd18e26c1b1ece08fff37c9b901912b103fdc8fa017ec8f0e1
+size 7185
diff --git a/qwen3/sft/checkpoint-1256/zero_to_fp32.py b/qwen3/sft/checkpoint-1256/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/sft/checkpoint-1256/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/sft/checkpoint-1570/README.md b/qwen3/sft/checkpoint-1570/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-1570/adapter_config.json b/qwen3/sft/checkpoint-1570/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9da5b578a766fe6816cb69301e1d55e3bea62cf
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "o_proj",
+ "k_proj",
+ "up_proj",
+ "gate_proj",
+ "q_proj",
+ "down_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-1570/adapter_model.safetensors b/qwen3/sft/checkpoint-1570/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..28ac5e768b6dc52841e4caa347ba9331aa28ae9f
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:310de365315a8e6ffbe9c11f764e5907a0ef6b0d1ffdbcd3b0e19b428a60454a
+size 40422560
diff --git a/qwen3/sft/checkpoint-1570/chat_template.jinja b/qwen3/sft/checkpoint-1570/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/sft/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cbb9a784b1f3e9bae10d2a2d7e4308718481f989
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bb3137af9bbfc59a367bc94a26e988370da99111bb0c2ea7572223cf9104fcf
+size 121135557
diff --git a/qwen3/sft/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/sft/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..810097c2a4f0883904464b3fe018a08d9938af30
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:978d2482de32e24b9f87a6d81342a13dc5b46fbd024013f4c2d14a86b8fd7158
+size 121135749
diff --git a/qwen3/sft/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt b/qwen3/sft/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..114e555858d37bc2c9b11069b7c6aa2b88b87ded
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3109dfe9559aa169c56ec173fa47848040552598f19da8d68999bf0b58693a4d
+size 351841765
diff --git a/qwen3/sft/checkpoint-1570/latest b/qwen3/sft/checkpoint-1570/latest
new file mode 100644
index 0000000000000000000000000000000000000000..18c2fcff58fa6daa0a2e7c3a015552a69085778b
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/latest
@@ -0,0 +1 @@
+global_step1570
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-1570/rng_state_0.pth b/qwen3/sft/checkpoint-1570/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..2e87dec4fe4fd7f9f745eb295d5f23a457d11421
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99e13558b40b0b1e2648bfb3ae1ea7e215262e0eb527934843a2095dc073d53d
+size 14917
diff --git a/qwen3/sft/checkpoint-1570/rng_state_1.pth b/qwen3/sft/checkpoint-1570/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4cf9eb27038fc0a9b07e51b62a771436c7b74706
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:503012c65b1129d2b115b95779581b6120d304ac590fd78e8f8643de6ac8f850
+size 14917
diff --git a/qwen3/sft/checkpoint-1570/scheduler.pt b/qwen3/sft/checkpoint-1570/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e22954463492d8a070ae427c95418df61893c54c
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6332fac17ae99a07597512c705658da2fd8429d1c39bb776b30e107c2cb7aae
+size 1465
diff --git a/qwen3/sft/checkpoint-1570/tokenizer.json b/qwen3/sft/checkpoint-1570/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/sft/checkpoint-1570/tokenizer_config.json b/qwen3/sft/checkpoint-1570/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/sft/checkpoint-1570/trainer_state.json b/qwen3/sft/checkpoint-1570/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..1bca066f1ca84248d98ec81ddb7322d5c73dcbee
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/trainer_state.json
@@ -0,0 +1,655 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 5.0,
+ "eval_steps": 500,
+ "global_step": 1570,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.8955494165420532,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.567183542251587,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 1.35407292842865,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.12746150493621827,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 0.9626659154891968,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.08457614183425903,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.4221804440021515,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.060509908199310306,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.4434971511363983,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.04329058527946472,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.7847144603729248,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.0370758205652237,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.30355238914489746,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.04995985627174378,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.598273754119873,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.03158450722694397,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 1.9779328107833862,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.0206314280629158,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.25729191303253174,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.021701553463935853,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.21868865191936493,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.02294553518295288,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 1.2186007499694824,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.013198558986186982,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.31176164746284485,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.013272044062614442,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.15429984033107758,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.011466830968856812,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.38197460770606995,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.010920770466327667,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 88.3178,
+ "eval_generator_rouge1": 99.4774,
+ "eval_generator_rouge2": 99.1631,
+ "eval_generator_rougeL": 99.0528,
+ "eval_loss": 0.008339357562363148,
+ "eval_runtime": 584.2228,
+ "eval_samples_per_second": 5.844,
+ "eval_selector_accuracy": 96.7874,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 0.4359477758407593,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.00993276685476303,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.13242340087890625,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.005089616030454635,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.13412831723690033,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.005343190580606461,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.2253202497959137,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.0047687780112028125,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.27512186765670776,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.0066933780908584595,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.1620800793170929,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.01029568463563919,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.153878852725029,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.005285752564668655,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.9735704064369202,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.006016410514712333,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.05531628057360649,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.0033036738634109495,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.08567451685667038,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.004241976141929627,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.07484884560108185,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.010447616130113602,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.38432225584983826,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.012001012265682221,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.2757236957550049,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.004458250850439072,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 1.082757830619812,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.011233760416507721,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.8778036236763,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.006709276884794235,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.1680862009525299,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.004196321591734886,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.0491,
+ "eval_generator_rouge1": 99.6708,
+ "eval_generator_rouge2": 99.4934,
+ "eval_generator_rougeL": 99.3984,
+ "eval_loss": 0.0054823532700538635,
+ "eval_runtime": 581.9182,
+ "eval_samples_per_second": 5.867,
+ "eval_selector_accuracy": 97.7804,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.184,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.2708258032798767,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.003280312567949295,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.1907150149345398,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.00262494757771492,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.0759034976363182,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0016721963882446289,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.8344013690948486,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0019289907068014145,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.08497262001037598,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.0021992195397615433,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.11710834503173828,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.00548185333609581,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.02737243101000786,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0029599376022815703,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.031406909227371216,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0012881334871053696,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.1769132763147354,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.004978828132152557,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.5065577030181885,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.0028983473777770994,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.13216915726661682,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.002082060277462006,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.08058275282382965,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.0047489568591117855,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.2788284122943878,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0014096658676862716,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.6501541137695312,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0017810370773077011,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.007601048331707716,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.002541198208928108,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.12641772627830505,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0016433246433734893,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.1519,
+ "eval_generator_rouge1": 99.8526,
+ "eval_generator_rouge2": 99.7382,
+ "eval_generator_rougeL": 99.647,
+ "eval_loss": 0.0044273375533521175,
+ "eval_runtime": 583.9099,
+ "eval_samples_per_second": 5.847,
+ "eval_selector_accuracy": 96.6706,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 942
+ },
+ {
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.06881710141897202,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.001966923289000988,
+ "step": 960
+ },
+ {
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.004793599247932434,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.001482640765607357,
+ "step": 980
+ },
+ {
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.005230629350990057,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.0005885613150894642,
+ "step": 1000
+ },
+ {
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.0802706778049469,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.0016000797972083093,
+ "step": 1020
+ },
+ {
+ "epoch": 3.3127243717590744,
+ "grad_norm": 0.04865531995892525,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.0007603847421705723,
+ "step": 1040
+ },
+ {
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.002228009281679988,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.0016863197088241577,
+ "step": 1060
+ },
+ {
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.12053167074918747,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.0008015411905944348,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.004856751766055822,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0011467586271464824,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.03997187688946724,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.001309672836214304,
+ "step": 1120
+ },
+ {
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.04420069605112076,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0008654085919260979,
+ "step": 1140
+ },
+ {
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.011205613613128662,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.004080542176961899,
+ "step": 1160
+ },
+ {
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.015621992759406567,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0017238393425941466,
+ "step": 1180
+ },
+ {
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.028632311150431633,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.0011840559542179107,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8871160749900278,
+ "grad_norm": 0.19821636378765106,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.001461276412010193,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.026029646396636963,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0006165837403386831,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.7944,
+ "eval_generator_rouge1": 99.8712,
+ "eval_generator_rouge2": 99.7877,
+ "eval_generator_rougeL": 99.6495,
+ "eval_loss": 0.003426080336794257,
+ "eval_runtime": 585.1791,
+ "eval_samples_per_second": 5.834,
+ "eval_selector_accuracy": 98.7734,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 1256
+ },
+ {
+ "epoch": 4.0127642600717985,
+ "grad_norm": 0.06868977099657059,
+ "learning_rate": 9.464081721960592e-06,
+ "loss": 0.001044323481619358,
+ "step": 1260
+ },
+ {
+ "epoch": 4.076585560430794,
+ "grad_norm": 0.1312812864780426,
+ "learning_rate": 8.331628936475818e-06,
+ "loss": 0.00047424803487956525,
+ "step": 1280
+ },
+ {
+ "epoch": 4.140406860789788,
+ "grad_norm": 0.0014219481963664293,
+ "learning_rate": 7.2659843568748385e-06,
+ "loss": 0.0009649133309721947,
+ "step": 1300
+ },
+ {
+ "epoch": 4.204228161148784,
+ "grad_norm": 0.016619998961687088,
+ "learning_rate": 6.268854516770008e-06,
+ "loss": 0.0006559681612998247,
+ "step": 1320
+ },
+ {
+ "epoch": 4.268049461507778,
+ "grad_norm": 0.057871364057064056,
+ "learning_rate": 5.341836229602331e-06,
+ "loss": 0.0012834925204515458,
+ "step": 1340
+ },
+ {
+ "epoch": 4.331870761866773,
+ "grad_norm": 0.005151741206645966,
+ "learning_rate": 4.4864140314888216e-06,
+ "loss": 0.0005498498678207398,
+ "step": 1360
+ },
+ {
+ "epoch": 4.395692062225768,
+ "grad_norm": 0.015129027888178825,
+ "learning_rate": 3.703957803871932e-06,
+ "loss": 0.000982836540788412,
+ "step": 1380
+ },
+ {
+ "epoch": 4.4595133625847625,
+ "grad_norm": 0.0474361851811409,
+ "learning_rate": 2.9957205797780495e-06,
+ "loss": 0.0009566731750965118,
+ "step": 1400
+ },
+ {
+ "epoch": 4.523334662943758,
+ "grad_norm": 0.079148069024086,
+ "learning_rate": 2.362836537198199e-06,
+ "loss": 0.0007865218445658684,
+ "step": 1420
+ },
+ {
+ "epoch": 4.587155963302752,
+ "grad_norm": 0.02590208128094673,
+ "learning_rate": 1.8063191828043703e-06,
+ "loss": 0.00046601640060544013,
+ "step": 1440
+ },
+ {
+ "epoch": 4.650977263661747,
+ "grad_norm": 0.028924427926540375,
+ "learning_rate": 1.3270597289100437e-06,
+ "loss": 0.0005176489241421223,
+ "step": 1460
+ },
+ {
+ "epoch": 4.714798564020742,
+ "grad_norm": 0.012378301471471786,
+ "learning_rate": 9.258256662741205e-07,
+ "loss": 0.0003302923170849681,
+ "step": 1480
+ },
+ {
+ "epoch": 4.778619864379737,
+ "grad_norm": 0.04462262615561485,
+ "learning_rate": 6.032595350337341e-07,
+ "loss": 0.0010699938982725143,
+ "step": 1500
+ },
+ {
+ "epoch": 4.842441164738732,
+ "grad_norm": 0.00910312868654728,
+ "learning_rate": 3.5987789573421356e-07,
+ "loss": 0.003197905048727989,
+ "step": 1520
+ },
+ {
+ "epoch": 4.9062624650977265,
+ "grad_norm": 0.13681654632091522,
+ "learning_rate": 1.9607050210401192e-07,
+ "loss": 0.0005365588702261447,
+ "step": 1540
+ },
+ {
+ "epoch": 4.970083765456721,
+ "grad_norm": 0.05481738969683647,
+ "learning_rate": 1.1209967689925832e-07,
+ "loss": 0.000743288267403841,
+ "step": 1560
+ },
+ {
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.0864,
+ "eval_generator_rouge1": 99.8772,
+ "eval_generator_rouge2": 99.805,
+ "eval_generator_rougeL": 99.679,
+ "eval_loss": 0.003336928552016616,
+ "eval_runtime": 638.4465,
+ "eval_samples_per_second": 5.347,
+ "eval_selector_accuracy": 99.1238,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.168,
+ "step": 1570
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 5.871444840310374e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/sft/checkpoint-1570/training_args.bin b/qwen3/sft/checkpoint-1570/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3903357a542bc3a9ccd098cf573248a02728c6e9
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:601ad253ebc37cfd18e26c1b1ece08fff37c9b901912b103fdc8fa017ec8f0e1
+size 7185
diff --git a/qwen3/sft/checkpoint-1570/zero_to_fp32.py b/qwen3/sft/checkpoint-1570/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/sft/checkpoint-1570/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/sft/checkpoint-314/README.md b/qwen3/sft/checkpoint-314/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-314/adapter_config.json b/qwen3/sft/checkpoint-314/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9da5b578a766fe6816cb69301e1d55e3bea62cf
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "o_proj",
+ "k_proj",
+ "up_proj",
+ "gate_proj",
+ "q_proj",
+ "down_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-314/adapter_model.safetensors b/qwen3/sft/checkpoint-314/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2a5efacb0374e0d2d0c3ffc2fb51c216442c1ee5
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81077ae083ef8bbf8bf15a0a6d9e162ba52f8a0ac52da3f6bf08d031948d12f3
+size 40422560
diff --git a/qwen3/sft/checkpoint-314/chat_template.jinja b/qwen3/sft/checkpoint-314/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/sft/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c55bf88aa5403cace6f70632efaa95799960972b
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74d782efbeb244af8b938bf135c7c63c39f276a45a266ce138262806dc6c6a1a
+size 121135557
diff --git a/qwen3/sft/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/sft/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fe61d06058f16452fd94e7cd4ca414e4d0137900
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d61325851204aea95808e293b63cefee87d4b09fbe8a0a2543fb8f1b3fdf5805
+size 121135749
diff --git a/qwen3/sft/checkpoint-314/global_step314/mp_rank_00_model_states.pt b/qwen3/sft/checkpoint-314/global_step314/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..02716c7740e4d5626b30432184ab569b16e146b1
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/global_step314/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:722b5268a772aad08749bcab7e51bb0759f7ce67342c2bab67c4a2c4d46e91e5
+size 351841765
diff --git a/qwen3/sft/checkpoint-314/latest b/qwen3/sft/checkpoint-314/latest
new file mode 100644
index 0000000000000000000000000000000000000000..a9786513b1a514db2c3a9da7408844635663d55d
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/latest
@@ -0,0 +1 @@
+global_step314
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-314/rng_state_0.pth b/qwen3/sft/checkpoint-314/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..f21a9884f104eb9e3d222eda5316c574d16b3197
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4bfabc29676cf06a1f6511ad9919a1150f271d08d0068f05b67d9faf17ce12b
+size 14917
diff --git a/qwen3/sft/checkpoint-314/rng_state_1.pth b/qwen3/sft/checkpoint-314/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..24a16e0f884f7a88c2cc7078a1b9cce3e4571f38
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cb933712030df150db4f4e3424f7a3e849097a1429bd812e1834837079a0245
+size 14917
diff --git a/qwen3/sft/checkpoint-314/scheduler.pt b/qwen3/sft/checkpoint-314/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3cb460802cda03bf0833a98a6f1428a3fd4f585
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17e254b5642bb75983fd5105bec79b35376cfbc99d29702287668b8767d46149
+size 1465
diff --git a/qwen3/sft/checkpoint-314/tokenizer.json b/qwen3/sft/checkpoint-314/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/sft/checkpoint-314/tokenizer_config.json b/qwen3/sft/checkpoint-314/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/sft/checkpoint-314/trainer_state.json b/qwen3/sft/checkpoint-314/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..3dbe65e77c27eacd4d1e5ae4e94a0dfb4f72daee
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/trainer_state.json
@@ -0,0 +1,154 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 1.0,
+ "eval_steps": 500,
+ "global_step": 314,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.8955494165420532,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.567183542251587,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 1.35407292842865,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.12746150493621827,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 0.9626659154891968,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.08457614183425903,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.4221804440021515,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.060509908199310306,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.4434971511363983,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.04329058527946472,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.7847144603729248,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.0370758205652237,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.30355238914489746,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.04995985627174378,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.598273754119873,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.03158450722694397,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 1.9779328107833862,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.0206314280629158,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.25729191303253174,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.021701553463935853,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.21868865191936493,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.02294553518295288,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 1.2186007499694824,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.013198558986186982,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.31176164746284485,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.013272044062614442,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.15429984033107758,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.011466830968856812,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.38197460770606995,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.010920770466327667,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 88.3178,
+ "eval_generator_rouge1": 99.4774,
+ "eval_generator_rouge2": 99.1631,
+ "eval_generator_rougeL": 99.0528,
+ "eval_loss": 0.008339357562363148,
+ "eval_runtime": 584.2228,
+ "eval_samples_per_second": 5.844,
+ "eval_selector_accuracy": 96.7874,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 314
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 1.172842003562496e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/sft/checkpoint-314/training_args.bin b/qwen3/sft/checkpoint-314/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3903357a542bc3a9ccd098cf573248a02728c6e9
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:601ad253ebc37cfd18e26c1b1ece08fff37c9b901912b103fdc8fa017ec8f0e1
+size 7185
diff --git a/qwen3/sft/checkpoint-314/zero_to_fp32.py b/qwen3/sft/checkpoint-314/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/sft/checkpoint-314/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/sft/checkpoint-628/README.md b/qwen3/sft/checkpoint-628/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-628/adapter_config.json b/qwen3/sft/checkpoint-628/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9da5b578a766fe6816cb69301e1d55e3bea62cf
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "o_proj",
+ "k_proj",
+ "up_proj",
+ "gate_proj",
+ "q_proj",
+ "down_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-628/adapter_model.safetensors b/qwen3/sft/checkpoint-628/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..55a79e63be198be1cf5c42bdb56868ffe72d41cb
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b8861ed532f1b222df48002705c2945249bc044f0df329120b7b634f2bba048
+size 40422560
diff --git a/qwen3/sft/checkpoint-628/chat_template.jinja b/qwen3/sft/checkpoint-628/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/sft/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..74860a9869a49fefe9ae9ffd5c9ebe72737d6e63
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8fe980527ce59fd739126ee61b57b5f998e110b4b4ddea0c1fd8a53ba0ad905
+size 121135557
diff --git a/qwen3/sft/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/sft/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4db6f79b6f58c1de67c75e225523d87ae9313279
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88338efc204a479a68bfc06b39ed1179e41b7345c49c38c7ea7d7dbc843e55bd
+size 121135749
diff --git a/qwen3/sft/checkpoint-628/global_step628/mp_rank_00_model_states.pt b/qwen3/sft/checkpoint-628/global_step628/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3948944995c842b5475212cfe69ccc8ab5ddad8b
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/global_step628/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66b6628967e7697a4cdd29b5abbb46f35d52b74270c67813f810edfa8210cddf
+size 351841765
diff --git a/qwen3/sft/checkpoint-628/latest b/qwen3/sft/checkpoint-628/latest
new file mode 100644
index 0000000000000000000000000000000000000000..af055c3db05dcb2af541005f22308dbd946aba65
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/latest
@@ -0,0 +1 @@
+global_step628
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-628/rng_state_0.pth b/qwen3/sft/checkpoint-628/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..6abc7e49040503e6fc3e95c189a3f692f34e6409
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14e07af6c2c976975b751f15ab748182eefe2d14ce4f9386405a1f882f23ccc3
+size 14917
diff --git a/qwen3/sft/checkpoint-628/rng_state_1.pth b/qwen3/sft/checkpoint-628/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b5e43a1572c103cf86890a10b317cc7f22c206ad
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9186e1a3dced9b795d7e5e772b0497e9309ff183770f172c7874a84b971dd68c
+size 14917
diff --git a/qwen3/sft/checkpoint-628/scheduler.pt b/qwen3/sft/checkpoint-628/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96c0cabcd99be4945ffbd7897ecc5367ac4beb28
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb823a2c51d7fb42dfe3b693a7159784285dbf38c5a797e6e73292096123ab7a
+size 1465
diff --git a/qwen3/sft/checkpoint-628/tokenizer.json b/qwen3/sft/checkpoint-628/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/sft/checkpoint-628/tokenizer_config.json b/qwen3/sft/checkpoint-628/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/sft/checkpoint-628/trainer_state.json b/qwen3/sft/checkpoint-628/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..6615160539a65ef433770cf34aac75a54a15fdb3
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/trainer_state.json
@@ -0,0 +1,281 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 2.0,
+ "eval_steps": 500,
+ "global_step": 628,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.8955494165420532,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.567183542251587,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 1.35407292842865,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.12746150493621827,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 0.9626659154891968,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.08457614183425903,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.4221804440021515,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.060509908199310306,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.4434971511363983,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.04329058527946472,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.7847144603729248,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.0370758205652237,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.30355238914489746,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.04995985627174378,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.598273754119873,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.03158450722694397,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 1.9779328107833862,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.0206314280629158,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.25729191303253174,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.021701553463935853,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.21868865191936493,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.02294553518295288,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 1.2186007499694824,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.013198558986186982,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.31176164746284485,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.013272044062614442,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.15429984033107758,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.011466830968856812,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.38197460770606995,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.010920770466327667,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 88.3178,
+ "eval_generator_rouge1": 99.4774,
+ "eval_generator_rouge2": 99.1631,
+ "eval_generator_rougeL": 99.0528,
+ "eval_loss": 0.008339357562363148,
+ "eval_runtime": 584.2228,
+ "eval_samples_per_second": 5.844,
+ "eval_selector_accuracy": 96.7874,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 0.4359477758407593,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.00993276685476303,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.13242340087890625,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.005089616030454635,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.13412831723690033,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.005343190580606461,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.2253202497959137,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.0047687780112028125,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.27512186765670776,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.0066933780908584595,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.1620800793170929,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.01029568463563919,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.153878852725029,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.005285752564668655,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.9735704064369202,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.006016410514712333,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.05531628057360649,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.0033036738634109495,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.08567451685667038,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.004241976141929627,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.07484884560108185,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.010447616130113602,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.38432225584983826,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.012001012265682221,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.2757236957550049,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.004458250850439072,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 1.082757830619812,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.011233760416507721,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.8778036236763,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.006709276884794235,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.1680862009525299,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.004196321591734886,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.0491,
+ "eval_generator_rouge1": 99.6708,
+ "eval_generator_rouge2": 99.4934,
+ "eval_generator_rougeL": 99.3984,
+ "eval_loss": 0.0054823532700538635,
+ "eval_runtime": 581.9182,
+ "eval_samples_per_second": 5.867,
+ "eval_selector_accuracy": 97.7804,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.184,
+ "step": 628
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 2.346351770520781e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/sft/checkpoint-628/training_args.bin b/qwen3/sft/checkpoint-628/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3903357a542bc3a9ccd098cf573248a02728c6e9
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:601ad253ebc37cfd18e26c1b1ece08fff37c9b901912b103fdc8fa017ec8f0e1
+size 7185
diff --git a/qwen3/sft/checkpoint-628/zero_to_fp32.py b/qwen3/sft/checkpoint-628/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/sft/checkpoint-628/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/sft/checkpoint-942/README.md b/qwen3/sft/checkpoint-942/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-942/adapter_config.json b/qwen3/sft/checkpoint-942/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9da5b578a766fe6816cb69301e1d55e3bea62cf
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "o_proj",
+ "k_proj",
+ "up_proj",
+ "gate_proj",
+ "q_proj",
+ "down_proj",
+ "v_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-942/adapter_model.safetensors b/qwen3/sft/checkpoint-942/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6c90a08448c13a8c3bbb51a52ae1f730762853f6
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76c15f1702cd3d1fe96b2c38fdf887aaf19616abe3e6a15c97481dff38b79ad5
+size 40422560
diff --git a/qwen3/sft/checkpoint-942/chat_template.jinja b/qwen3/sft/checkpoint-942/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/sft/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8dcd5c2b87b0c8c0d58d8ffebff99493b0d3a80a
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eec0d173947600e255a3e28323907320812a4f50dd1fbc196d310e92151cd38
+size 121135557
diff --git a/qwen3/sft/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/sft/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9817b12502e31e5a9c7272c77ff00f971e781abb
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dcc1b76f3cac95b4b468754e6c66e152962f136e331bbcefed1e6995abfcb59
+size 121135749
diff --git a/qwen3/sft/checkpoint-942/global_step942/mp_rank_00_model_states.pt b/qwen3/sft/checkpoint-942/global_step942/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0c66a52aefd3af99d9402bb5cd7a5b81a32d5c1
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/global_step942/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:118ffae9eb47f683212849d2666b9ce9d5c04a82000e23d99b7c15cc388f5ecc
+size 351841765
diff --git a/qwen3/sft/checkpoint-942/latest b/qwen3/sft/checkpoint-942/latest
new file mode 100644
index 0000000000000000000000000000000000000000..4e9a68e4460aca593c480611a486d2a4cd73ef42
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/latest
@@ -0,0 +1 @@
+global_step942
\ No newline at end of file
diff --git a/qwen3/sft/checkpoint-942/rng_state_0.pth b/qwen3/sft/checkpoint-942/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..a973c2f491a1b041b3414bf278809a4063f61436
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0eb4343151d5c8060d318f215330972d0ebf23bd556ea7262eb9282f29fcf94
+size 14917
diff --git a/qwen3/sft/checkpoint-942/rng_state_1.pth b/qwen3/sft/checkpoint-942/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..82a2a686c369aa4a9149a4cbb35ff9ab12f16bc6
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73f71f79086f07e6e2ecbc6f6c08ff4c01d59e6f847b58261e66f82084d3ad77
+size 14917
diff --git a/qwen3/sft/checkpoint-942/scheduler.pt b/qwen3/sft/checkpoint-942/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b1f8af0480bf83ab728abfeff11bc3288214349
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8076da6f897951c5bdf8649e4fba596e8c49819927746017442ffd85f40b18a3
+size 1465
diff --git a/qwen3/sft/checkpoint-942/tokenizer.json b/qwen3/sft/checkpoint-942/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/sft/checkpoint-942/tokenizer_config.json b/qwen3/sft/checkpoint-942/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/sft/checkpoint-942/trainer_state.json b/qwen3/sft/checkpoint-942/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..59618cc709b54783801dd5b83b78fd2f2117237a
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/trainer_state.json
@@ -0,0 +1,408 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 3.0,
+ "eval_steps": 500,
+ "global_step": 942,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.8955494165420532,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.567183542251587,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 1.35407292842865,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.12746150493621827,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 0.9626659154891968,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.08457614183425903,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.4221804440021515,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.060509908199310306,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.4434971511363983,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.04329058527946472,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.7847144603729248,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.0370758205652237,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.30355238914489746,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.04995985627174378,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.598273754119873,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.03158450722694397,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 1.9779328107833862,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.0206314280629158,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.25729191303253174,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.021701553463935853,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.21868865191936493,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.02294553518295288,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 1.2186007499694824,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.013198558986186982,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.31176164746284485,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.013272044062614442,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.15429984033107758,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.011466830968856812,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.38197460770606995,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.010920770466327667,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 88.3178,
+ "eval_generator_rouge1": 99.4774,
+ "eval_generator_rouge2": 99.1631,
+ "eval_generator_rougeL": 99.0528,
+ "eval_loss": 0.008339357562363148,
+ "eval_runtime": 584.2228,
+ "eval_samples_per_second": 5.844,
+ "eval_selector_accuracy": 96.7874,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 0.4359477758407593,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.00993276685476303,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.13242340087890625,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.005089616030454635,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.13412831723690033,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.005343190580606461,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.2253202497959137,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.0047687780112028125,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.27512186765670776,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.0066933780908584595,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.1620800793170929,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.01029568463563919,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.153878852725029,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.005285752564668655,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.9735704064369202,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.006016410514712333,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.05531628057360649,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.0033036738634109495,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.08567451685667038,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.004241976141929627,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.07484884560108185,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.010447616130113602,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.38432225584983826,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.012001012265682221,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.2757236957550049,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.004458250850439072,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 1.082757830619812,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.011233760416507721,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.8778036236763,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.006709276884794235,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.1680862009525299,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.004196321591734886,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.0491,
+ "eval_generator_rouge1": 99.6708,
+ "eval_generator_rouge2": 99.4934,
+ "eval_generator_rougeL": 99.3984,
+ "eval_loss": 0.0054823532700538635,
+ "eval_runtime": 581.9182,
+ "eval_samples_per_second": 5.867,
+ "eval_selector_accuracy": 97.7804,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.184,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.2708258032798767,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.003280312567949295,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.1907150149345398,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.00262494757771492,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.0759034976363182,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0016721963882446289,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.8344013690948486,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0019289907068014145,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.08497262001037598,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.0021992195397615433,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.11710834503173828,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.00548185333609581,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.02737243101000786,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0029599376022815703,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.031406909227371216,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0012881334871053696,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.1769132763147354,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.004978828132152557,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.5065577030181885,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.0028983473777770994,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.13216915726661682,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.002082060277462006,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.08058275282382965,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.0047489568591117855,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.2788284122943878,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0014096658676862716,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.6501541137695312,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0017810370773077011,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.007601048331707716,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.002541198208928108,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.12641772627830505,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0016433246433734893,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.1519,
+ "eval_generator_rouge1": 99.8526,
+ "eval_generator_rouge2": 99.7382,
+ "eval_generator_rougeL": 99.647,
+ "eval_loss": 0.0044273375533521175,
+ "eval_runtime": 583.9099,
+ "eval_samples_per_second": 5.847,
+ "eval_selector_accuracy": 96.6706,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 942
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 3.524157743353037e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/sft/checkpoint-942/training_args.bin b/qwen3/sft/checkpoint-942/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3903357a542bc3a9ccd098cf573248a02728c6e9
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:601ad253ebc37cfd18e26c1b1ece08fff37c9b901912b103fdc8fa017ec8f0e1
+size 7185
diff --git a/qwen3/sft/checkpoint-942/zero_to_fp32.py b/qwen3/sft/checkpoint-942/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/sft/checkpoint-942/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/sft/eval_results.json b/qwen3/sft/eval_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..956cf5e3eff6ece668719f14f5a70e4bca502cea
--- /dev/null
+++ b/qwen3/sft/eval_results.json
@@ -0,0 +1,14 @@
+{
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.9626,
+ "eval_generator_rouge1": 99.8875,
+ "eval_generator_rouge2": 99.8292,
+ "eval_generator_rougeL": 99.735,
+ "eval_loss": 0.003336928552016616,
+ "eval_runtime": 603.5681,
+ "eval_samples_per_second": 5.656,
+ "eval_selector_accuracy": 99.2407,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.177
+}
\ No newline at end of file
diff --git a/qwen3/sft/tokenizer.json b/qwen3/sft/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/sft/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/sft/tokenizer_config.json b/qwen3/sft/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/sft/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/sft/train_results.json b/qwen3/sft/train_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..c085351ae63895ac995a4a300ce3b97032d343e4
--- /dev/null
+++ b/qwen3/sft/train_results.json
@@ -0,0 +1,8 @@
+{
+ "epoch": 5.0,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.016630506924513705,
+ "train_runtime": 6427.6125,
+ "train_samples_per_second": 7.8,
+ "train_steps_per_second": 0.244
+}
\ No newline at end of file
diff --git a/qwen3/sft/trainer_state.json b/qwen3/sft/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..921169dd0cf9a496933a6efbb9e4fb48e7c3f506
--- /dev/null
+++ b/qwen3/sft/trainer_state.json
@@ -0,0 +1,664 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 5.0,
+ "eval_steps": 500,
+ "global_step": 1570,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.8955494165420532,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.567183542251587,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 1.35407292842865,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.12746150493621827,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 0.9626659154891968,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.08457614183425903,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.4221804440021515,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.060509908199310306,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.4434971511363983,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.04329058527946472,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.7847144603729248,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.0370758205652237,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.30355238914489746,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.04995985627174378,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.598273754119873,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.03158450722694397,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 1.9779328107833862,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.0206314280629158,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.25729191303253174,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.021701553463935853,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.21868865191936493,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.02294553518295288,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 1.2186007499694824,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.013198558986186982,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.31176164746284485,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.013272044062614442,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.15429984033107758,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.011466830968856812,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.38197460770606995,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.010920770466327667,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 88.3178,
+ "eval_generator_rouge1": 99.4774,
+ "eval_generator_rouge2": 99.1631,
+ "eval_generator_rougeL": 99.0528,
+ "eval_loss": 0.008339357562363148,
+ "eval_runtime": 584.2228,
+ "eval_samples_per_second": 5.844,
+ "eval_selector_accuracy": 96.7874,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 0.4359477758407593,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.00993276685476303,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.13242340087890625,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.005089616030454635,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.13412831723690033,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.005343190580606461,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.2253202497959137,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.0047687780112028125,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.27512186765670776,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.0066933780908584595,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.1620800793170929,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.01029568463563919,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.153878852725029,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.005285752564668655,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.9735704064369202,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.006016410514712333,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.05531628057360649,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.0033036738634109495,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.08567451685667038,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.004241976141929627,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.07484884560108185,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.010447616130113602,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.38432225584983826,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.012001012265682221,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.2757236957550049,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.004458250850439072,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 1.082757830619812,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.011233760416507721,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.8778036236763,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.006709276884794235,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.1680862009525299,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.004196321591734886,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 93.0491,
+ "eval_generator_rouge1": 99.6708,
+ "eval_generator_rouge2": 99.4934,
+ "eval_generator_rougeL": 99.3984,
+ "eval_loss": 0.0054823532700538635,
+ "eval_runtime": 581.9182,
+ "eval_samples_per_second": 5.867,
+ "eval_selector_accuracy": 97.7804,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.184,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.2708258032798767,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.003280312567949295,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.1907150149345398,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.00262494757771492,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.0759034976363182,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0016721963882446289,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.8344013690948486,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0019289907068014145,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.08497262001037598,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.0021992195397615433,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.11710834503173828,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.00548185333609581,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.02737243101000786,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0029599376022815703,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.031406909227371216,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0012881334871053696,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.1769132763147354,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.004978828132152557,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.5065577030181885,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.0028983473777770994,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.13216915726661682,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.002082060277462006,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.08058275282382965,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.0047489568591117855,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.2788284122943878,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0014096658676862716,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.6501541137695312,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0017810370773077011,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.007601048331707716,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.002541198208928108,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.12641772627830505,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0016433246433734893,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.1519,
+ "eval_generator_rouge1": 99.8526,
+ "eval_generator_rouge2": 99.7382,
+ "eval_generator_rougeL": 99.647,
+ "eval_loss": 0.0044273375533521175,
+ "eval_runtime": 583.9099,
+ "eval_samples_per_second": 5.847,
+ "eval_selector_accuracy": 96.6706,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 942
+ },
+ {
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.06881710141897202,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.001966923289000988,
+ "step": 960
+ },
+ {
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.004793599247932434,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.001482640765607357,
+ "step": 980
+ },
+ {
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.005230629350990057,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.0005885613150894642,
+ "step": 1000
+ },
+ {
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.0802706778049469,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.0016000797972083093,
+ "step": 1020
+ },
+ {
+ "epoch": 3.3127243717590744,
+ "grad_norm": 0.04865531995892525,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.0007603847421705723,
+ "step": 1040
+ },
+ {
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.002228009281679988,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.0016863197088241577,
+ "step": 1060
+ },
+ {
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.12053167074918747,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.0008015411905944348,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.004856751766055822,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0011467586271464824,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.03997187688946724,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.001309672836214304,
+ "step": 1120
+ },
+ {
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.04420069605112076,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0008654085919260979,
+ "step": 1140
+ },
+ {
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.011205613613128662,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.004080542176961899,
+ "step": 1160
+ },
+ {
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.015621992759406567,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0017238393425941466,
+ "step": 1180
+ },
+ {
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.028632311150431633,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.0011840559542179107,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8871160749900278,
+ "grad_norm": 0.19821636378765106,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.001461276412010193,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.026029646396636963,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0006165837403386831,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.7944,
+ "eval_generator_rouge1": 99.8712,
+ "eval_generator_rouge2": 99.7877,
+ "eval_generator_rougeL": 99.6495,
+ "eval_loss": 0.003426080336794257,
+ "eval_runtime": 585.1791,
+ "eval_samples_per_second": 5.834,
+ "eval_selector_accuracy": 98.7734,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.183,
+ "step": 1256
+ },
+ {
+ "epoch": 4.0127642600717985,
+ "grad_norm": 0.06868977099657059,
+ "learning_rate": 9.464081721960592e-06,
+ "loss": 0.001044323481619358,
+ "step": 1260
+ },
+ {
+ "epoch": 4.076585560430794,
+ "grad_norm": 0.1312812864780426,
+ "learning_rate": 8.331628936475818e-06,
+ "loss": 0.00047424803487956525,
+ "step": 1280
+ },
+ {
+ "epoch": 4.140406860789788,
+ "grad_norm": 0.0014219481963664293,
+ "learning_rate": 7.2659843568748385e-06,
+ "loss": 0.0009649133309721947,
+ "step": 1300
+ },
+ {
+ "epoch": 4.204228161148784,
+ "grad_norm": 0.016619998961687088,
+ "learning_rate": 6.268854516770008e-06,
+ "loss": 0.0006559681612998247,
+ "step": 1320
+ },
+ {
+ "epoch": 4.268049461507778,
+ "grad_norm": 0.057871364057064056,
+ "learning_rate": 5.341836229602331e-06,
+ "loss": 0.0012834925204515458,
+ "step": 1340
+ },
+ {
+ "epoch": 4.331870761866773,
+ "grad_norm": 0.005151741206645966,
+ "learning_rate": 4.4864140314888216e-06,
+ "loss": 0.0005498498678207398,
+ "step": 1360
+ },
+ {
+ "epoch": 4.395692062225768,
+ "grad_norm": 0.015129027888178825,
+ "learning_rate": 3.703957803871932e-06,
+ "loss": 0.000982836540788412,
+ "step": 1380
+ },
+ {
+ "epoch": 4.4595133625847625,
+ "grad_norm": 0.0474361851811409,
+ "learning_rate": 2.9957205797780495e-06,
+ "loss": 0.0009566731750965118,
+ "step": 1400
+ },
+ {
+ "epoch": 4.523334662943758,
+ "grad_norm": 0.079148069024086,
+ "learning_rate": 2.362836537198199e-06,
+ "loss": 0.0007865218445658684,
+ "step": 1420
+ },
+ {
+ "epoch": 4.587155963302752,
+ "grad_norm": 0.02590208128094673,
+ "learning_rate": 1.8063191828043703e-06,
+ "loss": 0.00046601640060544013,
+ "step": 1440
+ },
+ {
+ "epoch": 4.650977263661747,
+ "grad_norm": 0.028924427926540375,
+ "learning_rate": 1.3270597289100437e-06,
+ "loss": 0.0005176489241421223,
+ "step": 1460
+ },
+ {
+ "epoch": 4.714798564020742,
+ "grad_norm": 0.012378301471471786,
+ "learning_rate": 9.258256662741205e-07,
+ "loss": 0.0003302923170849681,
+ "step": 1480
+ },
+ {
+ "epoch": 4.778619864379737,
+ "grad_norm": 0.04462262615561485,
+ "learning_rate": 6.032595350337341e-07,
+ "loss": 0.0010699938982725143,
+ "step": 1500
+ },
+ {
+ "epoch": 4.842441164738732,
+ "grad_norm": 0.00910312868654728,
+ "learning_rate": 3.5987789573421356e-07,
+ "loss": 0.003197905048727989,
+ "step": 1520
+ },
+ {
+ "epoch": 4.9062624650977265,
+ "grad_norm": 0.13681654632091522,
+ "learning_rate": 1.9607050210401192e-07,
+ "loss": 0.0005365588702261447,
+ "step": 1540
+ },
+ {
+ "epoch": 4.970083765456721,
+ "grad_norm": 0.05481738969683647,
+ "learning_rate": 1.1209967689925832e-07,
+ "loss": 0.000743288267403841,
+ "step": 1560
+ },
+ {
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.0864,
+ "eval_generator_rouge1": 99.8772,
+ "eval_generator_rouge2": 99.805,
+ "eval_generator_rougeL": 99.679,
+ "eval_loss": 0.003336928552016616,
+ "eval_runtime": 638.4465,
+ "eval_samples_per_second": 5.347,
+ "eval_selector_accuracy": 99.1238,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.168,
+ "step": 1570
+ },
+ {
+ "epoch": 5.0,
+ "step": 1570,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.016630506924513705,
+ "train_runtime": 6427.6125,
+ "train_samples_per_second": 7.8,
+ "train_steps_per_second": 0.244
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 5.871444840310374e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/sft/training_args.bin b/qwen3/sft/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3903357a542bc3a9ccd098cf573248a02728c6e9
--- /dev/null
+++ b/qwen3/sft/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:601ad253ebc37cfd18e26c1b1ece08fff37c9b901912b103fdc8fa017ec8f0e1
+size 7185
diff --git a/qwen3/srkl/README.md b/qwen3/srkl/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..2d5c93a6f79979b359a2efd6cd2c1ac7ea3b564c
--- /dev/null
+++ b/qwen3/srkl/README.md
@@ -0,0 +1,79 @@
+---
+library_name: peft
+license: other
+base_model: Qwen/Qwen3-0.6B
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+pipeline_tag: text-generation
+model-index:
+- name: srkl
+ results: []
+---
+
+
+
+# srkl
+
+This model is a fine-tuned version of [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) on the cypher_prepared_train dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0037
+- Selector Count: 1712.0
+- Generator Count: 1712.0
+- Selector Accuracy: 98.715
+- Generator Exact Match: 96.729
+- Generator Rouge1: 99.894
+- Generator Rouge2: 99.8306
+- Generator Rougel: 99.7434
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 2
+- eval_batch_size: 16
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine_with_min_lr
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Selector Count | Generator Count | Selector Accuracy | Generator Exact Match | Generator Rouge1 | Generator Rouge2 | Generator Rougel |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:---------------:|:-----------------:|:---------------------:|:----------------:|:----------------:|:----------------:|
+| 0.0112 | 1.0 | 314 | 0.0110 | 1712.0 | 1712.0 | 94.3925 | 88.6682 | 99.5496 | 99.2687 | 99.1723 |
+| 0.0034 | 2.0 | 628 | 0.0049 | 1712.0 | 1712.0 | 97.6636 | 94.4509 | 99.8443 | 99.7143 | 99.636 |
+| 0.0012 | 3.0 | 942 | 0.0045 | 1712.0 | 1712.0 | 98.2477 | 95.7944 | 99.8488 | 99.7613 | 99.6161 |
+| 0.0005 | 4.0 | 1256 | 0.0037 | 1712.0 | 1712.0 | 98.6565 | 96.4953 | 99.9005 | 99.8353 | 99.7303 |
+| 0.0004 | 5.0 | 1570 | 0.0037 | 1712.0 | 1712.0 | 98.6565 | 96.9042 | 99.8996 | 99.839 | 99.7647 |
+
+
+### Framework versions
+
+- PEFT 0.18.1
+- Transformers 5.8.0
+- Pytorch 2.9.1+cu128
+- Datasets 4.0.0
+- Tokenizers 0.22.2
\ No newline at end of file
diff --git a/qwen3/srkl/adapter_config.json b/qwen3/srkl/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..56658aea33d649140802c320668b4fe2086ad8d0
--- /dev/null
+++ b/qwen3/srkl/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "o_proj",
+ "down_proj",
+ "k_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/srkl/adapter_model.safetensors b/qwen3/srkl/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c09d60342eb952f5ba1be632cb47e097b5db255e
--- /dev/null
+++ b/qwen3/srkl/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbf45450db8e7afe953cde7784cc7a344cd803fe2195e75c7db886be095a1780
+size 40422560
diff --git a/qwen3/srkl/all_results.json b/qwen3/srkl/all_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..562d012dc52879a2ea6749aa9cacdce5cc08fce0
--- /dev/null
+++ b/qwen3/srkl/all_results.json
@@ -0,0 +1,19 @@
+{
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.729,
+ "eval_generator_rouge1": 99.894,
+ "eval_generator_rouge2": 99.8306,
+ "eval_generator_rougeL": 99.7434,
+ "eval_loss": 0.003705898765474558,
+ "eval_runtime": 588.9515,
+ "eval_samples_per_second": 5.797,
+ "eval_selector_accuracy": 98.715,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.182,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.011693778396495113,
+ "train_runtime": 7720.2334,
+ "train_samples_per_second": 6.494,
+ "train_steps_per_second": 0.203
+}
\ No newline at end of file
diff --git a/qwen3/srkl/chat_template.jinja b/qwen3/srkl/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/srkl/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-1256/README.md b/qwen3/srkl/checkpoint-1256/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-1256/adapter_config.json b/qwen3/srkl/checkpoint-1256/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..56658aea33d649140802c320668b4fe2086ad8d0
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "o_proj",
+ "down_proj",
+ "k_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-1256/adapter_model.safetensors b/qwen3/srkl/checkpoint-1256/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1731d108b8595334f382808d4d24e6924ec82d88
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:407f43fc66e295fd971159f4b93abd6b2091b70d55be8cde6305919a09dfee34
+size 40422560
diff --git a/qwen3/srkl/checkpoint-1256/chat_template.jinja b/qwen3/srkl/checkpoint-1256/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9fd7c0561d91701930e7125ed1a65235b70c45b9
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f222560c41a05b212de073ba38f523a73e829dab340a42e5a39d67f2725990bf
+size 121135557
diff --git a/qwen3/srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7a6177db508a2f2a317f80b54bea1adf548eb38c
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0874afbd7f408910122fe47bbc092bb761a154c729589d79127c414a5e37d9dd
+size 121135749
diff --git a/qwen3/srkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt b/qwen3/srkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..383b417627db7bd2a1d68458b4186278f17229d6
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7077b183238a7317388239055b1c224744afb0cf4cb22b4b9241eed8783a8783
+size 351841765
diff --git a/qwen3/srkl/checkpoint-1256/latest b/qwen3/srkl/checkpoint-1256/latest
new file mode 100644
index 0000000000000000000000000000000000000000..74abb3d23ea83ebdac59918752db5d1782834ca7
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/latest
@@ -0,0 +1 @@
+global_step1256
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-1256/rng_state_0.pth b/qwen3/srkl/checkpoint-1256/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..a906c3635956fb87d00e0cba305f62fffa7f8142
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e392cbdfab33b1e961da8b1f9e1ee1c568a7b71ff07611041432586bfeac125
+size 14917
diff --git a/qwen3/srkl/checkpoint-1256/rng_state_1.pth b/qwen3/srkl/checkpoint-1256/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..861a906c16372aba7394b00cc67ca5d667a3a774
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ebcd726ea1c96ac657a47d408939b7ae39522429f9dea5303901d0a6d6699f5
+size 14917
diff --git a/qwen3/srkl/checkpoint-1256/scheduler.pt b/qwen3/srkl/checkpoint-1256/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40e863d07cfed22cd912014eb3ba3fd2b16b173f
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1775297f14e75f7463ae51983a66296f81f4b2e2e5d7234f1a6d295abfa4cf23
+size 1465
diff --git a/qwen3/srkl/checkpoint-1256/tokenizer.json b/qwen3/srkl/checkpoint-1256/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/srkl/checkpoint-1256/tokenizer_config.json b/qwen3/srkl/checkpoint-1256/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/srkl/checkpoint-1256/trainer_state.json b/qwen3/srkl/checkpoint-1256/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..97ec79619d170a4ad5f47ba799ec6b25f57edade
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/trainer_state.json
@@ -0,0 +1,528 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 4.0,
+ "eval_steps": 500,
+ "global_step": 1256,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.5670945644378662,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.37616620063781736,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 0.8954827785491943,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.0971335530281067,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 0.9263805747032166,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.06594716906547546,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.31774941086769104,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.04192234873771668,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.44272375106811523,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.030837148427963257,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.5705686807632446,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.017216089367866515,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.3161797821521759,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.025291597843170165,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.4870106875896454,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.022351644933223724,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 0.2289164662361145,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.01218288466334343,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.24952471256256104,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.019831493496894836,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.41395241022109985,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.012001317739486695,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.19487057626247406,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.011289550364017487,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.31937241554260254,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.008779586851596832,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.12296169996261597,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.011695237457752227,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.3308272659778595,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.011218554526567458,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 88.6682,
+ "eval_generator_rouge1": 99.5496,
+ "eval_generator_rouge2": 99.2687,
+ "eval_generator_rougeL": 99.1723,
+ "eval_loss": 0.011020934209227562,
+ "eval_runtime": 614.0637,
+ "eval_samples_per_second": 5.56,
+ "eval_selector_accuracy": 94.3925,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.174,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 0.6877385973930359,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.010994066298007966,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.1419052630662918,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.005199763923883438,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.10923358798027039,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.004498997330665588,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.2007422000169754,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.005002734437584877,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.08985759317874908,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.006419034302234649,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.19415093958377838,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.00729372426867485,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.06894819438457489,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.0035026255995035172,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.6073423624038696,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.005546892434358597,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.02788952738046646,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.00369630828499794,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.09827981144189835,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.006270292401313782,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.09948836266994476,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.005865002796053886,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.3055334985256195,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.007864721119403839,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.3340248167514801,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.0026097118854522703,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.3238213360309601,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.005317946150898934,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.06955885142087936,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.00310036838054657,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.029902024194598198,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.0033790454268455505,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 94.4509,
+ "eval_generator_rouge1": 99.8443,
+ "eval_generator_rouge2": 99.7143,
+ "eval_generator_rougeL": 99.636,
+ "eval_loss": 0.004883269313722849,
+ "eval_runtime": 610.2471,
+ "eval_samples_per_second": 5.594,
+ "eval_selector_accuracy": 97.6636,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.08393510431051254,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.002889341861009598,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.317868173122406,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.002996666356921196,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.03327847272157669,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0018069516867399217,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.06478872895240784,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0016909070312976838,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.19922314584255219,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.0016838768497109413,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.048591550439596176,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.0019782688468694687,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.0104411281645298,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0010088316164910793,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.04026724770665169,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0007452124729752541,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.07828482985496521,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.003089136257767677,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.206737220287323,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.0016609614714980126,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.0661257803440094,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.0017474986612796783,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.14079469442367554,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.003954095393419265,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.056867823004722595,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0016886677592992782,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.10408495366573334,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0013717899098992347,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.003307907609269023,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.001446123607456684,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.07255368679761887,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0011631335131824017,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.7944,
+ "eval_generator_rouge1": 99.8488,
+ "eval_generator_rouge2": 99.7613,
+ "eval_generator_rougeL": 99.6161,
+ "eval_loss": 0.004494966473430395,
+ "eval_runtime": 600.8182,
+ "eval_samples_per_second": 5.682,
+ "eval_selector_accuracy": 98.2477,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.178,
+ "step": 942
+ },
+ {
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.07352045923471451,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.002453933656215668,
+ "step": 960
+ },
+ {
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.0014208417851477861,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.0009017683565616608,
+ "step": 980
+ },
+ {
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.12319519370794296,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.0010159458965063096,
+ "step": 1000
+ },
+ {
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.016385605558753014,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.002848365902900696,
+ "step": 1020
+ },
+ {
+ "epoch": 3.3127243717590744,
+ "grad_norm": 0.14579907059669495,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.0006675420328974724,
+ "step": 1040
+ },
+ {
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.0019410790409892797,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.0011831691488623619,
+ "step": 1060
+ },
+ {
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.07082991302013397,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.0023162638768553733,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.01225790660828352,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0022976718842983245,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.012785660102963448,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.0013264333829283714,
+ "step": 1120
+ },
+ {
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.0714384987950325,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0007606681436300277,
+ "step": 1140
+ },
+ {
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.5528075695037842,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.002803182043135166,
+ "step": 1160
+ },
+ {
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.01241183653473854,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0012427497655153274,
+ "step": 1180
+ },
+ {
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.0505630299448967,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.001273194793611765,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8871160749900278,
+ "grad_norm": 0.03439785912632942,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.0007076818961650133,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.016928531229496002,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0005449302028864622,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.4953,
+ "eval_generator_rouge1": 99.9005,
+ "eval_generator_rouge2": 99.8353,
+ "eval_generator_rougeL": 99.7303,
+ "eval_loss": 0.0036805884446948767,
+ "eval_runtime": 601.6399,
+ "eval_samples_per_second": 5.674,
+ "eval_selector_accuracy": 98.6565,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.178,
+ "step": 1256
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 4.697643190059008e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/srkl/checkpoint-1256/training_args.bin b/qwen3/srkl/checkpoint-1256/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..31afaafa55cb5f0350c3985e07d863ac893286e4
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:109808d64c5d0b463c4a77490625c491baa90453dc3e9d7c8d7fe0958476b313
+size 7185
diff --git a/qwen3/srkl/checkpoint-1256/zero_to_fp32.py b/qwen3/srkl/checkpoint-1256/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1256/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/srkl/checkpoint-1570/README.md b/qwen3/srkl/checkpoint-1570/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-1570/adapter_config.json b/qwen3/srkl/checkpoint-1570/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..56658aea33d649140802c320668b4fe2086ad8d0
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "o_proj",
+ "down_proj",
+ "k_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-1570/adapter_model.safetensors b/qwen3/srkl/checkpoint-1570/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c09d60342eb952f5ba1be632cb47e097b5db255e
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbf45450db8e7afe953cde7784cc7a344cd803fe2195e75c7db886be095a1780
+size 40422560
diff --git a/qwen3/srkl/checkpoint-1570/chat_template.jinja b/qwen3/srkl/checkpoint-1570/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ae149a2d7c39812e353166426bf54c59e9156e9c
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f6ca2e64c259a209a31f3978bff6c69621e88a10d0b4f9fe46deb409f3b0687
+size 121135557
diff --git a/qwen3/srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4e678b27a5809f00d2c6e0a8363a55d627092faa
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ac5c3b6724f0e231d6f595af3ebb936ad9898ab5a186d67caf477030a8241b0
+size 121135749
diff --git a/qwen3/srkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt b/qwen3/srkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..79b38cb188e81781cf734313b607b5f90f553f79
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d316e9a8fa861f5fc4d3a0003fb70fb7ac8a6b1d0987383a9c5b709a091f9ba2
+size 351841765
diff --git a/qwen3/srkl/checkpoint-1570/latest b/qwen3/srkl/checkpoint-1570/latest
new file mode 100644
index 0000000000000000000000000000000000000000..18c2fcff58fa6daa0a2e7c3a015552a69085778b
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/latest
@@ -0,0 +1 @@
+global_step1570
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-1570/rng_state_0.pth b/qwen3/srkl/checkpoint-1570/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..0bd3e20da056a1b90868007e5156cc80d069c1e0
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1de90529f335524c587120c3350502042acdc8c79489c4c5e13cd6ef21df5a0e
+size 14917
diff --git a/qwen3/srkl/checkpoint-1570/rng_state_1.pth b/qwen3/srkl/checkpoint-1570/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..afae8b72b469c151700a9f79d581614c9c39d801
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10e71e8462f9912287f96747faba05fcc6e8b74a8e8dd01afbb3901b1cdda55b
+size 14917
diff --git a/qwen3/srkl/checkpoint-1570/scheduler.pt b/qwen3/srkl/checkpoint-1570/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e22954463492d8a070ae427c95418df61893c54c
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6332fac17ae99a07597512c705658da2fd8429d1c39bb776b30e107c2cb7aae
+size 1465
diff --git a/qwen3/srkl/checkpoint-1570/tokenizer.json b/qwen3/srkl/checkpoint-1570/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/srkl/checkpoint-1570/tokenizer_config.json b/qwen3/srkl/checkpoint-1570/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/srkl/checkpoint-1570/trainer_state.json b/qwen3/srkl/checkpoint-1570/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..9b9fc9f397be698e0e755aa48dfb9e8492b0b24b
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/trainer_state.json
@@ -0,0 +1,655 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 5.0,
+ "eval_steps": 500,
+ "global_step": 1570,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.5670945644378662,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.37616620063781736,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 0.8954827785491943,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.0971335530281067,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 0.9263805747032166,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.06594716906547546,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.31774941086769104,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.04192234873771668,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.44272375106811523,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.030837148427963257,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.5705686807632446,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.017216089367866515,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.3161797821521759,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.025291597843170165,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.4870106875896454,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.022351644933223724,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 0.2289164662361145,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.01218288466334343,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.24952471256256104,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.019831493496894836,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.41395241022109985,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.012001317739486695,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.19487057626247406,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.011289550364017487,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.31937241554260254,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.008779586851596832,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.12296169996261597,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.011695237457752227,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.3308272659778595,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.011218554526567458,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 88.6682,
+ "eval_generator_rouge1": 99.5496,
+ "eval_generator_rouge2": 99.2687,
+ "eval_generator_rougeL": 99.1723,
+ "eval_loss": 0.011020934209227562,
+ "eval_runtime": 614.0637,
+ "eval_samples_per_second": 5.56,
+ "eval_selector_accuracy": 94.3925,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.174,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 0.6877385973930359,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.010994066298007966,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.1419052630662918,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.005199763923883438,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.10923358798027039,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.004498997330665588,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.2007422000169754,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.005002734437584877,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.08985759317874908,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.006419034302234649,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.19415093958377838,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.00729372426867485,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.06894819438457489,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.0035026255995035172,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.6073423624038696,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.005546892434358597,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.02788952738046646,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.00369630828499794,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.09827981144189835,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.006270292401313782,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.09948836266994476,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.005865002796053886,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.3055334985256195,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.007864721119403839,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.3340248167514801,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.0026097118854522703,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.3238213360309601,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.005317946150898934,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.06955885142087936,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.00310036838054657,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.029902024194598198,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.0033790454268455505,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 94.4509,
+ "eval_generator_rouge1": 99.8443,
+ "eval_generator_rouge2": 99.7143,
+ "eval_generator_rougeL": 99.636,
+ "eval_loss": 0.004883269313722849,
+ "eval_runtime": 610.2471,
+ "eval_samples_per_second": 5.594,
+ "eval_selector_accuracy": 97.6636,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.08393510431051254,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.002889341861009598,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.317868173122406,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.002996666356921196,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.03327847272157669,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0018069516867399217,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.06478872895240784,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0016909070312976838,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.19922314584255219,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.0016838768497109413,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.048591550439596176,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.0019782688468694687,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.0104411281645298,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0010088316164910793,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.04026724770665169,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0007452124729752541,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.07828482985496521,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.003089136257767677,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.206737220287323,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.0016609614714980126,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.0661257803440094,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.0017474986612796783,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.14079469442367554,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.003954095393419265,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.056867823004722595,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0016886677592992782,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.10408495366573334,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0013717899098992347,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.003307907609269023,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.001446123607456684,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.07255368679761887,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0011631335131824017,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.7944,
+ "eval_generator_rouge1": 99.8488,
+ "eval_generator_rouge2": 99.7613,
+ "eval_generator_rougeL": 99.6161,
+ "eval_loss": 0.004494966473430395,
+ "eval_runtime": 600.8182,
+ "eval_samples_per_second": 5.682,
+ "eval_selector_accuracy": 98.2477,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.178,
+ "step": 942
+ },
+ {
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.07352045923471451,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.002453933656215668,
+ "step": 960
+ },
+ {
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.0014208417851477861,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.0009017683565616608,
+ "step": 980
+ },
+ {
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.12319519370794296,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.0010159458965063096,
+ "step": 1000
+ },
+ {
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.016385605558753014,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.002848365902900696,
+ "step": 1020
+ },
+ {
+ "epoch": 3.3127243717590744,
+ "grad_norm": 0.14579907059669495,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.0006675420328974724,
+ "step": 1040
+ },
+ {
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.0019410790409892797,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.0011831691488623619,
+ "step": 1060
+ },
+ {
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.07082991302013397,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.0023162638768553733,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.01225790660828352,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0022976718842983245,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.012785660102963448,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.0013264333829283714,
+ "step": 1120
+ },
+ {
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.0714384987950325,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0007606681436300277,
+ "step": 1140
+ },
+ {
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.5528075695037842,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.002803182043135166,
+ "step": 1160
+ },
+ {
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.01241183653473854,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0012427497655153274,
+ "step": 1180
+ },
+ {
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.0505630299448967,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.001273194793611765,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8871160749900278,
+ "grad_norm": 0.03439785912632942,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.0007076818961650133,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.016928531229496002,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0005449302028864622,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.4953,
+ "eval_generator_rouge1": 99.9005,
+ "eval_generator_rouge2": 99.8353,
+ "eval_generator_rougeL": 99.7303,
+ "eval_loss": 0.0036805884446948767,
+ "eval_runtime": 601.6399,
+ "eval_samples_per_second": 5.674,
+ "eval_selector_accuracy": 98.6565,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.178,
+ "step": 1256
+ },
+ {
+ "epoch": 4.0127642600717985,
+ "grad_norm": 0.026434868574142456,
+ "learning_rate": 9.464081721960592e-06,
+ "loss": 0.0009123041294515133,
+ "step": 1260
+ },
+ {
+ "epoch": 4.076585560430794,
+ "grad_norm": 0.054495662450790405,
+ "learning_rate": 8.331628936475818e-06,
+ "loss": 0.0002502598334103823,
+ "step": 1280
+ },
+ {
+ "epoch": 4.140406860789788,
+ "grad_norm": 0.004517420195043087,
+ "learning_rate": 7.2659843568748385e-06,
+ "loss": 0.0005662859417498112,
+ "step": 1300
+ },
+ {
+ "epoch": 4.204228161148784,
+ "grad_norm": 0.01496294979006052,
+ "learning_rate": 6.268854516770008e-06,
+ "loss": 0.0004728314466774464,
+ "step": 1320
+ },
+ {
+ "epoch": 4.268049461507778,
+ "grad_norm": 0.03497197851538658,
+ "learning_rate": 5.341836229602331e-06,
+ "loss": 0.0007120450958609581,
+ "step": 1340
+ },
+ {
+ "epoch": 4.331870761866773,
+ "grad_norm": 0.006169375032186508,
+ "learning_rate": 4.4864140314888216e-06,
+ "loss": 0.0014027132652699946,
+ "step": 1360
+ },
+ {
+ "epoch": 4.395692062225768,
+ "grad_norm": 0.02854115329682827,
+ "learning_rate": 3.703957803871932e-06,
+ "loss": 0.0008245049975812435,
+ "step": 1380
+ },
+ {
+ "epoch": 4.4595133625847625,
+ "grad_norm": 0.03609360754489899,
+ "learning_rate": 2.9957205797780495e-06,
+ "loss": 0.0009156657382845878,
+ "step": 1400
+ },
+ {
+ "epoch": 4.523334662943758,
+ "grad_norm": 0.04012208431959152,
+ "learning_rate": 2.362836537198199e-06,
+ "loss": 0.0006599740125238895,
+ "step": 1420
+ },
+ {
+ "epoch": 4.587155963302752,
+ "grad_norm": 0.018918916583061218,
+ "learning_rate": 1.8063191828043703e-06,
+ "loss": 0.0006631083786487579,
+ "step": 1440
+ },
+ {
+ "epoch": 4.650977263661747,
+ "grad_norm": 0.021513519808650017,
+ "learning_rate": 1.3270597289100437e-06,
+ "loss": 0.0003777497448027134,
+ "step": 1460
+ },
+ {
+ "epoch": 4.714798564020742,
+ "grad_norm": 0.008279121480882168,
+ "learning_rate": 9.258256662741205e-07,
+ "loss": 0.0005044300109148025,
+ "step": 1480
+ },
+ {
+ "epoch": 4.778619864379737,
+ "grad_norm": 0.012552818283438683,
+ "learning_rate": 6.032595350337341e-07,
+ "loss": 0.0006919664330780507,
+ "step": 1500
+ },
+ {
+ "epoch": 4.842441164738732,
+ "grad_norm": 0.0035533979535102844,
+ "learning_rate": 3.5987789573421356e-07,
+ "loss": 0.004239285737276078,
+ "step": 1520
+ },
+ {
+ "epoch": 4.9062624650977265,
+ "grad_norm": 0.0733393207192421,
+ "learning_rate": 1.9607050210401192e-07,
+ "loss": 0.0004377847071737051,
+ "step": 1540
+ },
+ {
+ "epoch": 4.970083765456721,
+ "grad_norm": 0.02573123201727867,
+ "learning_rate": 1.1209967689925832e-07,
+ "loss": 0.0004438157659024,
+ "step": 1560
+ },
+ {
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.9042,
+ "eval_generator_rouge1": 99.8996,
+ "eval_generator_rouge2": 99.839,
+ "eval_generator_rougeL": 99.7647,
+ "eval_loss": 0.003705898765474558,
+ "eval_runtime": 602.6617,
+ "eval_samples_per_second": 5.665,
+ "eval_selector_accuracy": 98.6565,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.178,
+ "step": 1570
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 5.871444840310374e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/srkl/checkpoint-1570/training_args.bin b/qwen3/srkl/checkpoint-1570/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..31afaafa55cb5f0350c3985e07d863ac893286e4
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:109808d64c5d0b463c4a77490625c491baa90453dc3e9d7c8d7fe0958476b313
+size 7185
diff --git a/qwen3/srkl/checkpoint-1570/zero_to_fp32.py b/qwen3/srkl/checkpoint-1570/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/srkl/checkpoint-1570/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/srkl/checkpoint-314/README.md b/qwen3/srkl/checkpoint-314/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-314/adapter_config.json b/qwen3/srkl/checkpoint-314/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..56658aea33d649140802c320668b4fe2086ad8d0
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "o_proj",
+ "down_proj",
+ "k_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-314/adapter_model.safetensors b/qwen3/srkl/checkpoint-314/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e909bc703217cbe17f0959f6ade01e60239e2081
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc8bfd92f32bc2b4955b041b9462b07c6a75381e3b28f4c3f914187e8686bd3c
+size 40422560
diff --git a/qwen3/srkl/checkpoint-314/chat_template.jinja b/qwen3/srkl/checkpoint-314/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d2d6eebbcd22093ccf47099dd238446fe145a45d
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a0a8110cef66891bfd2ae7ff2ec816c6a0aff736dd7a9be0fd363017a28a024
+size 121135557
diff --git a/qwen3/srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..00dfb11599d5a2bec056e5ccd1de3c0c953fafc9
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1eaebabb3f8c55fe1fd6c4ab8d68d3ce1e75ecf226f25fb012d919a9f1920750
+size 121135749
diff --git a/qwen3/srkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt b/qwen3/srkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eea248a4f38a8b95251796f280330b1d42500da6
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7c30b5d5ed79e69d8811f4a672c0e62ce6932d2151a4748a7993cacbfc0a18c
+size 351841765
diff --git a/qwen3/srkl/checkpoint-314/latest b/qwen3/srkl/checkpoint-314/latest
new file mode 100644
index 0000000000000000000000000000000000000000..a9786513b1a514db2c3a9da7408844635663d55d
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/latest
@@ -0,0 +1 @@
+global_step314
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-314/rng_state_0.pth b/qwen3/srkl/checkpoint-314/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..29a75cc288732f75f1665125d4dd7bd2d36d956e
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d64facf9e86d76db5cf733d98c22e2cab4aab87f225391fc7213739eed6445c9
+size 14917
diff --git a/qwen3/srkl/checkpoint-314/rng_state_1.pth b/qwen3/srkl/checkpoint-314/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..631a005741dbe3e162cbc2bee65b8935a5328b2d
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac8e39b16d0f85015ff43dc24e2febbedc1e19041a990ff04b06f8c153b34328
+size 14917
diff --git a/qwen3/srkl/checkpoint-314/scheduler.pt b/qwen3/srkl/checkpoint-314/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3cb460802cda03bf0833a98a6f1428a3fd4f585
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17e254b5642bb75983fd5105bec79b35376cfbc99d29702287668b8767d46149
+size 1465
diff --git a/qwen3/srkl/checkpoint-314/tokenizer.json b/qwen3/srkl/checkpoint-314/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/srkl/checkpoint-314/tokenizer_config.json b/qwen3/srkl/checkpoint-314/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/srkl/checkpoint-314/trainer_state.json b/qwen3/srkl/checkpoint-314/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..9512088d367044a8cc0d9d37a723fbb9f7d3af91
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/trainer_state.json
@@ -0,0 +1,154 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 1.0,
+ "eval_steps": 500,
+ "global_step": 314,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.5670945644378662,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.37616620063781736,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 0.8954827785491943,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.0971335530281067,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 0.9263805747032166,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.06594716906547546,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.31774941086769104,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.04192234873771668,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.44272375106811523,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.030837148427963257,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.5705686807632446,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.017216089367866515,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.3161797821521759,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.025291597843170165,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.4870106875896454,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.022351644933223724,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 0.2289164662361145,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.01218288466334343,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.24952471256256104,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.019831493496894836,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.41395241022109985,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.012001317739486695,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.19487057626247406,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.011289550364017487,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.31937241554260254,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.008779586851596832,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.12296169996261597,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.011695237457752227,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.3308272659778595,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.011218554526567458,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 88.6682,
+ "eval_generator_rouge1": 99.5496,
+ "eval_generator_rouge2": 99.2687,
+ "eval_generator_rougeL": 99.1723,
+ "eval_loss": 0.011020934209227562,
+ "eval_runtime": 614.0637,
+ "eval_samples_per_second": 5.56,
+ "eval_selector_accuracy": 94.3925,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.174,
+ "step": 314
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 1.172842003562496e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/srkl/checkpoint-314/training_args.bin b/qwen3/srkl/checkpoint-314/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..31afaafa55cb5f0350c3985e07d863ac893286e4
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:109808d64c5d0b463c4a77490625c491baa90453dc3e9d7c8d7fe0958476b313
+size 7185
diff --git a/qwen3/srkl/checkpoint-314/zero_to_fp32.py b/qwen3/srkl/checkpoint-314/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/srkl/checkpoint-314/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/srkl/checkpoint-628/README.md b/qwen3/srkl/checkpoint-628/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-628/adapter_config.json b/qwen3/srkl/checkpoint-628/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..56658aea33d649140802c320668b4fe2086ad8d0
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "o_proj",
+ "down_proj",
+ "k_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-628/adapter_model.safetensors b/qwen3/srkl/checkpoint-628/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..107d1ab5a3e025b0803f083c216b53cf5ce5d34d
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:323815f26fd0536931d3f95b1ef9e03f7cacbbb634bf685487c51488e8eb199b
+size 40422560
diff --git a/qwen3/srkl/checkpoint-628/chat_template.jinja b/qwen3/srkl/checkpoint-628/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2cef732f53efc024b3e322216c47c595f9bd67a7
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ca7d21da50c5be936959741ff7fdc1fa4c865f07cf51dfbf4a7b902579f392a
+size 121135557
diff --git a/qwen3/srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8616ec21a5344ea8fb0ce6b91891ee1827785877
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43073189d19ec9dc11474e596cf2c6d6f3653c3632145d64f352381e855d43f6
+size 121135749
diff --git a/qwen3/srkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt b/qwen3/srkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4e4671241d154e7f4d9bd41fabbcd2c031237b87
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b1d7708036706ff86c5f6839d94ba5d184edf4c42bafdc86fade97c90bc88fb
+size 351841765
diff --git a/qwen3/srkl/checkpoint-628/latest b/qwen3/srkl/checkpoint-628/latest
new file mode 100644
index 0000000000000000000000000000000000000000..af055c3db05dcb2af541005f22308dbd946aba65
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/latest
@@ -0,0 +1 @@
+global_step628
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-628/rng_state_0.pth b/qwen3/srkl/checkpoint-628/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b18e5e64a199ed0ceb192fdd3a4dba128e7d6acb
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0099eec1625f6c0a67ac4adec0324d861edfb5609428591754299184feca9cbf
+size 14917
diff --git a/qwen3/srkl/checkpoint-628/rng_state_1.pth b/qwen3/srkl/checkpoint-628/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..517d994d003fe6601a3ac5e5eb60e79bcbc0d644
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2eb20895c7ef5445818712c29d70be37d5872a2a872a52c77fe57cff42b984c
+size 14917
diff --git a/qwen3/srkl/checkpoint-628/scheduler.pt b/qwen3/srkl/checkpoint-628/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..96c0cabcd99be4945ffbd7897ecc5367ac4beb28
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb823a2c51d7fb42dfe3b693a7159784285dbf38c5a797e6e73292096123ab7a
+size 1465
diff --git a/qwen3/srkl/checkpoint-628/tokenizer.json b/qwen3/srkl/checkpoint-628/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/srkl/checkpoint-628/tokenizer_config.json b/qwen3/srkl/checkpoint-628/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/srkl/checkpoint-628/trainer_state.json b/qwen3/srkl/checkpoint-628/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..fec02ea034aabc0ca5fa0dc1f31ba55b0ec36f5f
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/trainer_state.json
@@ -0,0 +1,281 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 2.0,
+ "eval_steps": 500,
+ "global_step": 628,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.5670945644378662,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.37616620063781736,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 0.8954827785491943,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.0971335530281067,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 0.9263805747032166,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.06594716906547546,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.31774941086769104,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.04192234873771668,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.44272375106811523,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.030837148427963257,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.5705686807632446,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.017216089367866515,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.3161797821521759,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.025291597843170165,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.4870106875896454,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.022351644933223724,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 0.2289164662361145,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.01218288466334343,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.24952471256256104,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.019831493496894836,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.41395241022109985,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.012001317739486695,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.19487057626247406,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.011289550364017487,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.31937241554260254,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.008779586851596832,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.12296169996261597,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.011695237457752227,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.3308272659778595,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.011218554526567458,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 88.6682,
+ "eval_generator_rouge1": 99.5496,
+ "eval_generator_rouge2": 99.2687,
+ "eval_generator_rougeL": 99.1723,
+ "eval_loss": 0.011020934209227562,
+ "eval_runtime": 614.0637,
+ "eval_samples_per_second": 5.56,
+ "eval_selector_accuracy": 94.3925,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.174,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 0.6877385973930359,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.010994066298007966,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.1419052630662918,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.005199763923883438,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.10923358798027039,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.004498997330665588,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.2007422000169754,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.005002734437584877,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.08985759317874908,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.006419034302234649,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.19415093958377838,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.00729372426867485,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.06894819438457489,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.0035026255995035172,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.6073423624038696,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.005546892434358597,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.02788952738046646,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.00369630828499794,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.09827981144189835,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.006270292401313782,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.09948836266994476,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.005865002796053886,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.3055334985256195,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.007864721119403839,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.3340248167514801,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.0026097118854522703,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.3238213360309601,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.005317946150898934,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.06955885142087936,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.00310036838054657,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.029902024194598198,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.0033790454268455505,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 94.4509,
+ "eval_generator_rouge1": 99.8443,
+ "eval_generator_rouge2": 99.7143,
+ "eval_generator_rougeL": 99.636,
+ "eval_loss": 0.004883269313722849,
+ "eval_runtime": 610.2471,
+ "eval_samples_per_second": 5.594,
+ "eval_selector_accuracy": 97.6636,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 628
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 2.346351770520781e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/srkl/checkpoint-628/training_args.bin b/qwen3/srkl/checkpoint-628/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..31afaafa55cb5f0350c3985e07d863ac893286e4
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:109808d64c5d0b463c4a77490625c491baa90453dc3e9d7c8d7fe0958476b313
+size 7185
diff --git a/qwen3/srkl/checkpoint-628/zero_to_fp32.py b/qwen3/srkl/checkpoint-628/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/srkl/checkpoint-628/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/srkl/checkpoint-942/README.md b/qwen3/srkl/checkpoint-942/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-0.6B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-0.6B
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-942/adapter_config.json b/qwen3/srkl/checkpoint-942/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..56658aea33d649140802c320668b4fe2086ad8d0
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-0.6B",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "o_proj",
+ "down_proj",
+ "k_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-942/adapter_model.safetensors b/qwen3/srkl/checkpoint-942/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f522a22c72a4f02de23c21db91319b433448fa2b
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6bbe0fb6abd80e54809648e19285b520ca3417af270ea0b1025731b349c955c
+size 40422560
diff --git a/qwen3/srkl/checkpoint-942/chat_template.jinja b/qwen3/srkl/checkpoint-942/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/chat_template.jinja
@@ -0,0 +1,89 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in content %}
+ {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- set content = content.split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4b00f111908e6e4f2ef524d4fe8849d3ece2283e
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b47b0045ab94378c25c291d9be8f16e373be0e134c4335d2ab112abe1c3d2a44
+size 121135557
diff --git a/qwen3/srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..772321dbc30f5e3a9595cb26a79d418ab5d4ecde
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38c927e20da17d95b9f9151660fa2ff1cc54c128ef4af728e4c983d51a9f1fd9
+size 121135749
diff --git a/qwen3/srkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt b/qwen3/srkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c3cf19943ca11afa4cf76336f208795320a9cbdd
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70bab3ceb084134dce1f1bc0a7e8fabf238364787f9177a1fc2262c373fd07cf
+size 351841765
diff --git a/qwen3/srkl/checkpoint-942/latest b/qwen3/srkl/checkpoint-942/latest
new file mode 100644
index 0000000000000000000000000000000000000000..4e9a68e4460aca593c480611a486d2a4cd73ef42
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/latest
@@ -0,0 +1 @@
+global_step942
\ No newline at end of file
diff --git a/qwen3/srkl/checkpoint-942/rng_state_0.pth b/qwen3/srkl/checkpoint-942/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..1208db6f73f76e14e6ed72574b5c834dd96b96a8
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd1ab573bcec4e07719bbad480d778850a2ed94590377c04f04796e3604c111c
+size 14917
diff --git a/qwen3/srkl/checkpoint-942/rng_state_1.pth b/qwen3/srkl/checkpoint-942/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..3ea22163992e792545f1bc22105f75c85999e165
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d78e89d033f15f5e7c21dfcd3505ffb57cd8652ff8a47c3792b3f7e2fd5a24f
+size 14917
diff --git a/qwen3/srkl/checkpoint-942/scheduler.pt b/qwen3/srkl/checkpoint-942/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b1f8af0480bf83ab728abfeff11bc3288214349
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8076da6f897951c5bdf8649e4fba596e8c49819927746017442ffd85f40b18a3
+size 1465
diff --git a/qwen3/srkl/checkpoint-942/tokenizer.json b/qwen3/srkl/checkpoint-942/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/srkl/checkpoint-942/tokenizer_config.json b/qwen3/srkl/checkpoint-942/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/srkl/checkpoint-942/trainer_state.json b/qwen3/srkl/checkpoint-942/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..cc1ad8e6f2b9bf2272ff55c022f2329cbb45f1cd
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/trainer_state.json
@@ -0,0 +1,408 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 3.0,
+ "eval_steps": 500,
+ "global_step": 942,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.5670945644378662,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.37616620063781736,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 0.8954827785491943,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.0971335530281067,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 0.9263805747032166,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.06594716906547546,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.31774941086769104,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.04192234873771668,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.44272375106811523,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.030837148427963257,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.5705686807632446,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.017216089367866515,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.3161797821521759,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.025291597843170165,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.4870106875896454,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.022351644933223724,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 0.2289164662361145,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.01218288466334343,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.24952471256256104,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.019831493496894836,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.41395241022109985,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.012001317739486695,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.19487057626247406,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.011289550364017487,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.31937241554260254,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.008779586851596832,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.12296169996261597,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.011695237457752227,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.3308272659778595,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.011218554526567458,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 88.6682,
+ "eval_generator_rouge1": 99.5496,
+ "eval_generator_rouge2": 99.2687,
+ "eval_generator_rougeL": 99.1723,
+ "eval_loss": 0.011020934209227562,
+ "eval_runtime": 614.0637,
+ "eval_samples_per_second": 5.56,
+ "eval_selector_accuracy": 94.3925,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.174,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 0.6877385973930359,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.010994066298007966,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.1419052630662918,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.005199763923883438,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.10923358798027039,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.004498997330665588,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.2007422000169754,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.005002734437584877,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.08985759317874908,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.006419034302234649,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.19415093958377838,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.00729372426867485,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.06894819438457489,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.0035026255995035172,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.6073423624038696,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.005546892434358597,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.02788952738046646,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.00369630828499794,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.09827981144189835,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.006270292401313782,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.09948836266994476,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.005865002796053886,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.3055334985256195,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.007864721119403839,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.3340248167514801,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.0026097118854522703,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.3238213360309601,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.005317946150898934,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.06955885142087936,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.00310036838054657,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.029902024194598198,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.0033790454268455505,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 94.4509,
+ "eval_generator_rouge1": 99.8443,
+ "eval_generator_rouge2": 99.7143,
+ "eval_generator_rougeL": 99.636,
+ "eval_loss": 0.004883269313722849,
+ "eval_runtime": 610.2471,
+ "eval_samples_per_second": 5.594,
+ "eval_selector_accuracy": 97.6636,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.08393510431051254,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.002889341861009598,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.317868173122406,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.002996666356921196,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.03327847272157669,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0018069516867399217,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.06478872895240784,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0016909070312976838,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.19922314584255219,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.0016838768497109413,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.048591550439596176,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.0019782688468694687,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.0104411281645298,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0010088316164910793,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.04026724770665169,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0007452124729752541,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.07828482985496521,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.003089136257767677,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.206737220287323,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.0016609614714980126,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.0661257803440094,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.0017474986612796783,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.14079469442367554,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.003954095393419265,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.056867823004722595,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0016886677592992782,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.10408495366573334,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0013717899098992347,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.003307907609269023,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.001446123607456684,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.07255368679761887,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0011631335131824017,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.7944,
+ "eval_generator_rouge1": 99.8488,
+ "eval_generator_rouge2": 99.7613,
+ "eval_generator_rougeL": 99.6161,
+ "eval_loss": 0.004494966473430395,
+ "eval_runtime": 600.8182,
+ "eval_samples_per_second": 5.682,
+ "eval_selector_accuracy": 98.2477,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.178,
+ "step": 942
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 3.524157743353037e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/srkl/checkpoint-942/training_args.bin b/qwen3/srkl/checkpoint-942/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..31afaafa55cb5f0350c3985e07d863ac893286e4
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:109808d64c5d0b463c4a77490625c491baa90453dc3e9d7c8d7fe0958476b313
+size 7185
diff --git a/qwen3/srkl/checkpoint-942/zero_to_fp32.py b/qwen3/srkl/checkpoint-942/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/srkl/checkpoint-942/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/srkl/eval_results.json b/qwen3/srkl/eval_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..7e016a5003e7d424ba997e73464fe6405a12d6b7
--- /dev/null
+++ b/qwen3/srkl/eval_results.json
@@ -0,0 +1,14 @@
+{
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.729,
+ "eval_generator_rouge1": 99.894,
+ "eval_generator_rouge2": 99.8306,
+ "eval_generator_rougeL": 99.7434,
+ "eval_loss": 0.003705898765474558,
+ "eval_runtime": 588.9515,
+ "eval_samples_per_second": 5.797,
+ "eval_selector_accuracy": 98.715,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.182
+}
\ No newline at end of file
diff --git a/qwen3/srkl/tokenizer.json b/qwen3/srkl/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/srkl/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/srkl/tokenizer_config.json b/qwen3/srkl/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c
--- /dev/null
+++ b/qwen3/srkl/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/srkl/train_results.json b/qwen3/srkl/train_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..1b31062a8df348a51fe8ff6144c0b2534451e410
--- /dev/null
+++ b/qwen3/srkl/train_results.json
@@ -0,0 +1,8 @@
+{
+ "epoch": 5.0,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.011693778396495113,
+ "train_runtime": 7720.2334,
+ "train_samples_per_second": 6.494,
+ "train_steps_per_second": 0.203
+}
\ No newline at end of file
diff --git a/qwen3/srkl/trainer_state.json b/qwen3/srkl/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..596744d361fdbaf5e6f07754d05213a0801d7980
--- /dev/null
+++ b/qwen3/srkl/trainer_state.json
@@ -0,0 +1,664 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 5.0,
+ "eval_steps": 500,
+ "global_step": 1570,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06382130035899482,
+ "grad_norm": 1.5670945644378662,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.37616620063781736,
+ "step": 20
+ },
+ {
+ "epoch": 0.12764260071798963,
+ "grad_norm": 0.8954827785491943,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.0971335530281067,
+ "step": 40
+ },
+ {
+ "epoch": 0.19146390107698444,
+ "grad_norm": 0.9263805747032166,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.06594716906547546,
+ "step": 60
+ },
+ {
+ "epoch": 0.25528520143597927,
+ "grad_norm": 0.31774941086769104,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.04192234873771668,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191065017949741,
+ "grad_norm": 0.44272375106811523,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.030837148427963257,
+ "step": 100
+ },
+ {
+ "epoch": 0.3829278021539689,
+ "grad_norm": 0.5705686807632446,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.017216089367866515,
+ "step": 120
+ },
+ {
+ "epoch": 0.4467491025129637,
+ "grad_norm": 0.3161797821521759,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.025291597843170165,
+ "step": 140
+ },
+ {
+ "epoch": 0.5105704028719585,
+ "grad_norm": 0.4870106875896454,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.022351644933223724,
+ "step": 160
+ },
+ {
+ "epoch": 0.5743917032309533,
+ "grad_norm": 0.2289164662361145,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.01218288466334343,
+ "step": 180
+ },
+ {
+ "epoch": 0.6382130035899481,
+ "grad_norm": 0.24952471256256104,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.019831493496894836,
+ "step": 200
+ },
+ {
+ "epoch": 0.702034303948943,
+ "grad_norm": 0.41395241022109985,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.012001317739486695,
+ "step": 220
+ },
+ {
+ "epoch": 0.7658556043079378,
+ "grad_norm": 0.19487057626247406,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.011289550364017487,
+ "step": 240
+ },
+ {
+ "epoch": 0.8296769046669326,
+ "grad_norm": 0.31937241554260254,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.008779586851596832,
+ "step": 260
+ },
+ {
+ "epoch": 0.8934982050259274,
+ "grad_norm": 0.12296169996261597,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.011695237457752227,
+ "step": 280
+ },
+ {
+ "epoch": 0.9573195053849222,
+ "grad_norm": 0.3308272659778595,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.011218554526567458,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 88.6682,
+ "eval_generator_rouge1": 99.5496,
+ "eval_generator_rouge2": 99.2687,
+ "eval_generator_rougeL": 99.1723,
+ "eval_loss": 0.011020934209227562,
+ "eval_runtime": 614.0637,
+ "eval_samples_per_second": 5.56,
+ "eval_selector_accuracy": 94.3925,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.174,
+ "step": 314
+ },
+ {
+ "epoch": 1.0191463901076985,
+ "grad_norm": 0.6877385973930359,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.010994066298007966,
+ "step": 320
+ },
+ {
+ "epoch": 1.0829676904666932,
+ "grad_norm": 0.1419052630662918,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.005199763923883438,
+ "step": 340
+ },
+ {
+ "epoch": 1.146788990825688,
+ "grad_norm": 0.10923358798027039,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.004498997330665588,
+ "step": 360
+ },
+ {
+ "epoch": 1.210610291184683,
+ "grad_norm": 0.2007422000169754,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.005002734437584877,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744315915436777,
+ "grad_norm": 0.08985759317874908,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.006419034302234649,
+ "step": 400
+ },
+ {
+ "epoch": 1.3382528919026724,
+ "grad_norm": 0.19415093958377838,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.00729372426867485,
+ "step": 420
+ },
+ {
+ "epoch": 1.4020741922616673,
+ "grad_norm": 0.06894819438457489,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.0035026255995035172,
+ "step": 440
+ },
+ {
+ "epoch": 1.4658954926206622,
+ "grad_norm": 0.6073423624038696,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.005546892434358597,
+ "step": 460
+ },
+ {
+ "epoch": 1.5297167929796571,
+ "grad_norm": 0.02788952738046646,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.00369630828499794,
+ "step": 480
+ },
+ {
+ "epoch": 1.5935380933386518,
+ "grad_norm": 0.09827981144189835,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.006270292401313782,
+ "step": 500
+ },
+ {
+ "epoch": 1.6573593936976465,
+ "grad_norm": 0.09948836266994476,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.005865002796053886,
+ "step": 520
+ },
+ {
+ "epoch": 1.7211806940566414,
+ "grad_norm": 0.3055334985256195,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.007864721119403839,
+ "step": 540
+ },
+ {
+ "epoch": 1.7850019944156363,
+ "grad_norm": 0.3340248167514801,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.0026097118854522703,
+ "step": 560
+ },
+ {
+ "epoch": 1.848823294774631,
+ "grad_norm": 0.3238213360309601,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.005317946150898934,
+ "step": 580
+ },
+ {
+ "epoch": 1.9126445951336257,
+ "grad_norm": 0.06955885142087936,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.00310036838054657,
+ "step": 600
+ },
+ {
+ "epoch": 1.9764658954926206,
+ "grad_norm": 0.029902024194598198,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.0033790454268455505,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 94.4509,
+ "eval_generator_rouge1": 99.8443,
+ "eval_generator_rouge2": 99.7143,
+ "eval_generator_rougeL": 99.636,
+ "eval_loss": 0.004883269313722849,
+ "eval_runtime": 610.2471,
+ "eval_samples_per_second": 5.594,
+ "eval_selector_accuracy": 97.6636,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.175,
+ "step": 628
+ },
+ {
+ "epoch": 2.038292780215397,
+ "grad_norm": 0.08393510431051254,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.002889341861009598,
+ "step": 640
+ },
+ {
+ "epoch": 2.102114080574392,
+ "grad_norm": 0.317868173122406,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.002996666356921196,
+ "step": 660
+ },
+ {
+ "epoch": 2.1659353809333863,
+ "grad_norm": 0.03327847272157669,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0018069516867399217,
+ "step": 680
+ },
+ {
+ "epoch": 2.2297566812923812,
+ "grad_norm": 0.06478872895240784,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0016909070312976838,
+ "step": 700
+ },
+ {
+ "epoch": 2.293577981651376,
+ "grad_norm": 0.19922314584255219,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.0016838768497109413,
+ "step": 720
+ },
+ {
+ "epoch": 2.357399282010371,
+ "grad_norm": 0.048591550439596176,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.0019782688468694687,
+ "step": 740
+ },
+ {
+ "epoch": 2.421220582369366,
+ "grad_norm": 0.0104411281645298,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0010088316164910793,
+ "step": 760
+ },
+ {
+ "epoch": 2.4850418827283605,
+ "grad_norm": 0.04026724770665169,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.0007452124729752541,
+ "step": 780
+ },
+ {
+ "epoch": 2.5488631830873554,
+ "grad_norm": 0.07828482985496521,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.003089136257767677,
+ "step": 800
+ },
+ {
+ "epoch": 2.6126844834463503,
+ "grad_norm": 0.206737220287323,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.0016609614714980126,
+ "step": 820
+ },
+ {
+ "epoch": 2.6765057838053448,
+ "grad_norm": 0.0661257803440094,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.0017474986612796783,
+ "step": 840
+ },
+ {
+ "epoch": 2.74032708416434,
+ "grad_norm": 0.14079469442367554,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.003954095393419265,
+ "step": 860
+ },
+ {
+ "epoch": 2.8041483845233346,
+ "grad_norm": 0.056867823004722595,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0016886677592992782,
+ "step": 880
+ },
+ {
+ "epoch": 2.8679696848823295,
+ "grad_norm": 0.10408495366573334,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.0013717899098992347,
+ "step": 900
+ },
+ {
+ "epoch": 2.9317909852413244,
+ "grad_norm": 0.003307907609269023,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.001446123607456684,
+ "step": 920
+ },
+ {
+ "epoch": 2.995612285600319,
+ "grad_norm": 0.07255368679761887,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0011631335131824017,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 95.7944,
+ "eval_generator_rouge1": 99.8488,
+ "eval_generator_rouge2": 99.7613,
+ "eval_generator_rougeL": 99.6161,
+ "eval_loss": 0.004494966473430395,
+ "eval_runtime": 600.8182,
+ "eval_samples_per_second": 5.682,
+ "eval_selector_accuracy": 98.2477,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.178,
+ "step": 942
+ },
+ {
+ "epoch": 3.057439170323095,
+ "grad_norm": 0.07352045923471451,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.002453933656215668,
+ "step": 960
+ },
+ {
+ "epoch": 3.12126047068209,
+ "grad_norm": 0.0014208417851477861,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.0009017683565616608,
+ "step": 980
+ },
+ {
+ "epoch": 3.185081771041085,
+ "grad_norm": 0.12319519370794296,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.0010159458965063096,
+ "step": 1000
+ },
+ {
+ "epoch": 3.24890307140008,
+ "grad_norm": 0.016385605558753014,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.002848365902900696,
+ "step": 1020
+ },
+ {
+ "epoch": 3.3127243717590744,
+ "grad_norm": 0.14579907059669495,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.0006675420328974724,
+ "step": 1040
+ },
+ {
+ "epoch": 3.3765456721180693,
+ "grad_norm": 0.0019410790409892797,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.0011831691488623619,
+ "step": 1060
+ },
+ {
+ "epoch": 3.4403669724770642,
+ "grad_norm": 0.07082991302013397,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.0023162638768553733,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504188272836059,
+ "grad_norm": 0.01225790660828352,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0022976718842983245,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568009573195054,
+ "grad_norm": 0.012785660102963448,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.0013264333829283714,
+ "step": 1120
+ },
+ {
+ "epoch": 3.6318308735540485,
+ "grad_norm": 0.0714384987950325,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0007606681436300277,
+ "step": 1140
+ },
+ {
+ "epoch": 3.6956521739130435,
+ "grad_norm": 0.5528075695037842,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.002803182043135166,
+ "step": 1160
+ },
+ {
+ "epoch": 3.7594734742720384,
+ "grad_norm": 0.01241183653473854,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0012427497655153274,
+ "step": 1180
+ },
+ {
+ "epoch": 3.823294774631033,
+ "grad_norm": 0.0505630299448967,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.001273194793611765,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8871160749900278,
+ "grad_norm": 0.03439785912632942,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.0007076818961650133,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9509373753490227,
+ "grad_norm": 0.016928531229496002,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0005449302028864622,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.4953,
+ "eval_generator_rouge1": 99.9005,
+ "eval_generator_rouge2": 99.8353,
+ "eval_generator_rougeL": 99.7303,
+ "eval_loss": 0.0036805884446948767,
+ "eval_runtime": 601.6399,
+ "eval_samples_per_second": 5.674,
+ "eval_selector_accuracy": 98.6565,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.178,
+ "step": 1256
+ },
+ {
+ "epoch": 4.0127642600717985,
+ "grad_norm": 0.026434868574142456,
+ "learning_rate": 9.464081721960592e-06,
+ "loss": 0.0009123041294515133,
+ "step": 1260
+ },
+ {
+ "epoch": 4.076585560430794,
+ "grad_norm": 0.054495662450790405,
+ "learning_rate": 8.331628936475818e-06,
+ "loss": 0.0002502598334103823,
+ "step": 1280
+ },
+ {
+ "epoch": 4.140406860789788,
+ "grad_norm": 0.004517420195043087,
+ "learning_rate": 7.2659843568748385e-06,
+ "loss": 0.0005662859417498112,
+ "step": 1300
+ },
+ {
+ "epoch": 4.204228161148784,
+ "grad_norm": 0.01496294979006052,
+ "learning_rate": 6.268854516770008e-06,
+ "loss": 0.0004728314466774464,
+ "step": 1320
+ },
+ {
+ "epoch": 4.268049461507778,
+ "grad_norm": 0.03497197851538658,
+ "learning_rate": 5.341836229602331e-06,
+ "loss": 0.0007120450958609581,
+ "step": 1340
+ },
+ {
+ "epoch": 4.331870761866773,
+ "grad_norm": 0.006169375032186508,
+ "learning_rate": 4.4864140314888216e-06,
+ "loss": 0.0014027132652699946,
+ "step": 1360
+ },
+ {
+ "epoch": 4.395692062225768,
+ "grad_norm": 0.02854115329682827,
+ "learning_rate": 3.703957803871932e-06,
+ "loss": 0.0008245049975812435,
+ "step": 1380
+ },
+ {
+ "epoch": 4.4595133625847625,
+ "grad_norm": 0.03609360754489899,
+ "learning_rate": 2.9957205797780495e-06,
+ "loss": 0.0009156657382845878,
+ "step": 1400
+ },
+ {
+ "epoch": 4.523334662943758,
+ "grad_norm": 0.04012208431959152,
+ "learning_rate": 2.362836537198199e-06,
+ "loss": 0.0006599740125238895,
+ "step": 1420
+ },
+ {
+ "epoch": 4.587155963302752,
+ "grad_norm": 0.018918916583061218,
+ "learning_rate": 1.8063191828043703e-06,
+ "loss": 0.0006631083786487579,
+ "step": 1440
+ },
+ {
+ "epoch": 4.650977263661747,
+ "grad_norm": 0.021513519808650017,
+ "learning_rate": 1.3270597289100437e-06,
+ "loss": 0.0003777497448027134,
+ "step": 1460
+ },
+ {
+ "epoch": 4.714798564020742,
+ "grad_norm": 0.008279121480882168,
+ "learning_rate": 9.258256662741205e-07,
+ "loss": 0.0005044300109148025,
+ "step": 1480
+ },
+ {
+ "epoch": 4.778619864379737,
+ "grad_norm": 0.012552818283438683,
+ "learning_rate": 6.032595350337341e-07,
+ "loss": 0.0006919664330780507,
+ "step": 1500
+ },
+ {
+ "epoch": 4.842441164738732,
+ "grad_norm": 0.0035533979535102844,
+ "learning_rate": 3.5987789573421356e-07,
+ "loss": 0.004239285737276078,
+ "step": 1520
+ },
+ {
+ "epoch": 4.9062624650977265,
+ "grad_norm": 0.0733393207192421,
+ "learning_rate": 1.9607050210401192e-07,
+ "loss": 0.0004377847071737051,
+ "step": 1540
+ },
+ {
+ "epoch": 4.970083765456721,
+ "grad_norm": 0.02573123201727867,
+ "learning_rate": 1.1209967689925832e-07,
+ "loss": 0.0004438157659024,
+ "step": 1560
+ },
+ {
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.9042,
+ "eval_generator_rouge1": 99.8996,
+ "eval_generator_rouge2": 99.839,
+ "eval_generator_rougeL": 99.7647,
+ "eval_loss": 0.003705898765474558,
+ "eval_runtime": 602.6617,
+ "eval_samples_per_second": 5.665,
+ "eval_selector_accuracy": 98.6565,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.178,
+ "step": 1570
+ },
+ {
+ "epoch": 5.0,
+ "step": 1570,
+ "total_flos": 5.871444840310374e+16,
+ "train_loss": 0.011693778396495113,
+ "train_runtime": 7720.2334,
+ "train_samples_per_second": 6.494,
+ "train_steps_per_second": 0.203
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 5.871444840310374e+16,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/srkl/training_args.bin b/qwen3/srkl/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..31afaafa55cb5f0350c3985e07d863ac893286e4
--- /dev/null
+++ b/qwen3/srkl/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:109808d64c5d0b463c4a77490625c491baa90453dc3e9d7c8d7fe0958476b313
+size 7185
diff --git a/qwen3/teacher_lora/README.md b/qwen3/teacher_lora/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..0d47c989c9598da30f24449d8d18329c77504bb2
--- /dev/null
+++ b/qwen3/teacher_lora/README.md
@@ -0,0 +1,79 @@
+---
+library_name: peft
+license: other
+base_model: Qwen/Qwen3-4B-Instruct-2507
+tags:
+- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507
+- llama-factory
+- lora
+- transformers
+pipeline_tag: text-generation
+model-index:
+- name: teacher_lora
+ results: []
+---
+
+
+
+# teacher_lora
+
+This model is a fine-tuned version of [Qwen/Qwen3-4B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507) on the cypher_prepared_train dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0029
+- Selector Count: 1712.0
+- Generator Count: 1712.0
+- Selector Accuracy: 99.2407
+- Generator Exact Match: 97.0794
+- Generator Rouge1: 99.9517
+- Generator Rouge2: 99.914
+- Generator Rougel: 99.7744
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 1
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 32
+- total_eval_batch_size: 16
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine_with_min_lr
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Selector Count | Generator Count | Selector Accuracy | Generator Exact Match | Generator Rouge1 | Generator Rouge2 | Generator Rougel |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:---------------:|:-----------------:|:---------------------:|:----------------:|:----------------:|:----------------:|
+| 0.0124 | 1.0 | 314 | 0.0044 | 1712.0 | 1712.0 | 98.014 | 94.6262 | 99.8758 | 99.7812 | 99.5997 |
+| 0.0073 | 2.0 | 628 | 0.0033 | 1712.0 | 1712.0 | 98.4813 | 96.2617 | 99.9321 | 99.8752 | 99.7349 |
+| 0.0005 | 3.0 | 942 | 0.0028 | 1712.0 | 1712.0 | 99.1822 | 96.8458 | 99.9512 | 99.9138 | 99.7589 |
+| 0.0004 | 4.0 | 1256 | 0.0029 | 1712.0 | 1712.0 | 99.1822 | 96.7874 | 99.9518 | 99.9142 | 99.7472 |
+| 0.0003 | 5.0 | 1570 | 0.0029 | 1712.0 | 1712.0 | 99.2407 | 97.0794 | 99.9483 | 99.9036 | 99.7893 |
+
+
+### Framework versions
+
+- PEFT 0.18.1
+- Transformers 5.8.0
+- Pytorch 2.9.1+cu128
+- Datasets 4.0.0
+- Tokenizers 0.22.2
\ No newline at end of file
diff --git a/qwen3/teacher_lora/adapter_config.json b/qwen3/teacher_lora/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..376dee7aec128c9bdb50c2cc1ddcf6a68a37ab9b
--- /dev/null
+++ b/qwen3/teacher_lora/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "down_proj",
+ "k_proj",
+ "up_proj",
+ "o_proj",
+ "q_proj",
+ "v_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/teacher_lora/adapter_model.safetensors b/qwen3/teacher_lora/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9c06f5d11bbc6b39df9ee0caf6b30c7edbdafb0e
--- /dev/null
+++ b/qwen3/teacher_lora/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cefa38e0ed411f3656d30fffca3cad60870d758d78c6b5cc3d93a1d4494dc6c
+size 132188392
diff --git a/qwen3/teacher_lora/all_results.json b/qwen3/teacher_lora/all_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..5f82a36d6097dbb7125dc072246a7a106d24fa49
--- /dev/null
+++ b/qwen3/teacher_lora/all_results.json
@@ -0,0 +1,19 @@
+{
+ "epoch": 5.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 97.0794,
+ "eval_generator_rouge1": 99.9517,
+ "eval_generator_rouge2": 99.914,
+ "eval_generator_rougeL": 99.7744,
+ "eval_loss": 0.00291790091432631,
+ "eval_runtime": 1472.9037,
+ "eval_samples_per_second": 2.318,
+ "eval_selector_accuracy": 99.2407,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.145,
+ "total_flos": 3.981241098143007e+17,
+ "train_loss": 0.009030620363736704,
+ "train_runtime": 16300.8877,
+ "train_samples_per_second": 3.076,
+ "train_steps_per_second": 0.096
+}
\ No newline at end of file
diff --git a/qwen3/teacher_lora/chat_template.jinja b/qwen3/teacher_lora/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..70adff8a08fb31e0636f618564838d4bf3c05286
--- /dev/null
+++ b/qwen3/teacher_lora/chat_template.jinja
@@ -0,0 +1,61 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/teacher_lora/checkpoint-1256/README.md b/qwen3/teacher_lora/checkpoint-1256/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..4fabf4f222ce5d86630bcfbcae8e90fb08d87580
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-4B-Instruct-2507
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/qwen3/teacher_lora/checkpoint-1256/adapter_config.json b/qwen3/teacher_lora/checkpoint-1256/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..376dee7aec128c9bdb50c2cc1ddcf6a68a37ab9b
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 64,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 32,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "down_proj",
+ "k_proj",
+ "up_proj",
+ "o_proj",
+ "q_proj",
+ "v_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/qwen3/teacher_lora/checkpoint-1256/adapter_model.safetensors b/qwen3/teacher_lora/checkpoint-1256/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fcda29eb64ecc3d42a93d0b6ad5e1b24ff4cc166
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97eec08e38ac9549162a37edbf711edc0ae2457844a997d96b66ef22b6c8f227
+size 132188392
diff --git a/qwen3/teacher_lora/checkpoint-1256/chat_template.jinja b/qwen3/teacher_lora/checkpoint-1256/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..70adff8a08fb31e0636f618564838d4bf3c05286
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/chat_template.jinja
@@ -0,0 +1,61 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if message.content is string %}
+ {%- set content = message.content %}
+ {%- else %}
+ {%- set content = '' %}
+ {%- endif %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
\ No newline at end of file
diff --git a/qwen3/teacher_lora/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/teacher_lora/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3d63ac3ee15cd694a97099a45a284b36cfa8ee30
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9de3830b4daecc416785b80b7d96e8017ea51d488d26d18a3a754f821223033
+size 396393221
diff --git a/qwen3/teacher_lora/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/teacher_lora/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b5ed1f1ab18e7b868f11d50214adaa61063aeae2
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edd670315af41ae9f0da73d66f800847a7c3f3610b3c2f8e9693cb313be2065f
+size 396393349
diff --git a/qwen3/teacher_lora/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt b/qwen3/teacher_lora/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6a587a9a9996ede85e45b55cb17a5bc8623a78b5
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a5870e5f6ea1a5b67eb24b77996e5415db6cd28c26c75a07108b14a39c98706
+size 910426661
diff --git a/qwen3/teacher_lora/checkpoint-1256/latest b/qwen3/teacher_lora/checkpoint-1256/latest
new file mode 100644
index 0000000000000000000000000000000000000000..74abb3d23ea83ebdac59918752db5d1782834ca7
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/latest
@@ -0,0 +1 @@
+global_step1256
\ No newline at end of file
diff --git a/qwen3/teacher_lora/checkpoint-1256/rng_state_0.pth b/qwen3/teacher_lora/checkpoint-1256/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..53f2e6b2cc0f1d2605a82be84f57439b6bc482d3
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6f046c903a9ad0f41997fa74deeb98f8a1175223f028fbac6408821e4b03996
+size 14917
diff --git a/qwen3/teacher_lora/checkpoint-1256/rng_state_1.pth b/qwen3/teacher_lora/checkpoint-1256/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4c011dd0da7395bd146f77b14b4489a11ee730db
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a03eec70b415f8516415062a6431ab754c521d80e0ac4d763d76a6661c842da2
+size 14917
diff --git a/qwen3/teacher_lora/checkpoint-1256/scheduler.pt b/qwen3/teacher_lora/checkpoint-1256/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..40e863d07cfed22cd912014eb3ba3fd2b16b173f
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1775297f14e75f7463ae51983a66296f81f4b2e2e5d7234f1a6d295abfa4cf23
+size 1465
diff --git a/qwen3/teacher_lora/checkpoint-1256/tokenizer.json b/qwen3/teacher_lora/checkpoint-1256/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650
diff --git a/qwen3/teacher_lora/checkpoint-1256/tokenizer_config.json b/qwen3/teacher_lora/checkpoint-1256/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..907e863a5a4109fb0e373ccdeb4d657bbeca9416
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/tokenizer_config.json
@@ -0,0 +1,31 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 1010000,
+ "pad_token": "<|endoftext|>",
+ "padding_side": "right",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/qwen3/teacher_lora/checkpoint-1256/trainer_state.json b/qwen3/teacher_lora/checkpoint-1256/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..86f6dcd97cc1bb52386f51ca0f3bf7e78a336b3d
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/trainer_state.json
@@ -0,0 +1,528 @@
+{
+ "best_global_step": null,
+ "best_metric": null,
+ "best_model_checkpoint": null,
+ "epoch": 4.0,
+ "eval_steps": 500,
+ "global_step": 1256,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.06383403151805306,
+ "grad_norm": 0.4070272743701935,
+ "learning_rate": 9.996390385474569e-05,
+ "loss": 0.37070536613464355,
+ "step": 20
+ },
+ {
+ "epoch": 0.12766806303610612,
+ "grad_norm": 0.5005771517753601,
+ "learning_rate": 9.984797509888031e-05,
+ "loss": 0.03893736004829407,
+ "step": 40
+ },
+ {
+ "epoch": 0.19150209455415917,
+ "grad_norm": 0.17237664759159088,
+ "learning_rate": 9.965229938050925e-05,
+ "loss": 0.02901872992515564,
+ "step": 60
+ },
+ {
+ "epoch": 0.25533612607221223,
+ "grad_norm": 0.17700566351413727,
+ "learning_rate": 9.937719005663432e-05,
+ "loss": 0.012938253581523895,
+ "step": 80
+ },
+ {
+ "epoch": 0.3191701575902653,
+ "grad_norm": 0.29617995023727417,
+ "learning_rate": 9.902308769000724e-05,
+ "loss": 0.01476612538099289,
+ "step": 100
+ },
+ {
+ "epoch": 0.38300418910831835,
+ "grad_norm": 0.09139569103717804,
+ "learning_rate": 9.859055934360789e-05,
+ "loss": 0.013924291729927063,
+ "step": 120
+ },
+ {
+ "epoch": 0.4468382206263714,
+ "grad_norm": 0.6957159042358398,
+ "learning_rate": 9.808029767254437e-05,
+ "loss": 0.015140479803085327,
+ "step": 140
+ },
+ {
+ "epoch": 0.5106722521444245,
+ "grad_norm": 0.19447490572929382,
+ "learning_rate": 9.749311981482817e-05,
+ "loss": 0.016634005308151244,
+ "step": 160
+ },
+ {
+ "epoch": 0.5745062836624776,
+ "grad_norm": 0.13639570772647858,
+ "learning_rate": 9.682996608280154e-05,
+ "loss": 0.008574086427688598,
+ "step": 180
+ },
+ {
+ "epoch": 0.6383403151805306,
+ "grad_norm": 0.05407094210386276,
+ "learning_rate": 9.609189845731207e-05,
+ "loss": 0.00992567539215088,
+ "step": 200
+ },
+ {
+ "epoch": 0.7021743466985837,
+ "grad_norm": 0.08482213318347931,
+ "learning_rate": 9.52800988870462e-05,
+ "loss": 0.009285379946231843,
+ "step": 220
+ },
+ {
+ "epoch": 0.7660083782166367,
+ "grad_norm": 0.07621290534734726,
+ "learning_rate": 9.439586739574513e-05,
+ "loss": 0.004985855519771576,
+ "step": 240
+ },
+ {
+ "epoch": 0.8298424097346898,
+ "grad_norm": 0.3552233576774597,
+ "learning_rate": 9.344062000033414e-05,
+ "loss": 0.014777910709381104,
+ "step": 260
+ },
+ {
+ "epoch": 0.8936764412527428,
+ "grad_norm": 0.07681854814291,
+ "learning_rate": 9.241588644329919e-05,
+ "loss": 0.0040129125118255615,
+ "step": 280
+ },
+ {
+ "epoch": 0.9575104727707959,
+ "grad_norm": 0.08489467203617096,
+ "learning_rate": 9.132330774294252e-05,
+ "loss": 0.012399718165397644,
+ "step": 300
+ },
+ {
+ "epoch": 1.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 94.6262,
+ "eval_generator_rouge1": 99.8758,
+ "eval_generator_rouge2": 99.7812,
+ "eval_generator_rougeL": 99.5997,
+ "eval_loss": 0.004354734905064106,
+ "eval_runtime": 1522.2841,
+ "eval_samples_per_second": 2.243,
+ "eval_selector_accuracy": 98.014,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.141,
+ "step": 314
+ },
+ {
+ "epoch": 1.019150209455416,
+ "grad_norm": 0.23293638229370117,
+ "learning_rate": 9.01646335654399e-05,
+ "loss": 0.011312296986579895,
+ "step": 320
+ },
+ {
+ "epoch": 1.082984240973469,
+ "grad_norm": 0.07232551276683807,
+ "learning_rate": 8.894171942290817e-05,
+ "loss": 0.006549905985593796,
+ "step": 340
+ },
+ {
+ "epoch": 1.1468182724915221,
+ "grad_norm": 0.05371030047535896,
+ "learning_rate": 8.765652370197003e-05,
+ "loss": 0.003336009383201599,
+ "step": 360
+ },
+ {
+ "epoch": 1.2106523040095751,
+ "grad_norm": 0.1448812186717987,
+ "learning_rate": 8.631110452757467e-05,
+ "loss": 0.008410868048667908,
+ "step": 380
+ },
+ {
+ "epoch": 1.2744863355276281,
+ "grad_norm": 0.0636834055185318,
+ "learning_rate": 8.490761646709642e-05,
+ "loss": 0.006242972612380981,
+ "step": 400
+ },
+ {
+ "epoch": 1.3383203670456814,
+ "grad_norm": 0.05049936845898628,
+ "learning_rate": 8.344830707998953e-05,
+ "loss": 0.00744413286447525,
+ "step": 420
+ },
+ {
+ "epoch": 1.4021543985637344,
+ "grad_norm": 0.032405462116003036,
+ "learning_rate": 8.193551331852458e-05,
+ "loss": 0.0031117375940084456,
+ "step": 440
+ },
+ {
+ "epoch": 1.4659884300817874,
+ "grad_norm": 0.13639135658740997,
+ "learning_rate": 8.037165778537033e-05,
+ "loss": 0.005259513109922409,
+ "step": 460
+ },
+ {
+ "epoch": 1.5298224615998404,
+ "grad_norm": 0.021995026618242264,
+ "learning_rate": 7.875924485401427e-05,
+ "loss": 0.004164557158946991,
+ "step": 480
+ },
+ {
+ "epoch": 1.5936564931178934,
+ "grad_norm": 0.01941438391804695,
+ "learning_rate": 7.710085665823444e-05,
+ "loss": 0.0015524964779615401,
+ "step": 500
+ },
+ {
+ "epoch": 1.6574905246359466,
+ "grad_norm": 0.1271296739578247,
+ "learning_rate": 7.539914895704521e-05,
+ "loss": 0.004233330115675926,
+ "step": 520
+ },
+ {
+ "epoch": 1.7213245561539996,
+ "grad_norm": 0.22600136697292328,
+ "learning_rate": 7.365684688173908e-05,
+ "loss": 0.0029299482703208923,
+ "step": 540
+ },
+ {
+ "epoch": 1.7851585876720528,
+ "grad_norm": 0.0755968764424324,
+ "learning_rate": 7.18767405718347e-05,
+ "loss": 0.0044687166810035706,
+ "step": 560
+ },
+ {
+ "epoch": 1.8489926191901058,
+ "grad_norm": 0.08520206063985825,
+ "learning_rate": 7.006168070692036e-05,
+ "loss": 0.005520488321781159,
+ "step": 580
+ },
+ {
+ "epoch": 1.9128266507081588,
+ "grad_norm": 0.008491385728120804,
+ "learning_rate": 6.821457394154786e-05,
+ "loss": 0.0015786593779921533,
+ "step": 600
+ },
+ {
+ "epoch": 1.9766606822262118,
+ "grad_norm": 0.009171651676297188,
+ "learning_rate": 6.633837825048768e-05,
+ "loss": 0.007294930517673492,
+ "step": 620
+ },
+ {
+ "epoch": 2.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.2617,
+ "eval_generator_rouge1": 99.9321,
+ "eval_generator_rouge2": 99.8752,
+ "eval_generator_rougeL": 99.7349,
+ "eval_loss": 0.0033019811380654573,
+ "eval_runtime": 1482.8547,
+ "eval_samples_per_second": 2.302,
+ "eval_selector_accuracy": 98.4813,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.144,
+ "step": 628
+ },
+ {
+ "epoch": 2.038300418910832,
+ "grad_norm": 0.0519501231610775,
+ "learning_rate": 6.443609819179926e-05,
+ "loss": 0.003084934316575527,
+ "step": 640
+ },
+ {
+ "epoch": 2.102134450428885,
+ "grad_norm": 0.051104024052619934,
+ "learning_rate": 6.251078009530256e-05,
+ "loss": 0.0026566479355096817,
+ "step": 660
+ },
+ {
+ "epoch": 2.165968481946938,
+ "grad_norm": 0.011012597940862179,
+ "learning_rate": 6.0565507184155677e-05,
+ "loss": 0.0016018826514482497,
+ "step": 680
+ },
+ {
+ "epoch": 2.229802513464991,
+ "grad_norm": 0.02783738076686859,
+ "learning_rate": 5.860339463735136e-05,
+ "loss": 0.0031667064875364305,
+ "step": 700
+ },
+ {
+ "epoch": 2.2936365449830443,
+ "grad_norm": 0.008262189105153084,
+ "learning_rate": 5.662758460103895e-05,
+ "loss": 0.0029385803267359734,
+ "step": 720
+ },
+ {
+ "epoch": 2.3574705765010973,
+ "grad_norm": 0.03108038939535618,
+ "learning_rate": 5.464124115666095e-05,
+ "loss": 0.0051560476422309875,
+ "step": 740
+ },
+ {
+ "epoch": 2.4213046080191503,
+ "grad_norm": 0.04321969673037529,
+ "learning_rate": 5.264754525396212e-05,
+ "loss": 0.0011130817234516143,
+ "step": 760
+ },
+ {
+ "epoch": 2.4851386395372033,
+ "grad_norm": 0.022080617025494576,
+ "learning_rate": 5.06496896169857e-05,
+ "loss": 0.002006329409778118,
+ "step": 780
+ },
+ {
+ "epoch": 2.5489726710552563,
+ "grad_norm": 0.04725557193160057,
+ "learning_rate": 4.865087363121384e-05,
+ "loss": 0.0021070817485451697,
+ "step": 800
+ },
+ {
+ "epoch": 2.6128067025733093,
+ "grad_norm": 0.27594882249832153,
+ "learning_rate": 4.6654298220040673e-05,
+ "loss": 0.0016123956069350243,
+ "step": 820
+ },
+ {
+ "epoch": 2.6766407340913627,
+ "grad_norm": 0.0028557360637933016,
+ "learning_rate": 4.466316071878229e-05,
+ "loss": 0.00044896998442709444,
+ "step": 840
+ },
+ {
+ "epoch": 2.7404747656094157,
+ "grad_norm": 0.03328096121549606,
+ "learning_rate": 4.2680649754432913e-05,
+ "loss": 0.001640898734331131,
+ "step": 860
+ },
+ {
+ "epoch": 2.8043087971274687,
+ "grad_norm": 0.03642182797193527,
+ "learning_rate": 4.070994013936646e-05,
+ "loss": 0.0008663022890686989,
+ "step": 880
+ },
+ {
+ "epoch": 2.8681428286455217,
+ "grad_norm": 0.06211363896727562,
+ "learning_rate": 3.8754187787161155e-05,
+ "loss": 0.000627265078946948,
+ "step": 900
+ },
+ {
+ "epoch": 2.9319768601635747,
+ "grad_norm": 0.0025077553000301123,
+ "learning_rate": 3.681652465868875e-05,
+ "loss": 0.0009966723620891571,
+ "step": 920
+ },
+ {
+ "epoch": 2.9958108916816277,
+ "grad_norm": 0.08202111721038818,
+ "learning_rate": 3.490005374656184e-05,
+ "loss": 0.0005241389386355877,
+ "step": 940
+ },
+ {
+ "epoch": 3.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.8458,
+ "eval_generator_rouge1": 99.9512,
+ "eval_generator_rouge2": 99.9138,
+ "eval_generator_rougeL": 99.7589,
+ "eval_loss": 0.002848332514986396,
+ "eval_runtime": 1491.6076,
+ "eval_samples_per_second": 2.289,
+ "eval_selector_accuracy": 99.1822,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.143,
+ "step": 942
+ },
+ {
+ "epoch": 3.0574506283662477,
+ "grad_norm": 0.05988731235265732,
+ "learning_rate": 3.3007844105971336e-05,
+ "loss": 0.003412053734064102,
+ "step": 960
+ },
+ {
+ "epoch": 3.1212846598843007,
+ "grad_norm": 0.0027547082863748074,
+ "learning_rate": 3.114292593987149e-05,
+ "loss": 0.0007339773699641227,
+ "step": 980
+ },
+ {
+ "epoch": 3.1851186914023537,
+ "grad_norm": 0.004056593868881464,
+ "learning_rate": 2.9308285746383408e-05,
+ "loss": 0.0007870178669691086,
+ "step": 1000
+ },
+ {
+ "epoch": 3.248952722920407,
+ "grad_norm": 0.02156253345310688,
+ "learning_rate": 2.7506861536187867e-05,
+ "loss": 0.0004239068366587162,
+ "step": 1020
+ },
+ {
+ "epoch": 3.31278675443846,
+ "grad_norm": 0.01963026262819767,
+ "learning_rate": 2.5741538127566396e-05,
+ "loss": 0.0004422076977789402,
+ "step": 1040
+ },
+ {
+ "epoch": 3.376620785956513,
+ "grad_norm": 0.0010617657098919153,
+ "learning_rate": 2.4015142526625115e-05,
+ "loss": 0.001177133247256279,
+ "step": 1060
+ },
+ {
+ "epoch": 3.440454817474566,
+ "grad_norm": 0.04265115037560463,
+ "learning_rate": 2.233043940009967e-05,
+ "loss": 0.00048412443138659,
+ "step": 1080
+ },
+ {
+ "epoch": 3.504288848992619,
+ "grad_norm": 0.0026085481513291597,
+ "learning_rate": 2.0690126647990974e-05,
+ "loss": 0.0007659945171326399,
+ "step": 1100
+ },
+ {
+ "epoch": 3.568122880510672,
+ "grad_norm": 0.008790579624474049,
+ "learning_rate": 1.9096831083121828e-05,
+ "loss": 0.000472747627645731,
+ "step": 1120
+ },
+ {
+ "epoch": 3.631956912028725,
+ "grad_norm": 0.010375800542533398,
+ "learning_rate": 1.7553104224533324e-05,
+ "loss": 0.0004488838836550713,
+ "step": 1140
+ },
+ {
+ "epoch": 3.695790943546778,
+ "grad_norm": 0.005610933993011713,
+ "learning_rate": 1.606141821145735e-05,
+ "loss": 0.0006687684915959835,
+ "step": 1160
+ },
+ {
+ "epoch": 3.759624975064831,
+ "grad_norm": 0.01078067161142826,
+ "learning_rate": 1.4624161844408763e-05,
+ "loss": 0.0008440790697932243,
+ "step": 1180
+ },
+ {
+ "epoch": 3.8234590065828846,
+ "grad_norm": 0.009699724614620209,
+ "learning_rate": 1.324363675973698e-05,
+ "loss": 0.0008271764032542706,
+ "step": 1200
+ },
+ {
+ "epoch": 3.8872930381009376,
+ "grad_norm": 0.018169160932302475,
+ "learning_rate": 1.1922053743763052e-05,
+ "loss": 0.0003760921768844128,
+ "step": 1220
+ },
+ {
+ "epoch": 3.9511270696189906,
+ "grad_norm": 0.010761883109807968,
+ "learning_rate": 1.0661529192404973e-05,
+ "loss": 0.0003943405114114285,
+ "step": 1240
+ },
+ {
+ "epoch": 4.0,
+ "eval_generator_count": 1712.0,
+ "eval_generator_exact_match": 96.7874,
+ "eval_generator_rouge1": 99.9518,
+ "eval_generator_rouge2": 99.9142,
+ "eval_generator_rougeL": 99.7472,
+ "eval_loss": 0.0028539097402244806,
+ "eval_runtime": 1492.5205,
+ "eval_samples_per_second": 2.287,
+ "eval_selector_accuracy": 99.1822,
+ "eval_selector_count": 1712.0,
+ "eval_steps_per_second": 0.143,
+ "step": 1256
+ }
+ ],
+ "logging_steps": 20,
+ "max_steps": 1570,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 5,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 3.184962329955533e+17,
+ "train_batch_size": 1,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/qwen3/teacher_lora/checkpoint-1256/training_args.bin b/qwen3/teacher_lora/checkpoint-1256/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..1ef98d1a171b439f2f8968f31ccbc846a421fe48
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84fda57546f783ec338bd3a292c7dba3d917a8a52ac59ca7cebdd546da52b586
+size 7185
diff --git a/qwen3/teacher_lora/checkpoint-1256/zero_to_fp32.py b/qwen3/teacher_lora/checkpoint-1256/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1256/zero_to_fp32.py
@@ -0,0 +1,790 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+# python zero_to_fp32.py . output_dir/
+# or
+# python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY,
+ AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY,
+ AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT)
+
+
+@dataclass
+class zero_model_state:
+ buffers: dict()
+ param_shapes: dict()
+ shared_params: list
+ ds_version: int
+ frozen_param_shapes: dict()
+ frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+ return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+ '''
+ alist.sort(key=natural_keys) sorts in human order
+ http://nedbatchelder.com/blog/200712/human_sorting.html
+ (See Toothy's implementation in the comments)
+ '''
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+ if not os.path.isdir(checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+ # there should be only one file
+ if zero_stage <= 2:
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+ elif zero_stage == 3:
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+ if not os.path.exists(file):
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+ return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+ # XXX: need to test that this simple glob rule works for multi-node setup too
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+ if len(ckpt_files) == 0:
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+ return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def _has_autoep_zero3_partitioned_metadata(state_dict):
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY)
+ if autoep_layers is None:
+ autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY)
+ if not isinstance(autoep_layers, list):
+ return False
+ return any(
+ isinstance(entry, dict)
+ and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT
+ for entry in autoep_layers)
+
+
+def _raise_if_autoep_zero3_partitioned_state(state_dict):
+ if _has_autoep_zero3_partitioned_metadata(state_dict):
+ raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. "
+ "AutoEP expert parameters are partitioned over expert replica groups, so "
+ "global data-parallel consolidation would produce incomplete expert tensors. "
+ "Use ds_to_universal.py for expert-aware conversion.")
+
+
+def _raise_if_autoep_zero3_partitioned_checkpoint(model_files):
+ for file in model_files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+
+def parse_model_states(files):
+ zero_model_states = []
+ for file in files:
+ state_dict = torch.load(file, map_location=device, weights_only=False)
+ _raise_if_autoep_zero3_partitioned_state(state_dict)
+
+ if BUFFER_NAMES not in state_dict:
+ raise ValueError(f"{file} is not a model state checkpoint")
+ buffer_names = state_dict[BUFFER_NAMES]
+ if debug:
+ print("Found buffers:", buffer_names)
+
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+ param_shapes = state_dict[PARAM_SHAPES]
+
+ # collect parameters that are included in param_shapes
+ param_names = []
+ for s in param_shapes:
+ for name in s.keys():
+ param_names.append(name)
+
+ # update with frozen parameters
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+ if frozen_param_shapes is not None:
+ if debug:
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+ param_names += list(frozen_param_shapes.keys())
+
+ # handle shared params
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+ ds_version = state_dict.get(DS_VERSION, None)
+
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+ z_model_state = zero_model_state(buffers=buffers,
+ param_shapes=param_shapes,
+ shared_params=shared_params,
+ ds_version=ds_version,
+ frozen_param_shapes=frozen_param_shapes,
+ frozen_param_fragments=frozen_param_fragments)
+ zero_model_states.append(z_model_state)
+
+ return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+ total_files = len(files)
+ state_dicts = []
+ for f in tqdm(files, desc='Loading checkpoint shards'):
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+ # and also handle the case where it was already removed by another helper script
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+ state_dicts.append(state_dict)
+
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
+ # use the max of the partition_count to get the dp world_size.
+
+ if type(world_size) is list:
+ world_size = max(world_size)
+
+ if world_size != total_files:
+ raise ValueError(
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+ )
+
+ # the groups are named differently in each stage
+ if zero_stage <= 2:
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+ elif zero_stage == 3:
+ fp32_groups_key = FP32_FLAT_GROUPS
+ else:
+ raise ValueError(f"unknown zero stage {zero_stage}")
+
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+ return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+ """
+ Returns fp32 state_dict reconstructed from ds checkpoint
+
+ Args:
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+ """
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+ # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints
+ # before the expensive optimizer-shard load below.
+ model_files = get_model_state_files(ds_checkpoint_dir)
+ zero_model_states = parse_model_states(model_files)
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+ optim_files = get_optim_files(ds_checkpoint_dir)
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+ if zero_stage <= 2:
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+ elif zero_stage == 3:
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+ if debug:
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ state_dict[name] = frozen_param_fragments[name]
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+ attr = getattr(obj, fn, None)
+ return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+
+ # Reconstruction protocol:
+ #
+ # XXX: document this
+
+ if debug:
+ for i in range(world_size):
+ for j in range(len(fp32_flat_groups[0])):
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+ # XXX: memory usage doubles here (zero2)
+ num_param_groups = len(fp32_flat_groups[0])
+ merged_single_partition_of_fp32_groups = []
+ for i in range(num_param_groups):
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+ avail_numel = sum(
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+ if debug:
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+ # not asserting if there is a mismatch due to possible padding
+ print(f"Have {avail_numel} numels to process.")
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ total_numel = 0
+ total_params = 0
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+ offset = 0
+ avail_numel = full_single_fp32_vector.numel()
+ for name, shape in shapes.items():
+
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+ total_numel += unpartitioned_numel
+ total_params += 1
+
+ if debug:
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+ offset += unpartitioned_numel
+
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+ # live optimizer object, so we are checking that the numbers are within the right range
+ align_to = 2 * world_size
+
+ def zero2_align(x):
+ return align_to * math.ceil(x / align_to)
+
+ if debug:
+ print(f"original offset={offset}, avail_numel={avail_numel}")
+
+ offset = zero2_align(offset)
+ avail_numel = zero2_align(avail_numel)
+
+ if debug:
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+ remainder = unpartitioned_numel % world_size
+ padding_numel = (world_size - remainder) if remainder else 0
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+ return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+ return
+
+ if debug:
+ for i in range(world_size):
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+ wanted_params = len(frozen_param_shapes)
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+ print(f'Frozen params: Have {avail_numel} numels to process.')
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+ total_params = 0
+ total_numel = 0
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
+ total_params += 1
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+ """
+ A pseudo tensor that collects partitioned weights.
+ It is more memory efficient when there are multiple groups.
+ """
+
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+ self.flat_groups = flat_groups
+ self.flat_groups_offset = flat_groups_offset
+ self.offset = offset
+ self.partitioned_numel = partitioned_numel
+ self.shape = shape
+ self.dtype = self.flat_groups[0][0].dtype
+
+ def contiguous(self):
+ """
+ Merge partitioned weights from flat_groups into a single tensor.
+ """
+ end_idx = self.offset + self.partitioned_numel
+ world_size = len(self.flat_groups)
+ pad_flat_param_chunks = []
+
+ for rank_i in range(world_size):
+ # for each rank, we need to collect weights from related group/groups
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
+ start_group_id = None
+ end_group_id = None
+ for group_id in range(len(self.flat_groups_offset)):
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+ start_group_id = group_id
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+ end_group_id = group_id
+ break
+ # collect weights from related group/groups
+ for group_id in range(start_group_id, end_group_id + 1):
+ flat_tensor = flat_groups_at_rank_i[group_id]
+ start_offset = self.offset - self.flat_groups_offset[group_id]
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+ # collect weights from all ranks
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+ return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+ param_shapes = zero_model_states[0].param_shapes
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+ # param, re-consolidating each param, while dealing with padding if any
+
+ # merge list of dicts, preserving order
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+ if debug:
+ for i in range(world_size):
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+ wanted_params = len(param_shapes)
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+ # not asserting if there is a mismatch due to possible padding
+ avail_numel = fp32_flat_groups[0].numel() * world_size
+ print(f"Trainable params: Have {avail_numel} numels to process.")
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+ # params
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+ # out-of-core computing solution
+ offset = 0
+ total_numel = 0
+ total_params = 0
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+ unpartitioned_numel = shape.numel()
+ total_numel += unpartitioned_numel
+ total_params += 1
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+ if debug:
+ print(
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+ )
+
+ # memory efficient tensor
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+ state_dict[name] = tensor
+ offset += partitioned_numel
+
+ offset *= world_size
+
+ # Sanity check
+ if offset != avail_numel:
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+ exclude_frozen_parameters):
+ state_dict = OrderedDict()
+
+ # buffers
+ buffers = zero_model_states[0].buffers
+ state_dict.update(buffers)
+ if debug:
+ print(f"added {len(buffers)} buffers")
+
+ if not exclude_frozen_parameters:
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+ # recover shared parameters
+ for pair in zero_model_states[0].shared_params:
+ if pair[1] in state_dict:
+ state_dict[pair[0]] = state_dict[pair[1]]
+
+ return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+ """
+ Convert state_dict of GatheredTensor to torch tensor
+ """
+ torch_state_dict = {}
+ converted_tensors = {}
+ for name, tensor in state_dict.items():
+ tensor_id = id(tensor)
+ if tensor_id in converted_tensors: # shared tensors
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+ torch_state_dict[name] = shared_tensor
+ else:
+ converted_tensors[tensor_id] = name
+ if return_empty_tensor:
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+ else:
+ torch_state_dict[name] = tensor.contiguous()
+ return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag=None,
+ exclude_frozen_parameters=False,
+ lazy_mode=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+ via a model hub.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+ Returns:
+ - pytorch ``state_dict``
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ # do the training and checkpoint saving
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+ model = model.cpu() # move to cpu
+ model.load_state_dict(state_dict)
+ # submit to model hub or save the model to share with others
+
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
+ application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+ the checkpoint. Or you can load state_dict in lazy mode ::
+
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+ for name, lazy_tensor in state_dict.item():
+ tensor = lazy_tensor.contiguous() # to cpu
+ print(name, tensor)
+ # del tensor to release memory if it no longer in use
+ """
+ if tag is None:
+ latest_path = os.path.join(checkpoint_dir, 'latest')
+ if os.path.isfile(latest_path):
+ with open(latest_path, 'r') as fd:
+ tag = fd.read().strip()
+ else:
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+ if not os.path.isdir(ds_checkpoint_dir):
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+ if lazy_mode:
+ return state_dict
+ else:
+ return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+ output_dir,
+ max_shard_size="5GB",
+ safe_serialization=False,
+ tag=None,
+ exclude_frozen_parameters=False):
+ """
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+ Args:
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+ - ``exclude_frozen_parameters``: exclude frozen parameters
+ """
+
+ # Dependency pre-check
+ if safe_serialization:
+ try:
+ from safetensors.torch import save_file
+ except ImportError:
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
+ raise
+ if max_shard_size is not None:
+ try:
+ from huggingface_hub import split_torch_state_dict_into_shards
+ except ImportError:
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+ raise
+
+ # Convert zero checkpoint to state_dict
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+ tag,
+ exclude_frozen_parameters,
+ lazy_mode=True)
+
+ # Shard the model if it is too big.
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+ if max_shard_size is not None:
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+ # an memory-efficient approach for sharding
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+ filename_pattern=filename_pattern,
+ max_shard_size=max_shard_size)
+ else:
+ from collections import namedtuple
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+ state_dict_split = StateDictSplit(is_sharded=False,
+ filename_to_tensors={weights_name: list(state_dict.keys())})
+
+ # Save the model by shard
+ os.makedirs(output_dir, exist_ok=True)
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+ shard_state_dict = to_torch_tensor(shard_state_dict)
+ output_path = os.path.join(output_dir, shard_file)
+ if safe_serialization:
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+ else:
+ torch.save(shard_state_dict, output_path)
+ # release the memory of current shard
+ for tensor_name in list(shard_state_dict.keys()):
+ del state_dict[tensor_name]
+ del shard_state_dict[tensor_name]
+ del shard_state_dict
+ gc.collect()
+
+ # Save index if sharded
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+ save_index_file = os.path.join(output_dir, save_index_file)
+ with open(save_index_file, "w", encoding="utf-8") as f:
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+ f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+ """
+ 1. Put the provided model to cpu
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+ 3. Load it into the provided model
+
+ Args:
+ - ``model``: the model object to update
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+ Returns:
+ - ``model`: modified model
+
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+ conveniently placed for you in the checkpoint folder.
+
+ A typical usage might be ::
+
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+ # submit to model hub or save the model to share with others
+
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+ """
+ logger.info("Extracting fp32 weights")
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+ logger.info("Overwriting model with fp32 weights")
+ model = model.cpu()
+ model.load_state_dict(state_dict, strict=False)
+
+ return model
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("checkpoint_dir",
+ type=str,
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+ parser.add_argument("output_dir",
+ type=str,
+ help="directory to the pytorch fp32 state_dict output files"
+ "(e.g. path/checkpoint-12-output/)")
+ parser.add_argument(
+ "--max_shard_size",
+ type=str,
+ default="5GB",
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+ "without CPU OOM issues.")
+ parser.add_argument(
+ "--safe_serialization",
+ default=False,
+ action='store_true',
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+ parser.add_argument("-t",
+ "--tag",
+ type=str,
+ default=None,
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+ args = parser.parse_args()
+
+ debug = args.debug
+
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+ args.output_dir,
+ max_shard_size=args.max_shard_size,
+ safe_serialization=args.safe_serialization,
+ tag=args.tag,
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/qwen3/teacher_lora/checkpoint-1570/README.md b/qwen3/teacher_lora/checkpoint-1570/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..4fabf4f222ce5d86630bcfbcae8e90fb08d87580
--- /dev/null
+++ b/qwen3/teacher_lora/checkpoint-1570/README.md
@@ -0,0 +1,208 @@
+---
+base_model: Qwen/Qwen3-4B-Instruct-2507
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file