Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/f3/intermediate/F3_stage2.jsonl filter=lfs diff=lfs merge=lfs -text
+data/f3/intermediate/F3_stage1.jsonl filter=lfs diff=lfs merge=lfs -text
+data/f3/intermediate/F3_stage3.jsonl filter=lfs diff=lfs merge=lfs -text

MANIFEST.json

@@ -0,0 +1,555 @@
+{
+  "schema_version": "delivery-0p5b-pretrain-f3refine-v1",
+  "generated_utc": "2026-08-20T12:24:01+00:00",
+  "delivery_root": "/raid/data/weifeng/MileStone_2",
+  "headline": {
+    "pretrained_model": "models/pretrained/stage1_cpt",
+    "instruction_tuned_model": "models/sft/sft_f3_refined_instruct",
+    "reported_pass_at_1_pct": 9.1,
+    "reported_pass_at_5_pct": 13.0,
+    "evaluation": "Runnable alignment benchmark; 200 tasks; 5 samples/task; 1000 attempts"
+  },
+  "exclusions": [
+    "optimizer.pt, scheduler.pt, rng_state.pth and training_args.bin are not included",
+    "upstream Qwen base/instruct checkpoints are not included",
+    "the exact May training environment snapshot was not captured"
+  ],
+  "storage": {
+    "hardlinked_bytes": 13598238177,
+    "copied_or_authored_bytes": 108219,
+    "note": "Hard links save local disk; tar/zip/upload reads complete file bytes."
+  },
+  "file_count": 58,
+  "files": [
+    {
+      "path": "README.md",
+      "role": "README.md",
+      "bytes": 3354,
+      "sha256": "b9a66fd10a508e09e92335bf4b1f69124ec142805a04c70b4d9ba725d14c182f",
+      "mode": "authored",
+      "source": null,
+      "source_sha256_matches": null
+    },
+    {
+      "path": "code/f3_filtering/build_F3_v3.py",
+      "role": "code",
+      "bytes": 7198,
+      "sha256": "570dfb0cd8b5991714d7c52b2d8e65ebd22a874e9d85877ab8ad22162f778ed1",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/build_F3_v3.py",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/build_F3_v4.py",
+      "role": "code",
+      "bytes": 11482,
+      "sha256": "5107add2818270eb0309219d5b73aedc43153a1da97165750b22dae80e7f047e",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/build_F3_v4.py",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/clean_F3_stage1.py",
+      "role": "code",
+      "bytes": 6456,
+      "sha256": "c93313faa9e23019bc87f6805d0fe1c83ab8fe4812cae1209f16d08f57e9f1a3",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/clean_F3_stage1.py",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/clean_F3_stage2.py",
+      "role": "code",
+      "bytes": 6045,
+      "sha256": "2b3fa3632262f4dfe49b6a20899f0d7551523c040d904d70734f2f3b6312add2",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/clean_F3_stage2.py",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/clean_F3_stage3.py",
+      "role": "code",
+      "bytes": 2975,
+      "sha256": "13b402f6fbce073507a2b51099e6b27ae266e238597db51566025b584fa35d19",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/clean_F3_stage3.py",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/clean_F3_stage4.py",
+      "role": "code",
+      "bytes": 6985,
+      "sha256": "f6c72da037f62154fa02ef3a7187ac80debbd82bb2fdaa2f82166311983aded3",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/clean_F3_stage4.py",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/clean_F3_stage4_calibrate.py",
+      "role": "code",
+      "bytes": 5483,
+      "sha256": "b41cd23f44de05d1a79385a57482af2f166ddfd73788c1fe5d6e07568922e270",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/clean_F3_stage4_calibrate.py",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/reports/F3_stage1_report.json",
+      "role": "code",
+      "bytes": 650,
+      "sha256": "340895ba5021dac1d8e10a47f8d6cc9899c5c7a6cefa1815760e42a7660583b2",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage1_report.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/reports/F3_stage2_report.json",
+      "role": "code",
+      "bytes": 481,
+      "sha256": "f934470c04fbf3c4da8b94d7a0d582da454a8ffa0010f2ad49b851f5ad88e094",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage2_report.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/reports/F3_stage3_report.json",
+      "role": "code",
+      "bytes": 389,
+      "sha256": "197f96b66a39cb157d4b879ad7d04f19d381348625aba7fe1a8b5d5704da6012",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage3_report.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/reports/F3_stage4_calibration.json",
+      "role": "code",
+      "bytes": 349,
+      "sha256": "94427b4c24d0924f3c48837d49931273bd08a032bdc1c69167a8a4b6b7eef2ee",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage4_calibration.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/reports/F3_stage4_report.json",
+      "role": "code",
+      "bytes": 700,
+      "sha256": "db2a7b2e19ea296bf1391a7c81f6f8313d5dee189f6410018deedce41a8fdbdc",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage4_report.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/reports/F3_v3_build.log",
+      "role": "code",
+      "bytes": 2058,
+      "sha256": "8c358341e1934f6356b6268c84d37c7f20cef0733aacd0f362712847327adf8d",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_v3_build.log",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/reports/F3_v3_report.json",
+      "role": "code",
+      "bytes": 1059,
+      "sha256": "9e2f9939c93c2e5540a90c89ad23f4190e90122cf2c53768f8481cc0dda29a0e",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_v3_report.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/reports/F3_v4_build.log",
+      "role": "code",
+      "bytes": 1490,
+      "sha256": "e07f2a814b5b62b0dd49d62916525d40eb17d34ba5f7d5be6c846024aa5cd374",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_v4_build.log",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/reports/F3_v4_report.json",
+      "role": "code",
+      "bytes": 854,
+      "sha256": "9759b9f137f8ec4ec8c49256cf322e86f73360c81465b30400d52a83a7fb3baf",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_v4_report.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/f3_filtering/reports/PIPELINE_REPORT.json",
+      "role": "code",
+      "bytes": 2304,
+      "sha256": "a395330e3e2aa77be0ae68c0ccdaa6e12348cce2d8622a215e112bf0df8c6bf4",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/PIPELINE_REPORT.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/pretrain_data_prep/analyze_template_repetition.py",
+      "role": "code",
+      "bytes": 8791,
+      "sha256": "92e41b02d8d478dbb4b1b49741f4be28b2dc042020c6b510948efee977758769",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/analyze_template_repetition.py",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/pretrain_data_prep/build_filtered_pretrain.py",
+      "role": "code",
+      "bytes": 2586,
+      "sha256": "b1c4d603a9f85052e899b2b60af4c0248f588f6e3b58b887a8df29ee28bbfe8e",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/build_filtered_pretrain.py",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/pretrain_data_prep/dryrun_filter_repetition.py",
+      "role": "code",
+      "bytes": 6128,
+      "sha256": "6fe18b7c656f7ae767d67080614e5d5e76691ff3ff1c5a26d6382d61ef04cecf",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/dryrun_filter_repetition.py",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/pretrain_data_prep/reports/code_messages_pre_train_filtered_report.json",
+      "role": "code",
+      "bytes": 4670,
+      "sha256": "d2cdf70fdf2725fd6c7c2b83c2815d7c8f4e5411bb40f1c707e9e45e917e053b",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/code_messages_pre_train_filtered_report.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/pretrain_data_prep/reports/code_messages_pre_train_filtered_stages_report.json",
+      "role": "code",
+      "bytes": 679,
+      "sha256": "d7dd8ca28113c9eb3147ca72df822c186128f2e3198f8a582937735b16c9cc52",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/code_messages_pre_train_filtered_stages_report.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/pretrain_data_prep/reports/dryrun_filter_report.json",
+      "role": "code",
+      "bytes": 1261,
+      "sha256": "757e260e4da4940d88f001ade7492a5654b52408116a36b1fe1e675f2ed6e25b",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/dryrun_filter_report.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "code/pretrain_data_prep/split_filtered_by_token_length.py",
+      "role": "code",
+      "bytes": 3110,
+      "sha256": "ec2b2788a759e157ed4591e5252549467cfdff5910b31ce43d131d163860cb42",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Datasets/split_filtered_by_token_length.py",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "data/f3/input/scicode_sft_swift_F3_clean.jsonl",
+      "role": "data",
+      "bytes": 2636501943,
+      "sha256": "63ad49e1f7a035f420ae02f2bab8ed4e08f93d7a14a81c4d51cb424e46c505a2",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean.jsonl",
+      "source_sha256_matches": true,
+      "records": 473465
+    },
+    {
+      "path": "data/f3/intermediate/F3_stage1.jsonl",
+      "role": "data",
+      "bytes": 2276490009,
+      "sha256": "167d17cd03fbbdbca40e3ef289c81898120107ba13f975db2ce23e3d6516c1a3",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage1.jsonl",
+      "source_sha256_matches": true,
+      "records": 437671
+    },
+    {
+      "path": "data/f3/intermediate/F3_stage2.jsonl",
+      "role": "data",
+      "bytes": 1956575594,
+      "sha256": "97353d440b3b0d292ff443cf2e705140ff5fa25ebbc2a65163d240b1eff03432",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage2.jsonl",
+      "source_sha256_matches": true,
+      "records": 366672
+    },
+    {
+      "path": "data/f3/intermediate/F3_stage3.jsonl",
+      "role": "data",
+      "bytes": 1935884904,
+      "sha256": "c54871e7896bb83e49bed1f0fc07cef2021a8c7ddb5053c1f2c7d5eed8ddcb56",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage3.jsonl",
+      "source_sha256_matches": true,
+      "records": 362367
+    },
+    {
+      "path": "data/f3/intermediate/F3_stage4_calibration_scores.jsonl",
+      "role": "data",
+      "bytes": 132712,
+      "sha256": "4c7d29e8c412690776a52972e89e960613389363509898a6d188bf642c049f9e",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage4_calibration_scores.jsonl",
+      "source_sha256_matches": true,
+      "records": 500
+    },
+    {
+      "path": "data/f3/intermediate/F3_stage4_scores.jsonl",
+      "role": "data",
+      "bytes": 10035166,
+      "sha256": "da22d8bb0135c30bc3679f75d6195616bee3f32b50b41240780e09fa869f484a",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage4_scores.jsonl",
+      "source_sha256_matches": true,
+      "records": 362367
+    },
+    {
+      "path": "data/f3/output/scicode_sft_swift_F3_clean_v2.jsonl",
+      "role": "data",
+      "bytes": 1125890366,
+      "sha256": "fa3521aaac5ed5cb1575a2f92c0386cec598002175aa47dbbaaaf71b6aad5ac7",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v2.jsonl",
+      "source_sha256_matches": true,
+      "records": 207009
+    },
+    {
+      "path": "data/pretrain/code_messages_pre_train_filtered_stage1.jsonl",
+      "role": "data",
+      "bytes": 1657338573,
+      "sha256": "e582cefbfb377aee6f3cc0c23f8159d5a77b14de27cf9fb2c4722078b905459a",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Datasets/code_messages_pre_train_filtered_stage1.jsonl",
+      "source_sha256_matches": true,
+      "records": 902115
+    },
+    {
+      "path": "environment/KNOWN_LIMITATIONS.md",
+      "role": "environment",
+      "bytes": 2108,
+      "sha256": "55aed7c95007191cef4b4bce02aabca8377b2fddc17c630f77df3e49ec1a2433",
+      "mode": "authored",
+      "source": null,
+      "source_sha256_matches": null
+    },
+    {
+      "path": "evidence/eval/f3_refined_9p1_summary.json",
+      "role": "evidence",
+      "bytes": 979,
+      "sha256": "36b8a5aa5b20f8bb6b671cedbc0cf9f7a7246535e9e8494e068ba825f1b03003",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/DELIVERY_0p5B_CPT_SFT/eval/summaries/summary_hf__raid_data_weifeng_Pre_train_output_user_qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu_v0-20260508-164342_checkpoint-6435_20260508_191037.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "evidence/instruction_recipe_results.md",
+      "role": "evidence",
+      "bytes": 2895,
+      "sha256": "2da8bebeea85e5185a8560f21fc73faef47aedb5c5dde9783887f27e1331658c",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/DELIVERY_0p5B_CPT_SFT/reports/instruction_recipe_results.md",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "evidence/smoke_test_results.json",
+      "role": "evidence",
+      "bytes": 793,
+      "sha256": "aadb01d5d9609833a97bdfda6c0ef816b80093f380cf0944b13ba01ce94f2855",
+      "mode": "authored",
+      "source": null,
+      "source_sha256_matches": null
+    },
+    {
+      "path": "models/pretrained/stage1_cpt/args.json",
+      "role": "models",
+      "bytes": 39617,
+      "sha256": "89055b0d70f0787df200fb350546a72f0780d2afb1db8de8e1dcc8d33428b22c",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114/checkpoint-28175/args.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/pretrained/stage1_cpt/chat_template.jinja",
+      "role": "models",
+      "bytes": 2427,
+      "sha256": "44d5f08f3f72b837eaad09f13a54c1f9f4eb58d75240334548b7fd52a5437fa5",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114/checkpoint-28175/chat_template.jinja",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/pretrained/stage1_cpt/config.json",
+      "role": "models",
+      "bytes": 1284,
+      "sha256": "0d2cfcbf5e317ffaa0b8d8897b94a4b9c5e716ddd47ecd768f3ad0c2174ab2ce",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114/checkpoint-28175/config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/pretrained/stage1_cpt/generation_config.json",
+      "role": "models",
+      "bytes": 191,
+      "sha256": "8a210910e3bf58bdbdc6355ef06aa40ea311a7f579cd518cacd929955e9e43fa",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114/checkpoint-28175/generation_config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/pretrained/stage1_cpt/model.safetensors",
+      "role": "models",
+      "bytes": 988097824,
+      "sha256": "c119b284587312405231891869c326215379ec62890899a0b6af324204f8d290",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114/checkpoint-28175/model.safetensors",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/pretrained/stage1_cpt/tokenizer.json",
+      "role": "models",
+      "bytes": 11421892,
+      "sha256": "3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114/checkpoint-28175/tokenizer.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/pretrained/stage1_cpt/tokenizer_config.json",
+      "role": "models",
+      "bytes": 695,
+      "sha256": "35c37fe281bb6e5ed2a6699349c8dc7191a79c4c829abbffab33133908f11daa",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114/checkpoint-28175/tokenizer_config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/pretrained/stage1_cpt/trainer_state.json",
+      "role": "models",
+      "bytes": 121936,
+      "sha256": "5fb959d771564eae82f9187cd41e4da522d985d8585158898d3a7525c4b292ca",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114/checkpoint-28175/trainer_state.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_refined_instruct/args.json",
+      "role": "models",
+      "bytes": 39662,
+      "sha256": "4a04289e2de79f87bed89881e01536369dd8c34c1b01a4806fd0aa08ccfc8293",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/checkpoint-6435/args.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_refined_instruct/chat_template.jinja",
+      "role": "models",
+      "bytes": 2507,
+      "sha256": "cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/checkpoint-6435/chat_template.jinja",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_refined_instruct/config.json",
+      "role": "models",
+      "bytes": 1284,
+      "sha256": "2a0bbab03998171855260860fbab81152ec51e67c7d69b85f32c7f624341fb0e",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/checkpoint-6435/config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_refined_instruct/generation_config.json",
+      "role": "models",
+      "bytes": 242,
+      "sha256": "87c51d61bd70a204993da36384296ccfeb56d46ee2cc7247929f917d0d8118f7",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/checkpoint-6435/generation_config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_refined_instruct/model.safetensors",
+      "role": "models",
+      "bytes": 988097824,
+      "sha256": "05bca1c3ba20caad5a14ef0f450b6788aff7c9594bf90fe6deb5ecd2c5561847",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/checkpoint-6435/model.safetensors",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_refined_instruct/tokenizer.json",
+      "role": "models",
+      "bytes": 11421892,
+      "sha256": "3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/checkpoint-6435/tokenizer.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_refined_instruct/tokenizer_config.json",
+      "role": "models",
+      "bytes": 692,
+      "sha256": "4d8ad3fe9729e37338fae50bac94ff03cb915071eb886b937077114e874dfa06",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/checkpoint-6435/tokenizer_config.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "models/sft/sft_f3_refined_instruct/trainer_state.json",
+      "role": "models",
+      "bytes": 138941,
+      "sha256": "7259447195d9af3cad318bed168eb341c6ac2d171ccaddab7e195e173d033642",
+      "mode": "hardlink",
+      "source": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/checkpoint-6435/trainer_state.json",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "tools/build_manifest.py",
+      "role": "tools",
+      "bytes": 7386,
+      "sha256": "2be5dce6f73513de2b0ef0e5b41c9f3b5691dba0e8867193dae6d19f18518da6",
+      "mode": "authored",
+      "source": null,
+      "source_sha256_matches": null
+    },
+    {
+      "path": "tools/smoke_test.py",
+      "role": "tools",
+      "bytes": 3057,
+      "sha256": "8219ffc3a307516000a0c9612ecade9d0c274d85347f23757b8b30f240f92740",
+      "mode": "authored",
+      "source": null,
+      "source_sha256_matches": null
+    },
+    {
+      "path": "training_scripts/README.md",
+      "role": "training_scripts",
+      "bytes": 597,
+      "sha256": "259bf953bb9254443f6a164db05f02c1dfdc54c3ec6ee57502fae910b0eb2165",
+      "mode": "authored",
+      "source": null,
+      "source_sha256_matches": null
+    },
+    {
+      "path": "training_scripts/pretrain/stage1_short.sh",
+      "role": "training_scripts",
+      "bytes": 1401,
+      "sha256": "f25a669388095d705cc190e3f0351aaf1d310759bd1c7f3f452b3601364a4005",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Pre_train/swift_runs_curriculum_filtered_0p5b/stage1_short.sh",
+      "source_sha256_matches": true
+    },
+    {
+      "path": "training_scripts/sft/run_qwen25_coder_0p5b_instruct_F3_v2_1gpu.sh",
+      "role": "training_scripts",
+      "bytes": 1466,
+      "sha256": "8feeae24b837c7ebfa77c41057d4286eec4eae5d66c870c32a274f773ae76d39",
+      "mode": "copy",
+      "source": "/raid/data/weifeng/Pre_train/swift_runs_F3_v2_0p5b_instruct/run_qwen25_coder_0p5b_instruct_F3_v2_1gpu.sh",
+      "source_sha256_matches": true
+    }
+  ]
+}

README.md

@@ -0,0 +1,66 @@
+# 0.5B Pre-trained and F3-refined Model Delivery Package
+
+This is a compact, traceable delivery package containing the two previously reported
+0.5B-class model checkpoints, their training datasets, data-preparation code, training
+scripts, and evaluation evidence.
+
+## Delivered Models
+
+| Model | Path | Initialisation | Training data | Reported result |
+|---|---|---|---|---|
+| Stage-1 CPT | `models/pretrained/stage1_cpt/` | `Qwen/Qwen2.5-Coder-0.5B` | `data/pretrain/code_messages_pre_train_filtered_stage1.jsonl` (902,115 records) | Continued-pre-training deliverable |
+| F3-refined SFT | `models/sft/sft_f3_refined_instruct/` | `Qwen/Qwen2.5-Coder-0.5B-Instruct` | `data/f3/output/scicode_sft_swift_F3_clean_v2.jsonl` (207,009 records) | pass@1 9.1%; pass@5 13.0% |
+
+Both directories are standard Hugging Face checkpoints. Model weights, tokenisers,
+`args.json`, and `trainer_state.json` are included. Optimiser, scheduler, and random-number
+generator states (approximately 2 GB per model) are excluded because this package is
+intended for model delivery and inference rather than mid-run training resumption.
+
+## Data and F3 Filtering Chain
+
+`data/f3/` contains the input, principal intermediate artefacts, score output, and final
+training dataset used for the reported F3-refined model:
+
+```text
+473,465  00 input: scicode_sft_swift_F3_clean.jsonl
+437,671  01 rule-based AST, structure, duplicate, and placeholder filtering
+366,672  02 per-project caps and stratified sampling
+362,367  03 assistant-response deduplication within projects
+207,009  04 quality score >= 3: scicode_sft_swift_F3_clean_v2.jsonl
+```
+
+The corresponding scripts are in `code/f3_filtering/`. Stages 1–3 are local deterministic
+processing stages. Stage 4 uses an external `Qwen3-Coder-30B-A3B-Instruct` scoring model;
+the historical script retains its original absolute path, which must be replaced when the
+stage is run in another environment. The saved scores and final dataset are included.
+
+
+The continued-pre-training dataset and its preparation code are in `data/pretrain/` and
+`code/pretrain_data_prep/`, respectively.
+
+## Training Scripts
+
+- `training_scripts/pretrain/stage1_short.sh`
+- `training_scripts/sft/run_qwen25_coder_0p5b_instruct_F3_v2_1gpu.sh`
+
+These are the original historical launch scripts. They intentionally retain the GPU
+selection, cache locations, ports, and absolute output paths used at the time. Review and
+replace those paths before rerunning them on another system. The checkpoint `args.json`
+files are the primary record of the effective training parameters.
+
+## Evaluation Evidence and Limitations
+
+`evidence/eval/f3_refined_9p1_summary.json` records the original evaluation: 200 tasks,
+five samples per task, 1,000 attempts, pass@1 of 9.1%, and pass@5 of 13.0%. This result is
+from the historical Runnable alignment benchmark and must not be conflated with the later
+repository-level Core/Hold benchmark.
+
+The two delivered training runs do not have exact, run-specific `pip freeze` snapshots.
+Training data and effective parameters are traceable, but bit-for-bit retraining is not
+claimed. See `environment/KNOWN_LIMITATIONS.md` for details.
+
+## Integrity and Storage
+
+- `tools/build_manifest.py`: regenerates the manifest and checksums.
+- `tools/smoke_test.py`: loads each checkpoint on CPU and checks finite, non-degenerate logits.
+

SHA256SUMS

@@ -0,0 +1,59 @@
+c4dfa67956deb6eec305523de03d380a923cd70f9522d1e740846f57bd264fb8  MANIFEST.json
+b9a66fd10a508e09e92335bf4b1f69124ec142805a04c70b4d9ba725d14c182f  README.md
+570dfb0cd8b5991714d7c52b2d8e65ebd22a874e9d85877ab8ad22162f778ed1  code/f3_filtering/build_F3_v3.py
+5107add2818270eb0309219d5b73aedc43153a1da97165750b22dae80e7f047e  code/f3_filtering/build_F3_v4.py
+c93313faa9e23019bc87f6805d0fe1c83ab8fe4812cae1209f16d08f57e9f1a3  code/f3_filtering/clean_F3_stage1.py
+2b3fa3632262f4dfe49b6a20899f0d7551523c040d904d70734f2f3b6312add2  code/f3_filtering/clean_F3_stage2.py
+13b402f6fbce073507a2b51099e6b27ae266e238597db51566025b584fa35d19  code/f3_filtering/clean_F3_stage3.py
+f6c72da037f62154fa02ef3a7187ac80debbd82bb2fdaa2f82166311983aded3  code/f3_filtering/clean_F3_stage4.py
+b41cd23f44de05d1a79385a57482af2f166ddfd73788c1fe5d6e07568922e270  code/f3_filtering/clean_F3_stage4_calibrate.py
+340895ba5021dac1d8e10a47f8d6cc9899c5c7a6cefa1815760e42a7660583b2  code/f3_filtering/reports/F3_stage1_report.json
+f934470c04fbf3c4da8b94d7a0d582da454a8ffa0010f2ad49b851f5ad88e094  code/f3_filtering/reports/F3_stage2_report.json
+197f96b66a39cb157d4b879ad7d04f19d381348625aba7fe1a8b5d5704da6012  code/f3_filtering/reports/F3_stage3_report.json
+94427b4c24d0924f3c48837d49931273bd08a032bdc1c69167a8a4b6b7eef2ee  code/f3_filtering/reports/F3_stage4_calibration.json
+db2a7b2e19ea296bf1391a7c81f6f8313d5dee189f6410018deedce41a8fdbdc  code/f3_filtering/reports/F3_stage4_report.json
+8c358341e1934f6356b6268c84d37c7f20cef0733aacd0f362712847327adf8d  code/f3_filtering/reports/F3_v3_build.log
+9e2f9939c93c2e5540a90c89ad23f4190e90122cf2c53768f8481cc0dda29a0e  code/f3_filtering/reports/F3_v3_report.json
+e07f2a814b5b62b0dd49d62916525d40eb17d34ba5f7d5be6c846024aa5cd374  code/f3_filtering/reports/F3_v4_build.log
+9759b9f137f8ec4ec8c49256cf322e86f73360c81465b30400d52a83a7fb3baf  code/f3_filtering/reports/F3_v4_report.json
+a395330e3e2aa77be0ae68c0ccdaa6e12348cce2d8622a215e112bf0df8c6bf4  code/f3_filtering/reports/PIPELINE_REPORT.json
+92e41b02d8d478dbb4b1b49741f4be28b2dc042020c6b510948efee977758769  code/pretrain_data_prep/analyze_template_repetition.py
+b1c4d603a9f85052e899b2b60af4c0248f588f6e3b58b887a8df29ee28bbfe8e  code/pretrain_data_prep/build_filtered_pretrain.py
+6fe18b7c656f7ae767d67080614e5d5e76691ff3ff1c5a26d6382d61ef04cecf  code/pretrain_data_prep/dryrun_filter_repetition.py
+d2cdf70fdf2725fd6c7c2b83c2815d7c8f4e5411bb40f1c707e9e45e917e053b  code/pretrain_data_prep/reports/code_messages_pre_train_filtered_report.json
+d7dd8ca28113c9eb3147ca72df822c186128f2e3198f8a582937735b16c9cc52  code/pretrain_data_prep/reports/code_messages_pre_train_filtered_stages_report.json
+757e260e4da4940d88f001ade7492a5654b52408116a36b1fe1e675f2ed6e25b  code/pretrain_data_prep/reports/dryrun_filter_report.json
+ec2b2788a759e157ed4591e5252549467cfdff5910b31ce43d131d163860cb42  code/pretrain_data_prep/split_filtered_by_token_length.py
+63ad49e1f7a035f420ae02f2bab8ed4e08f93d7a14a81c4d51cb424e46c505a2  data/f3/input/scicode_sft_swift_F3_clean.jsonl
+167d17cd03fbbdbca40e3ef289c81898120107ba13f975db2ce23e3d6516c1a3  data/f3/intermediate/F3_stage1.jsonl
+97353d440b3b0d292ff443cf2e705140ff5fa25ebbc2a65163d240b1eff03432  data/f3/intermediate/F3_stage2.jsonl
+c54871e7896bb83e49bed1f0fc07cef2021a8c7ddb5053c1f2c7d5eed8ddcb56  data/f3/intermediate/F3_stage3.jsonl
+4c7d29e8c412690776a52972e89e960613389363509898a6d188bf642c049f9e  data/f3/intermediate/F3_stage4_calibration_scores.jsonl
+da22d8bb0135c30bc3679f75d6195616bee3f32b50b41240780e09fa869f484a  data/f3/intermediate/F3_stage4_scores.jsonl
+fa3521aaac5ed5cb1575a2f92c0386cec598002175aa47dbbaaaf71b6aad5ac7  data/f3/output/scicode_sft_swift_F3_clean_v2.jsonl
+e582cefbfb377aee6f3cc0c23f8159d5a77b14de27cf9fb2c4722078b905459a  data/pretrain/code_messages_pre_train_filtered_stage1.jsonl
+55aed7c95007191cef4b4bce02aabca8377b2fddc17c630f77df3e49ec1a2433  environment/KNOWN_LIMITATIONS.md
+36b8a5aa5b20f8bb6b671cedbc0cf9f7a7246535e9e8494e068ba825f1b03003  evidence/eval/f3_refined_9p1_summary.json
+2da8bebeea85e5185a8560f21fc73faef47aedb5c5dde9783887f27e1331658c  evidence/instruction_recipe_results.md
+aadb01d5d9609833a97bdfda6c0ef816b80093f380cf0944b13ba01ce94f2855  evidence/smoke_test_results.json
+89055b0d70f0787df200fb350546a72f0780d2afb1db8de8e1dcc8d33428b22c  models/pretrained/stage1_cpt/args.json
+44d5f08f3f72b837eaad09f13a54c1f9f4eb58d75240334548b7fd52a5437fa5  models/pretrained/stage1_cpt/chat_template.jinja
+0d2cfcbf5e317ffaa0b8d8897b94a4b9c5e716ddd47ecd768f3ad0c2174ab2ce  models/pretrained/stage1_cpt/config.json
+8a210910e3bf58bdbdc6355ef06aa40ea311a7f579cd518cacd929955e9e43fa  models/pretrained/stage1_cpt/generation_config.json
+c119b284587312405231891869c326215379ec62890899a0b6af324204f8d290  models/pretrained/stage1_cpt/model.safetensors
+3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8  models/pretrained/stage1_cpt/tokenizer.json
+35c37fe281bb6e5ed2a6699349c8dc7191a79c4c829abbffab33133908f11daa  models/pretrained/stage1_cpt/tokenizer_config.json
+5fb959d771564eae82f9187cd41e4da522d985d8585158898d3a7525c4b292ca  models/pretrained/stage1_cpt/trainer_state.json
+4a04289e2de79f87bed89881e01536369dd8c34c1b01a4806fd0aa08ccfc8293  models/sft/sft_f3_refined_instruct/args.json
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  models/sft/sft_f3_refined_instruct/chat_template.jinja
+2a0bbab03998171855260860fbab81152ec51e67c7d69b85f32c7f624341fb0e  models/sft/sft_f3_refined_instruct/config.json
+87c51d61bd70a204993da36384296ccfeb56d46ee2cc7247929f917d0d8118f7  models/sft/sft_f3_refined_instruct/generation_config.json
+05bca1c3ba20caad5a14ef0f450b6788aff7c9594bf90fe6deb5ecd2c5561847  models/sft/sft_f3_refined_instruct/model.safetensors
+3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8  models/sft/sft_f3_refined_instruct/tokenizer.json
+4d8ad3fe9729e37338fae50bac94ff03cb915071eb886b937077114e874dfa06  models/sft/sft_f3_refined_instruct/tokenizer_config.json
+7259447195d9af3cad318bed168eb341c6ac2d171ccaddab7e195e173d033642  models/sft/sft_f3_refined_instruct/trainer_state.json
+2be5dce6f73513de2b0ef0e5b41c9f3b5691dba0e8867193dae6d19f18518da6  tools/build_manifest.py
+8219ffc3a307516000a0c9612ecade9d0c274d85347f23757b8b30f240f92740  tools/smoke_test.py
+259bf953bb9254443f6a164db05f02c1dfdc54c3ec6ee57502fae910b0eb2165  training_scripts/README.md
+f25a669388095d705cc190e3f0351aaf1d310759bd1c7f3f452b3601364a4005  training_scripts/pretrain/stage1_short.sh
+8feeae24b837c7ebfa77c41057d4286eec4eae5d66c870c32a274f773ae76d39  training_scripts/sft/run_qwen25_coder_0p5b_instruct_F3_v2_1gpu.sh

code/f3_filtering/build_F3_v3.py

@@ -0,0 +1,184 @@
+"""Build F3_clean_v3 from F3_clean_v2 by removing residual repetition / dup problems.
+
+Rules (applied per sample, all must pass to keep):
+  R1: assistant length <= 8000 chars
+  R2: no line >=1 char (stripped) repeats >=10 times inside assistant
+  R3: no 5-gram (token-level) repeats >=20 times inside assistant
+  R4: cross-corpus exact-content dedup of assistant (keep first occurrence)
+
+Streaming, two-pass for R4 (first pass: build hash counter; second pass: keep
+first occurrence of each hash).
+
+Output: scicode_sft_swift_F3_clean_v3.jsonl + report.
+"""
+from __future__ import annotations
+
+import collections
+import hashlib
+import json
+import re
+import time
+from pathlib import Path
+
+SRC = Path("/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v2.jsonl")
+DST = Path("/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v3.jsonl")
+REPORT = Path("/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_v3_report.json")
+
+MAX_ASST_CHARS = 8000
+LINE_REPEAT_THRESH = 10        # any stripped line repeating >= 10 times
+NGRAM_N = 5
+NGRAM_THRESH = 20              # 5-gram repeating >= 20 times
+TOKEN_RE = re.compile(r"[A-Za-z_]\w*|[^\s\w]")
+
+
+def get_assistant(o: dict) -> str | None:
+    for m in o.get("messages", []) or []:
+        if m.get("role") == "assistant":
+            return m.get("content", "") or ""
+    return None
+
+
+def check_repetition(a: str) -> tuple[bool, bool, bool]:
+    """Return (line_rep_bad, ngram_rep_bad, too_long).
+    """
+    too_long = len(a) > MAX_ASST_CHARS
+    lines = [ln.strip() for ln in a.split("\n") if ln.strip()]
+    line_rep_bad = False
+    if lines:
+        c = collections.Counter(lines)
+        if c.most_common(1)[0][1] >= LINE_REPEAT_THRESH:
+            line_rep_bad = True
+    toks = TOKEN_RE.findall(a)
+    ngram_rep_bad = False
+    if len(toks) >= NGRAM_N:
+        c2 = collections.Counter(tuple(toks[i:i + NGRAM_N]) for i in range(len(toks) - NGRAM_N + 1))
+        if c2 and c2.most_common(1)[0][1] >= NGRAM_THRESH:
+            ngram_rep_bad = True
+    return line_rep_bad, ngram_rep_bad, too_long
+
+
+def main() -> None:
+    print("=== F3_v3 cleaning pipeline ===", flush=True)
+    t0 = time.time()
+
+    # ---------- Pass 1: scan + collect content hashes ----------
+    n = 0
+    bad_long = bad_line = bad_ngram = 0
+    line_only = ngram_only = both = 0
+    hash_counts: collections.Counter[str] = collections.Counter()
+    # we also remember whether each hash has any "good" representative to keep.
+    # cheaper: at pass2 we'll keep the first occurrence whose own per-row checks passed.
+    print(f"Pass 1: scanning {SRC}", flush=True)
+    with open(SRC, encoding="utf-8") as f:
+        for line in f:
+            try:
+                o = json.loads(line)
+            except Exception:
+                continue
+            n += 1
+            a = get_assistant(o)
+            if a is None:
+                continue
+            ll, ng, tl = check_repetition(a)
+            if tl:
+                bad_long += 1
+            if ll:
+                bad_line += 1
+            if ng:
+                bad_ngram += 1
+            if ll and ng:
+                both += 1
+            elif ll:
+                line_only += 1
+            elif ng:
+                ngram_only += 1
+            # hash assistant content for dedup pass
+            h = hashlib.md5(a.encode("utf-8", errors="ignore")).hexdigest()
+            hash_counts[h] += 1
+            if n % 50000 == 0:
+                print(f"  [{n:,}]  bad_long={bad_long}  bad_line={bad_line}  bad_ngram={bad_ngram}  uniq_hashes={len(hash_counts):,}", flush=True)
+    t_p1 = time.time() - t0
+    print(f"Pass 1 done in {t_p1:.0f}s.  total samples = {n:,}", flush=True)
+
+    dup_clusters = sum(1 for c in hash_counts.values() if c > 1)
+    dup_rows = sum(c for c in hash_counts.values() if c > 1)
+    print(f"  exact-content dup clusters: {dup_clusters:,}  covering {dup_rows:,} rows", flush=True)
+
+    # ---------- Pass 2: write filtered output ----------
+    seen_hashes: set[str] = set()
+    kept = 0
+    dropped_long = dropped_line = dropped_ngram = dropped_dup = 0
+    print(f"Pass 2: writing {DST}", flush=True)
+    DST.parent.mkdir(parents=True, exist_ok=True)
+    with open(SRC, encoding="utf-8") as fin, open(DST, "w", encoding="utf-8") as fout:
+        for line in fin:
+            try:
+                o = json.loads(line)
+            except Exception:
+                continue
+            a = get_assistant(o)
+            if a is None:
+                continue
+            # rule 1: length
+            if len(a) > MAX_ASST_CHARS:
+                dropped_long += 1
+                continue
+            # rule 2/3: repetition
+            ll, ng, _ = check_repetition(a)
+            if ll:
+                dropped_line += 1
+                continue
+            if ng:
+                dropped_ngram += 1
+                continue
+            # rule 4: cross-corpus dedup (keep first occurrence)
+            h = hashlib.md5(a.encode("utf-8", errors="ignore")).hexdigest()
+            if h in seen_hashes:
+                dropped_dup += 1
+                continue
+            seen_hashes.add(h)
+            fout.write(json.dumps(o, ensure_ascii=False) + "\n")
+            kept += 1
+            if (kept + dropped_long + dropped_line + dropped_ngram + dropped_dup) % 50000 == 0:
+                so_far = kept + dropped_long + dropped_line + dropped_ngram + dropped_dup
+                print(f"  [{so_far:,}]  kept={kept}  dropped: long={dropped_long}, line={dropped_line}, ngram={dropped_ngram}, dup={dropped_dup}", flush=True)
+
+    elapsed = time.time() - t0
+    report = {
+        "input": str(SRC),
+        "output": str(DST),
+        "input_size": n,
+        "final_size": kept,
+        "kept_rate": round(kept / max(1, n), 4),
+        "rules": [
+            f"R1: drop if assistant > {MAX_ASST_CHARS} chars",
+            f"R2: drop if any stripped line repeats >= {LINE_REPEAT_THRESH} times in assistant",
+            f"R3: drop if any {NGRAM_N}-gram repeats >= {NGRAM_THRESH} times in assistant",
+            f"R4: drop cross-corpus exact-content duplicates (keep first occurrence)",
+        ],
+        "scan_stats_before_filter": {
+            "samples_with_long_assistant": bad_long,
+            "samples_with_line_repeat": bad_line,
+            "samples_with_ngram_repeat": bad_ngram,
+            "samples_with_both_line_and_ngram": both,
+            "samples_with_only_line": line_only,
+            "samples_with_only_ngram": ngram_only,
+            "exact_dup_clusters": dup_clusters,
+            "exact_dup_rows_total": dup_rows,
+        },
+        "applied_drops": {
+            "R1_too_long": dropped_long,
+            "R2_line_repeat": dropped_line,
+            "R3_ngram_repeat": dropped_ngram,
+            "R4_dup": dropped_dup,
+            "total_dropped": dropped_long + dropped_line + dropped_ngram + dropped_dup,
+        },
+        "elapsed_sec": round(elapsed, 1),
+    }
+    REPORT.write_text(json.dumps(report, ensure_ascii=False, indent=2))
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+    print(f"\nWrote {DST}  ({DST.stat().st_size / 1e9:.2f} GB, {kept:,} samples)")
+
+
+if __name__ == "__main__":
+    main()

code/f3_filtering/build_F3_v4.py

@@ -0,0 +1,306 @@
+"""Build F3_clean_v4 from F3_clean_v3 by applying additional quality rules.
+
+Source data only has `messages` (no file/path), so S2 uses CONTENT heuristics:
+  - dunder __init__ whose body is ONLY attribute assignments (template ctor)
+  - main/setup/parse_args with trivial body
+  - one-line property getters/setters
+
+Rules applied:
+  S1: MinHash near-duplicate (bottom-128 shingles, Jaccard threshold 0.85)
+  S2: template-style functions (content-based substitute)
+  S3: trivial function body  (< 3 non-trivial statements after stripping docstring)
+  S4: comment/docstring ratio extreme (> 0.7 or == 0 with >=10 code lines)
+  S6: import-only or docstring-only assistant
+  S8: placeholder residue (TODO/FIXME/pdb/breakpoint/NotImplementedError/...)
+
+Output: scicode_sft_swift_F3_clean_v4.jsonl + report.
+"""
+from __future__ import annotations
+
+import collections
+import json
+import re
+import time
+from pathlib import Path
+
+import xxhash
+
+SRC = Path("/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v3.jsonl")
+DST = Path("/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v4.jsonl")
+REPORT = Path("/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_v4_report.json")
+
+# ---------- knobs ----------
+SHINGLE_K = 8           # 8-token shingles
+SIG_K = 128             # bottom-128 minhash sketch
+LSH_BANDS = 16          # 16 bands × 8 rows
+JACCARD_THRESH = 0.85
+TRIVIAL_STMT_THRESH = 3 # < 3 non-trivial lines = trivial
+COMMENT_RATIO_HIGH = 0.70
+MIN_CODE_LINES_FOR_NO_COMMENT_CHECK = 10
+PLACEHOLDERS = (
+    "TODO", "FIXME", "XXX", "HACK",
+    "pdb.set_trace", "breakpoint(", "import pdb",
+    "raise NotImplementedError",
+    "pass  # placeholder", "pass  # implement",
+)
+
+CODE_FENCE_RE = re.compile(r"```(?:python|cpp|c|java|rust|go)?\s*\n([\s\S]*?)\n```", re.IGNORECASE)
+TOKEN_RE = re.compile(r"[A-Za-z_]\w*|[^\s\w]")
+COMMENT_RE = re.compile(r"^\s*(#|//|/\*|\*)")
+DOCSTRING_TRIPLE = re.compile(r'(?s)"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'')
+ATTR_ASSIGN_RE = re.compile(r"^\s*self\.\w+\s*=\s*\w+\s*$")
+IMPORT_RE = re.compile(r"^\s*(?:from\s+\S+\s+import\s+|import\s+\S+)")
+
+
+def get_messages(o: dict) -> tuple[str, str]:
+    u = a = ""
+    for m in o.get("messages", []) or []:
+        if m.get("role") == "user":
+            u = m.get("content", "") or ""
+        elif m.get("role") == "assistant":
+            a = m.get("content", "") or ""
+    return u, a
+
+
+def extract_code(asst: str) -> str:
+    m = CODE_FENCE_RE.search(asst)
+    return m.group(1) if m else asst
+
+
+def is_placeholder(code: str) -> bool:
+    for kw in PLACEHOLDERS:
+        if kw in code:
+            return True
+    return False
+
+
+def is_import_or_docstring_only(code: str) -> bool:
+    stripped = DOCSTRING_TRIPLE.sub("", code)
+    lines = [ln.strip() for ln in stripped.split("\n") if ln.strip()]
+    if not lines:
+        return True
+    non_import_non_comment = [ln for ln in lines
+                              if not IMPORT_RE.match(ln) and not COMMENT_RE.match(ln)]
+    return len(non_import_non_comment) < 3
+
+
+def is_trivial_body(code: str) -> bool:
+    # strip docstrings + comments, count statements
+    stripped = DOCSTRING_TRIPLE.sub("", code)
+    lines = [ln.strip() for ln in stripped.split("\n") if ln.strip()]
+    non_trivial = [ln for ln in lines
+                   if not COMMENT_RE.match(ln)
+                   and not IMPORT_RE.match(ln)
+                   and not ln.startswith("def ")
+                   and not ln.startswith("class ")
+                   and not ln.startswith("@")
+                   and ln not in ("pass", "...", "return", "return None")]
+    return len(non_trivial) < TRIVIAL_STMT_THRESH
+
+
+def comment_ratio_bad(code: str) -> bool:
+    lines = [ln for ln in code.split("\n") if ln.strip()]
+    if not lines:
+        return True
+    # count comment lines + docstring approximate
+    in_doc = False
+    doc_lines = 0
+    comment_lines = 0
+    code_lines = 0
+    for ln in lines:
+        s = ln.strip()
+        if in_doc:
+            doc_lines += 1
+            if s.endswith('"""') or s.endswith("'''"):
+                in_doc = False
+            continue
+        if s.startswith('"""') or s.startswith("'''"):
+            doc_lines += 1
+            if not (s.endswith('"""') and len(s) > 3) and not (s.endswith("'''") and len(s) > 3):
+                in_doc = True
+            continue
+        if COMMENT_RE.match(ln):
+            comment_lines += 1
+        else:
+            code_lines += 1
+    total = doc_lines + comment_lines + code_lines
+    if total == 0:
+        return True
+    doc_ratio = (doc_lines + comment_lines) / total
+    if doc_ratio > COMMENT_RATIO_HIGH:
+        return True
+    # also too few comments + lots of code: only penalize if NO comments AND >=10 code lines
+    if doc_lines + comment_lines == 0 and code_lines >= MIN_CODE_LINES_FOR_NO_COMMENT_CHECK:
+        return False  # not punishing this; we keep it (only the >0.7 side is filtered)
+    return False
+
+
+def is_template_func(asst: str, code: str) -> bool:
+    # detect ctor with only self.x=y body
+    if not re.search(r"def\s+__init__\s*\(", code):
+        pass
+    else:
+        body_lines = [ln.strip() for ln in code.split("\n") if ln.strip()]
+        body_lines = [ln for ln in body_lines
+                      if not ln.startswith("def ") and not ln.startswith("@")
+                      and not COMMENT_RE.match(ln) and not ln.startswith('"""') and not ln.startswith("'''")
+                      and ln not in ("pass", "...")]
+        if body_lines and all(ATTR_ASSIGN_RE.match(ln) or ln == "" for ln in body_lines):
+            return True
+    # main/setup with trivial body
+    if re.search(r"def\s+(?:main|setup|parse_args|get_args)\s*\(", code) and is_trivial_body(code):
+        return True
+    return False
+
+
+def code_shingles(code: str, k: int = SHINGLE_K) -> list[int]:
+    """Return shingled token n-gram hashes (deduplicated)."""
+    toks = TOKEN_RE.findall(code)
+    if len(toks) < k:
+        return []
+    sh = set()
+    for i in range(len(toks) - k + 1):
+        h = xxhash.xxh64(" ".join(toks[i:i + k])).intdigest()
+        sh.add(h)
+    return sorted(sh)[:5000]  # cap to bound mem for huge samples
+
+
+def minhash_signature(shingles: list[int], sig_k: int = SIG_K) -> tuple[int, ...]:
+    """Bottom-K minhash."""
+    if not shingles:
+        return tuple([0] * sig_k)
+    if len(shingles) >= sig_k:
+        return tuple(shingles[:sig_k])
+    return tuple(shingles + [0] * (sig_k - len(shingles)))
+
+
+def lsh_band_keys(sig: tuple[int, ...], bands: int = LSH_BANDS) -> list[bytes]:
+    rows_per_band = len(sig) // bands
+    keys = []
+    for b in range(bands):
+        chunk = sig[b * rows_per_band:(b + 1) * rows_per_band]
+        keys.append(xxhash.xxh64(bytes(str(chunk), "utf-8")).digest())
+    return keys
+
+
+def jaccard(a: tuple, b: tuple) -> float:
+    sa, sb = set(a) - {0}, set(b) - {0}
+    if not sa or not sb:
+        return 0.0
+    return len(sa & sb) / len(sa | sb)
+
+
+def main() -> None:
+    print("=== F3_v4 cleaning pipeline ===", flush=True)
+    t0 = time.time()
+
+    # ---------- Pass 1: scan, compute per-row features and signatures ----------
+    rows: list[dict] = []
+    n = 0
+    print("Pass 1: scanning + computing features + minhash", flush=True)
+    with open(SRC, encoding="utf-8") as f:
+        for line in f:
+            try:
+                o = json.loads(line)
+            except Exception:
+                continue
+            n += 1
+            _, a = get_messages(o)
+            code = extract_code(a)
+            features = {
+                "s8_placeholder": is_placeholder(code),
+                "s6_import_doc_only": is_import_or_docstring_only(code),
+                "s3_trivial": is_trivial_body(code),
+                "s4_comment_ratio": comment_ratio_bad(code),
+                "s2_template": is_template_func(a, code),
+            }
+            sig = minhash_signature(code_shingles(code))
+            rows.append({"raw": line, "sig": sig, "features": features})
+            if n % 20000 == 0:
+                print(f"  [{n:,}]  features computed", flush=True)
+    t_p1 = time.time() - t0
+    print(f"Pass 1 done in {t_p1:.0f}s.  total = {n:,}", flush=True)
+
+    # ---------- Pass 2: LSH near-dup detection ----------
+    print("Pass 2: LSH bucketing for near-dup", flush=True)
+    buckets: dict[bytes, list[int]] = collections.defaultdict(list)
+    for i, r in enumerate(rows):
+        for k in lsh_band_keys(r["sig"]):
+            buckets[k].append(i)
+    candidate_pairs: set[tuple[int, int]] = set()
+    for ids in buckets.values():
+        if len(ids) < 2:
+            continue
+        if len(ids) > 50:  # huge buckets are likely noise, cap
+            ids = ids[:50]
+        for i in range(len(ids)):
+            for j in range(i + 1, len(ids)):
+                candidate_pairs.add((ids[i], ids[j]))
+    print(f"  candidate pairs: {len(candidate_pairs):,}", flush=True)
+
+    near_dup: set[int] = set()
+    for a, b in candidate_pairs:
+        if a in near_dup or b in near_dup:
+            continue
+        if jaccard(rows[a]["sig"], rows[b]["sig"]) >= JACCARD_THRESH:
+            # keep the longer one (= more informative)
+            la = len(rows[a]["raw"]); lb = len(rows[b]["raw"])
+            near_dup.add(b if la >= lb else a)
+    print(f"  near-dup marked: {len(near_dup):,}", flush=True)
+
+    # ---------- Pass 3: write filtered output ----------
+    print(f"Pass 3: writing {DST}", flush=True)
+    DST.parent.mkdir(parents=True, exist_ok=True)
+    drops = collections.Counter()
+    kept = 0
+    with open(DST, "w", encoding="utf-8") as fout:
+        for i, r in enumerate(rows):
+            f = r["features"]
+            # priority order — drop on first match for clean accounting
+            if i in near_dup:
+                drops["S1_near_dup"] += 1
+                continue
+            if f["s2_template"]:
+                drops["S2_template"] += 1
+                continue
+            if f["s3_trivial"]:
+                drops["S3_trivial"] += 1
+                continue
+            if f["s4_comment_ratio"]:
+                drops["S4_comment_ratio"] += 1
+                continue
+            if f["s6_import_doc_only"]:
+                drops["S6_import_doc_only"] += 1
+                continue
+            if f["s8_placeholder"]:
+                drops["S8_placeholder"] += 1
+                continue
+            fout.write(r["raw"])
+            kept += 1
+
+    elapsed = time.time() - t0
+    report = {
+        "input": str(SRC),
+        "output": str(DST),
+        "input_size": n,
+        "final_size": kept,
+        "kept_rate": round(kept / max(1, n), 4),
+        "rules": [
+            f"S1: MinHash near-dup, Jaccard >= {JACCARD_THRESH}, bottom-{SIG_K} shingle-{SHINGLE_K}",
+            "S2: template ctor / setup / main with trivial body (content-based)",
+            f"S3: trivial body (< {TRIVIAL_STMT_THRESH} non-trivial statements)",
+            f"S4: comment+docstring ratio > {COMMENT_RATIO_HIGH}",
+            "S6: import-only / docstring-only assistant",
+            f"S8: placeholder residue ({', '.join(PLACEHOLDERS[:5])}, ...)",
+        ],
+        "applied_drops": dict(drops),
+        "total_dropped": sum(drops.values()),
+        "elapsed_sec": round(elapsed, 1),
+    }
+    REPORT.write_text(json.dumps(report, ensure_ascii=False, indent=2))
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+    print(f"\nWrote {DST}  ({DST.stat().st_size / 1e9:.2f} GB, {kept:,} samples)")
+
+
+if __name__ == "__main__":
+    main()

code/f3_filtering/clean_F3_stage1.py

@@ -0,0 +1,158 @@
+"""F3 → F3_v2 cleaning pipeline — Stage 1: structural filters.
+
+Drops samples whose assistant code block matches any of:
+  1a  AST cannot parse
+  1b  any line >=5 stripped chars repeats >=5 times
+  1c  function body is < 3 lines or > 500 lines
+  1d  contains hardcoded local paths (/home/, /Users/, C:\\, /tmp/<name>, /mnt/<name>)
+  1e  pure placeholder body (only pass / ... / raise NotImplementedError / return None)
+  1f  debug residue: breakpoint(), pdb.set_trace(), or >=3 TODO/FIXME markers
+"""
+from __future__ import annotations
+
+import ast
+import json
+import re
+from collections import Counter
+from pathlib import Path
+
+SRC = Path("/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean.jsonl")
+DST = Path("/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage1.jsonl")
+REPORT = Path("/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage1_report.json")
+
+CODE_FENCE_RE = re.compile(r"```(?:python|py)?\s*\n([\s\S]*?)```", re.IGNORECASE)
+# Generic fence (any language) for language-aware filtering
+GENERIC_FENCE_RE = re.compile(r"```(\S*)\s*\n([\s\S]*?)```", re.IGNORECASE)
+HARDCODED_PATH_RE = re.compile(
+    r"(?:[\"'])(?:/home/[^/\s\"']+/|/Users/[^/\s\"']+/|/tmp/[A-Za-z0-9_-]+|/mnt/[^/\s\"']+/|/data/[A-Za-z0-9_-]+|[A-Z]:\\\\?[A-Za-z0-9_-]+\\\\)"
+)
+DEBUG_RE = re.compile(r"\b(?:breakpoint\(\)|pdb\.set_trace\(\)|ipdb\.set_trace\(\))")
+TODO_RE = re.compile(r"#\s*(?:TODO|FIXME|XXX|HACK)\b", re.IGNORECASE)
+PLACEHOLDER_RE = re.compile(
+    r"^\s*(?:pass|\.\.\.|raise\s+NotImplementedError(?:\(.*?\))?|return\s+None)\s*$",
+    re.MULTILINE,
+)
+
+
+def get_assistant_code_with_lang(obj: dict) -> tuple[str, str]:
+    """Return (code, fence_language). Language defaults to 'python' when fence omitted/empty."""
+    for m in obj.get("messages", []):
+        if m.get("role") == "assistant":
+            text = m.get("content", "")
+            mfence = GENERIC_FENCE_RE.search(text)
+            if mfence:
+                lang = (mfence.group(1) or "").strip().lower()
+                code = mfence.group(2)
+                if not lang or lang == "py":
+                    lang = "python"
+                return code, lang
+            return text, "python"
+    return "", "python"
+
+
+def is_pure_placeholder(code: str) -> bool:
+    """True if the function body has only placeholder/no-op statements."""
+    try:
+        tree = ast.parse(code)
+    except SyntaxError:
+        return False
+    has_real = False
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            body = node.body
+            # ignore docstring
+            if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant) and isinstance(body[0].value.value, str):
+                body = body[1:]
+            if not body:
+                continue
+            for stmt in body:
+                if isinstance(stmt, ast.Pass):
+                    continue
+                if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant) and stmt.value.value is Ellipsis:
+                    continue
+                if isinstance(stmt, ast.Raise) and stmt.exc and (
+                    (isinstance(stmt.exc, ast.Name) and stmt.exc.id == "NotImplementedError")
+                    or (isinstance(stmt.exc, ast.Call) and isinstance(stmt.exc.func, ast.Name) and stmt.exc.func.id == "NotImplementedError")
+                ):
+                    continue
+                if isinstance(stmt, ast.Return) and (stmt.value is None or (isinstance(stmt.value, ast.Constant) and stmt.value.value is None)):
+                    continue
+                # any other stmt counts as real code
+                has_real = True
+                break
+            if has_real:
+                break
+    return not has_real  # all defs are pure placeholder
+
+
+def filter_sample(code: str, lang: str) -> tuple[bool, str]:
+    """Return (keep, reason_if_drop). `lang` decides whether to AST-parse Python."""
+    if not code or not code.strip():
+        return False, "1c_empty"
+    # 1c: length sanity
+    n_lines = len(code.splitlines())
+    if n_lines < 3:
+        return False, "1c_too_short"
+    if n_lines > 500:
+        return False, "1c_too_long"
+    # 1a: AST parse — only enforced for python-fenced samples
+    if lang == "python":
+        try:
+            ast.parse(code)
+        except SyntaxError:
+            return False, "1a_ast_fail_python"
+    # 1b: line repeat >=5 (applies to all languages — repetition is universally bad signal)
+    qual_lines = [ln for ln in code.splitlines() if len(ln.strip()) >= 5]
+    if qual_lines:
+        max_rep = Counter(qual_lines).most_common(1)[0][1]
+        if max_rep >= 5:
+            return False, "1b_line_repeat"
+    # 1d: hardcoded local path
+    if HARDCODED_PATH_RE.search(code):
+        return False, "1d_hardcoded_path"
+    # 1f: debug residue (Python-specific markers; harmless to apply broadly)
+    if DEBUG_RE.search(code):
+        return False, "1f_debug_pdb"
+    if len(TODO_RE.findall(code)) >= 3:
+        return False, "1f_many_todo"
+    # 1e: pure placeholder (Python-only check; non-Python skipped)
+    if lang == "python" and is_pure_placeholder(code):
+        return False, "1e_placeholder"
+    return True, ""
+
+
+def main() -> None:
+    DST.parent.mkdir(parents=True, exist_ok=True)
+    drop_reasons: Counter[str] = Counter()
+    lang_counts_kept: Counter[str] = Counter()
+    n_total = n_kept = 0
+    with open(SRC) as fin, open(DST, "w", encoding="utf-8") as fout:
+        for line in fin:
+            n_total += 1
+            obj = json.loads(line)
+            code, lang = get_assistant_code_with_lang(obj)
+            keep, reason = filter_sample(code, lang)
+            if not keep:
+                drop_reasons[reason] += 1
+                continue
+            n_kept += 1
+            lang_counts_kept[lang] += 1
+            fout.write(line)
+
+    n_dropped = n_total - n_kept
+    report = {
+        "src": str(SRC),
+        "dst": str(DST),
+        "n_total": n_total,
+        "n_kept": n_kept,
+        "n_dropped": n_dropped,
+        "drop_rate": round(n_dropped / max(n_total, 1), 4),
+        "drop_reasons": dict(drop_reasons.most_common()),
+        "kept_languages": dict(lang_counts_kept.most_common(20)),
+    }
+    REPORT.write_text(json.dumps(report, indent=2, ensure_ascii=False))
+    print(json.dumps(report, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

code/f3_filtering/clean_F3_stage2.py

@@ -0,0 +1,173 @@
+"""F3 → F3_v2 — Stage 2: per-project cap=300 with D1xD2 stratified sampling.
+
+D1 (code length): S<15, M1=15-39, M2=40-99, L>=100
+D2 (line repeat): norep<5, rep>=5
+
+Per project (>300 samples): allocate 75 per D1 bucket; rep cap=5 per D1; reallocate shortfall to other D1 norep buckets. Random sample within bucket (seed=0).
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import random
+import re
+from collections import Counter, defaultdict
+from pathlib import Path
+
+SRC = Path("/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage1.jsonl")
+DST = Path("/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage2.jsonl")
+REPORT = Path("/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage2_report.json")
+
+CAP = 300
+D1_TARGET = {"S": 75, "M1": 75, "M2": 75, "L": 75}
+REP_LIMIT_PER_D1 = 5
+SEED = 0
+
+CODE_FENCE_RE = re.compile(r"```(?:python|py)?\s*\n([\s\S]*?)```", re.IGNORECASE)
+
+
+def get_assistant_code(obj: dict) -> str:
+    for m in obj.get("messages", []):
+        if m.get("role") == "assistant":
+            text = m.get("content", "")
+            mfence = CODE_FENCE_RE.search(text)
+            return mfence.group(1) if mfence else text
+    return ""
+
+
+def get_user(obj: dict) -> str:
+    for m in obj.get("messages", []):
+        if m.get("role") == "user":
+            return m.get("content", "")
+    return ""
+
+
+def d1_bucket(n_lines: int) -> str:
+    if n_lines < 15: return "S"
+    if n_lines < 40: return "M1"
+    if n_lines < 100: return "M2"
+    return "L"
+
+
+def main() -> None:
+    DST.parent.mkdir(parents=True, exist_ok=True)
+    rng = random.Random(SEED)
+
+    # PASS 1: bucket all samples by project, store sample with index for stable refs
+    proj_buckets: dict[str, dict[tuple[str, str], list[int]]] = defaultdict(lambda: defaultdict(list))
+    rows: list[str] = []
+    with open(SRC) as fin:
+        for idx, line in enumerate(fin):
+            obj = json.loads(line)
+            u = get_user(obj)
+            code = get_assistant_code(obj)
+            n_lines = len(code.splitlines())
+            qual = [ln for ln in code.splitlines() if len(ln.strip()) >= 5]
+            max_rep = max(Counter(qual).values()) if qual else 0
+            d1 = d1_bucket(n_lines)
+            d2 = "rep" if max_rep >= 5 else "norep"  # should be 0 after stage1
+            proj_h = hashlib.md5(u[:500].encode()).hexdigest()
+            proj_buckets[proj_h][(d1, d2)].append(idx)
+            rows.append(line)
+    print(f"loaded {len(rows)} samples from {len(proj_buckets)} projects")
+
+    # PASS 2: per project sampling
+    keep_indices: set[int] = set()
+    n_proj_capped = 0
+    n_proj_untouched = 0
+    bucket_kept_counter: Counter[tuple[str, str]] = Counter()
+
+    for proj_h, buckets in proj_buckets.items():
+        n_in_proj = sum(len(v) for v in buckets.values())
+        if n_in_proj <= CAP:
+            # untouched
+            for v in buckets.values():
+                for idx in v:
+                    keep_indices.add(idx)
+                    # for stats only
+            n_proj_untouched += 1
+            continue
+
+        n_proj_capped += 1
+        # allocate per D1 (target 75 each, 4 buckets = 300)
+        chosen: list[int] = []
+        d1_remaining: dict[str, int] = dict(D1_TARGET)
+        unfilled: list[str] = []  # D1 that couldn't fill from norep
+
+        for d1 in ["S", "M1", "M2", "L"]:
+            target = d1_remaining[d1]
+            norep_pool = list(buckets.get((d1, "norep"), []))
+            rep_pool = list(buckets.get((d1, "rep"), []))
+            rng.shuffle(norep_pool)
+            rng.shuffle(rep_pool)
+
+            # take from norep first
+            take_norep = min(len(norep_pool), target)
+            chosen.extend(norep_pool[:take_norep])
+            target -= take_norep
+
+            # fill remaining from rep, capped by REP_LIMIT_PER_D1
+            rep_take = min(target, REP_LIMIT_PER_D1, len(rep_pool))
+            chosen.extend(rep_pool[:rep_take])
+            target -= rep_take
+
+            # update remaining: norep_pool[take_norep:] is leftover norep
+            norep_left = norep_pool[take_norep:]
+            buckets[(d1, "norep_leftover")] = norep_left
+            buckets[(d1, "rep_leftover")] = rep_pool[rep_take:]
+
+            if target > 0:
+                unfilled.append(d1)
+                d1_remaining[d1] = target  # how much D1 still needs
+            else:
+                d1_remaining[d1] = 0
+
+        # reallocate shortfall: pull norep_leftover from other D1 buckets
+        total_short = sum(d1_remaining.values())
+        if total_short > 0:
+            extra_pool = []
+            for d1 in ["S", "M1", "M2", "L"]:
+                extra_pool.extend(buckets.get((d1, "norep_leftover"), []))
+            rng.shuffle(extra_pool)
+            take = min(total_short, len(extra_pool))
+            chosen.extend(extra_pool[:take])
+
+        # final cap (in case of overshoot due to algorithm bug)
+        if len(chosen) > CAP:
+            chosen = chosen[:CAP]
+
+        for idx in chosen:
+            keep_indices.add(idx)
+
+        # stats: for each chosen, look up its bucket
+        # (skip detailed bucket stats per project; aggregate later)
+
+    # write output
+    with open(DST, "w", encoding="utf-8") as fout:
+        for idx in sorted(keep_indices):
+            fout.write(rows[idx])
+
+    n_kept = len(keep_indices)
+    n_total = len(rows)
+
+    report = {
+        "src": str(SRC),
+        "dst": str(DST),
+        "n_total": n_total,
+        "n_kept": n_kept,
+        "n_dropped": n_total - n_kept,
+        "drop_rate": round(1 - n_kept / max(n_total, 1), 4),
+        "n_projects": len(proj_buckets),
+        "n_projects_capped": n_proj_capped,
+        "n_projects_untouched": n_proj_untouched,
+        "cap": CAP,
+        "d1_target": D1_TARGET,
+        "rep_limit_per_d1": REP_LIMIT_PER_D1,
+        "seed": SEED,
+    }
+    REPORT.write_text(json.dumps(report, indent=2, ensure_ascii=False))
+    print(json.dumps(report, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

code/f3_filtering/clean_F3_stage3.py

@@ -0,0 +1,101 @@
+"""F3 → F3_v2 — Stage 3: per-project assistant deduplication, K=3.
+
+For each project (identified by user-prompt-prefix hash), keep at most K samples
+per unique assistant code. Sampling is random within over-represented groups
+(seed=0).
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import random
+import re
+from collections import defaultdict
+from pathlib import Path
+
+SRC = Path("/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage2.jsonl")
+DST = Path("/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage3.jsonl")
+REPORT = Path("/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage3_report.json")
+
+K = 3
+SEED = 0
+
+CODE_FENCE_RE = re.compile(r"```(?:python|py)?\s*\n([\s\S]*?)```", re.IGNORECASE)
+
+
+def get_assistant_code(obj: dict) -> str:
+    for m in obj.get("messages", []):
+        if m.get("role") == "assistant":
+            text = m.get("content", "")
+            mfence = CODE_FENCE_RE.search(text)
+            return mfence.group(1) if mfence else text
+    return ""
+
+
+def get_user(obj: dict) -> str:
+    for m in obj.get("messages", []):
+        if m.get("role") == "user":
+            return m.get("content", "")
+    return ""
+
+
+def md5(s: str) -> str:
+    return hashlib.md5(s.encode()).hexdigest()
+
+
+def main() -> None:
+    DST.parent.mkdir(parents=True, exist_ok=True)
+    rng = random.Random(SEED)
+
+    # group sample indices by (proj_hash, asst_hash)
+    groups: dict[tuple[str, str], list[int]] = defaultdict(list)
+    rows: list[str] = []
+    with open(SRC) as fin:
+        for idx, line in enumerate(fin):
+            obj = json.loads(line)
+            u = get_user(obj)
+            a = get_assistant_code(obj)
+            proj_h = md5(u[:500])
+            asst_h = md5(a)
+            groups[(proj_h, asst_h)].append(idx)
+            rows.append(line)
+
+    n_total = len(rows)
+    keep: set[int] = set()
+    n_groups = len(groups)
+    n_groups_truncated = 0
+    over_count = 0
+
+    for (proj_h, asst_h), idxs in groups.items():
+        if len(idxs) <= K:
+            for i in idxs: keep.add(i)
+        else:
+            n_groups_truncated += 1
+            over_count += len(idxs) - K
+            rng.shuffle(idxs)
+            for i in idxs[:K]: keep.add(i)
+
+    with open(DST, "w", encoding="utf-8") as fout:
+        for idx in sorted(keep):
+            fout.write(rows[idx])
+
+    n_kept = len(keep)
+    report = {
+        "src": str(SRC),
+        "dst": str(DST),
+        "n_total": n_total,
+        "n_kept": n_kept,
+        "n_dropped": n_total - n_kept,
+        "drop_rate": round(1 - n_kept / max(n_total, 1), 4),
+        "n_unique_(proj,asst)_groups": n_groups,
+        "n_groups_truncated": n_groups_truncated,
+        "n_dups_dropped": over_count,
+        "K": K,
+        "seed": SEED,
+    }
+    REPORT.write_text(json.dumps(report, indent=2, ensure_ascii=False))
+    print(json.dumps(report, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

code/f3_filtering/clean_F3_stage4.py

@@ -0,0 +1,186 @@
+"""F3 → F3_v2 — Stage 4 (full): Qwen3-Coder-30B-A3B-Instruct quality scoring.
+
+For every sample in F3_stage3.jsonl, ask the model to rate 1-5. Keep score >= 3.
+Run with the skill conda env (vLLM)."""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import time
+from collections import Counter
+from pathlib import Path
+
+CODE_FENCE_RE = re.compile(r"```(?:python|py)?\s*\n([\s\S]*?)```", re.IGNORECASE)
+GENERIC_FENCE_RE = re.compile(r"```(\S*)\s*\n([\s\S]*?)```", re.IGNORECASE)
+
+DEFAULT_MODEL = "/mnt/moon-data/jfjiang/swf/skillsecure/data/models/Qwen3-Coder-30B-A3B-Instruct"
+DEFAULT_SRC = "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage3.jsonl"
+DEFAULT_DST = "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage4.jsonl"
+DEFAULT_REPORT = "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage4_report.json"
+DEFAULT_SCORES = "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage4_scores.jsonl"
+
+KEEP_THRESHOLD = 3
+
+SCORE_PROMPT = """You are a senior Python code reviewer. Rate the quality of the following function on a scale of 1-5:
+
+1 = unusable (broken, placeholder, or nonsense)
+2 = poor (works but low quality, bad style, or hardcoded values)
+3 = ok (works, average style)
+4 = good (clean, readable, reasonable structure)
+5 = excellent (production-ready, well-documented, robust)
+
+Function description (from project context):
+{user_brief}
+
+Code:
+```python
+{code}
+```
+
+Output ONLY a single digit from 1 to 5. No explanation."""
+
+
+def get_user_brief(u: str, max_chars: int = 1500) -> str:
+    return u[:max_chars]
+
+
+def get_assistant_code(obj: dict) -> str:
+    for m in obj.get("messages", []):
+        if m.get("role") == "assistant":
+            text = m.get("content", "")
+            mfence = GENERIC_FENCE_RE.search(text)
+            return mfence.group(2) if mfence else text
+    return ""
+
+
+def get_user(obj: dict) -> str:
+    for m in obj.get("messages", []):
+        if m.get("role") == "user":
+            return m.get("content", "")
+    return ""
+
+
+def parse_score(text: str) -> int:
+    if not text:
+        return 0
+    m = re.search(r"\b([1-5])\b", text)
+    return int(m.group(1)) if m else 0
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--src", default=DEFAULT_SRC)
+    ap.add_argument("--dst", default=DEFAULT_DST)
+    ap.add_argument("--report", default=DEFAULT_REPORT)
+    ap.add_argument("--scores", default=DEFAULT_SCORES)
+    ap.add_argument("--model", default=DEFAULT_MODEL)
+    ap.add_argument("--threshold", type=int, default=KEEP_THRESHOLD)
+    ap.add_argument("--gpu-mem-util", type=float, default=0.85)
+    ap.add_argument("--max-model-len", type=int, default=8192)
+    ap.add_argument("--batch-chunk", type=int, default=20000,
+                    help="how many samples to feed vLLM in one .generate() call")
+    args = ap.parse_args()
+
+    src = Path(args.src)
+    rows: list[str] = []
+    with open(src) as f:
+        for line in f:
+            rows.append(line)
+    print(f"loaded {len(rows)} samples")
+
+    from vllm import LLM, SamplingParams
+    from transformers import AutoTokenizer
+
+    tok = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    llm = LLM(
+        model=args.model, tokenizer=args.model, trust_remote_code=True,
+        max_model_len=args.max_model_len, gpu_memory_utilization=args.gpu_mem_util,
+        dtype="bfloat16",
+    )
+    sp = SamplingParams(temperature=0.0, top_p=1.0, max_tokens=8)
+
+    # Pre-build all chat prompts; if a prompt would exceed max_model_len after tokenize, skip (rare).
+    print("building chat prompts...")
+    chat_prompts: list[str] = []
+    skip_indices: set[int] = set()
+    for idx, line in enumerate(rows):
+        obj = json.loads(line)
+        user_brief = get_user_brief(get_user(obj))
+        code = get_assistant_code(obj)
+        prompt = SCORE_PROMPT.format(user_brief=user_brief, code=code)
+        msgs = [{"role": "user", "content": prompt}]
+        try:
+            chat_prompt = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
+        except Exception:
+            chat_prompt = prompt
+        # quick token-length check to avoid validation error
+        tok_ids = tok(chat_prompt, add_special_tokens=False)["input_ids"]
+        if len(tok_ids) >= args.max_model_len - 10:
+            skip_indices.add(idx)
+            chat_prompts.append("")  # placeholder
+        else:
+            chat_prompts.append(chat_prompt)
+    print(f"too-long (skipped, default-keep): {len(skip_indices)}")
+
+    scores: list[int] = [0] * len(rows)
+    n_chunk = (len(rows) + args.batch_chunk - 1) // args.batch_chunk
+    cumulative = 0
+    t0 = time.time()
+    for c in range(n_chunk):
+        lo = c * args.batch_chunk
+        hi = min((c + 1) * args.batch_chunk, len(rows))
+        batch_idx = [i for i in range(lo, hi) if i not in skip_indices]
+        batch_prompts = [chat_prompts[i] for i in batch_idx]
+        if not batch_prompts:
+            continue
+        print(f"chunk {c+1}/{n_chunk}: scoring {len(batch_prompts)} prompts (lo={lo})")
+        outs = llm.generate(batch_prompts, sp)
+        for i, o in zip(batch_idx, outs):
+            scores[i] = parse_score(o.outputs[0].text)
+        cumulative += len(batch_prompts)
+        elapsed = time.time() - t0
+        rate = cumulative / max(elapsed, 1e-9)
+        eta = (len(rows) - len(skip_indices) - cumulative) / max(rate, 1e-9)
+        print(f"  done {cumulative}; rate={rate:.1f}/s; eta={eta/60:.1f}min")
+
+    # write scores
+    with open(args.scores, "w") as f:
+        for i, s in enumerate(scores):
+            f.write(json.dumps({"idx": i, "score": s}, ensure_ascii=False) + "\n")
+
+    # filter and write
+    n_keep = 0
+    score_dist: Counter[int] = Counter()
+    with open(args.dst, "w", encoding="utf-8") as fout:
+        for i, line in enumerate(rows):
+            s = scores[i]
+            score_dist[s] += 1
+            if i in skip_indices or s >= args.threshold:
+                fout.write(line)
+                n_keep += 1
+
+    elapsed_total = time.time() - t0
+    report = {
+        "src": str(src),
+        "dst": args.dst,
+        "model": args.model,
+        "threshold": args.threshold,
+        "n_total": len(rows),
+        "n_kept": n_keep,
+        "n_dropped": len(rows) - n_keep,
+        "drop_rate": round(1 - n_keep / max(len(rows), 1), 4),
+        "n_skipped_too_long_kept_default": len(skip_indices),
+        "score_distribution": dict(sorted(score_dist.items())),
+        "score_distribution_pct": {str(k): round(v / len(rows) * 100, 2) for k, v in sorted(score_dist.items())},
+        "elapsed_sec": round(elapsed_total, 1),
+    }
+    Path(args.report).parent.mkdir(parents=True, exist_ok=True)
+    with open(args.report, "w") as f:
+        json.dump(report, f, indent=2, ensure_ascii=False)
+    print("\n=== summary ===")
+    print(json.dumps(report, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

code/f3_filtering/clean_F3_stage4_calibrate.py

@@ -0,0 +1,156 @@
+"""F3 → F3_v2 — Stage 4 calibration: Qwen3-Coder-30B-A3B-Instruct 1-5 quality score on 500 sampled rows.
+
+Prints score distribution + a few low/high-score samples for sanity check.
+Run with the skill conda env (has vLLM)."""
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import re
+import time
+from collections import Counter
+from pathlib import Path
+
+CODE_FENCE_RE = re.compile(r"```(?:python|py)?\s*\n([\s\S]*?)```", re.IGNORECASE)
+
+DEFAULT_MODEL = "/mnt/moon-data/jfjiang/swf/skillsecure/data/models/Qwen3-Coder-30B-A3B-Instruct"
+DEFAULT_SRC = "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage3.jsonl"
+DEFAULT_OUT = "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage4_calibration.json"
+DEFAULT_SCORES = "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage4_calibration_scores.jsonl"
+
+
+SCORE_PROMPT = """You are a senior Python code reviewer. Rate the quality of the following function on a scale of 1-5:
+
+1 = unusable (broken, placeholder, or nonsense)
+2 = poor (works but low quality, bad style, or hardcoded values)
+3 = ok (works, average style)
+4 = good (clean, readable, reasonable structure)
+5 = excellent (production-ready, well-documented, robust)
+
+Function description (from project context):
+{user_brief}
+
+Code:
+```python
+{code}
+```
+
+Output ONLY a single digit from 1 to 5. No explanation."""
+
+
+def get_user_brief(u: str, max_chars: int = 1500) -> str:
+    """Trim user prompt to function-relevant section (skip Project Overview if too long)."""
+    return u[:max_chars]
+
+
+def get_assistant_code(obj: dict) -> str:
+    for m in obj.get("messages", []):
+        if m.get("role") == "assistant":
+            text = m.get("content", "")
+            mfence = CODE_FENCE_RE.search(text)
+            return mfence.group(1) if mfence else text
+    return ""
+
+
+def get_user(obj: dict) -> str:
+    for m in obj.get("messages", []):
+        if m.get("role") == "user":
+            return m.get("content", "")
+    return ""
+
+
+def parse_score(text: str) -> int:
+    if not text: return 0
+    m = re.search(r"\b([1-5])\b", text)
+    if m: return int(m.group(1))
+    return 0  # unparseable
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--src", default=DEFAULT_SRC)
+    ap.add_argument("--model", default=DEFAULT_MODEL)
+    ap.add_argument("--n-sample", type=int, default=500)
+    ap.add_argument("--seed", type=int, default=0)
+    ap.add_argument("--out-summary", default=DEFAULT_OUT)
+    ap.add_argument("--out-scores", default=DEFAULT_SCORES)
+    ap.add_argument("--gpu-mem-util", type=float, default=0.85)
+    ap.add_argument("--max-model-len", type=int, default=8192)
+    args = ap.parse_args()
+
+    rng = random.Random(args.seed)
+    src = Path(args.src)
+
+    rows = []
+    with open(src) as f:
+        for line in f:
+            rows.append(line)
+    print(f"loaded {len(rows)} samples from {src.name}")
+
+    sample_idx = rng.sample(range(len(rows)), args.n_sample)
+    sampled = [(i, json.loads(rows[i])) for i in sample_idx]
+
+    prompts = []
+    for _, obj in sampled:
+        user_brief = get_user_brief(get_user(obj))
+        code = get_assistant_code(obj)
+        prompts.append(SCORE_PROMPT.format(user_brief=user_brief, code=code))
+
+    from vllm import LLM, SamplingParams
+    from transformers import AutoTokenizer
+
+    tok = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    llm = LLM(
+        model=args.model, tokenizer=args.model, trust_remote_code=True,
+        max_model_len=args.max_model_len, gpu_memory_utilization=args.gpu_mem_util,
+        dtype="bfloat16",
+    )
+    sp = SamplingParams(temperature=0.0, top_p=1.0, max_tokens=8)
+
+    chat_prompts = []
+    for p in prompts:
+        msgs = [{"role": "user", "content": p}]
+        chat_prompts.append(tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True))
+
+    print(f"scoring {len(chat_prompts)}...")
+    t0 = time.time()
+    outputs = llm.generate(chat_prompts, sp)
+    secs = time.time() - t0
+
+    scores = []
+    raw_outs = []
+    for o in outputs:
+        text = o.outputs[0].text
+        raw_outs.append(text)
+        scores.append(parse_score(text))
+
+    dist = Counter(scores)
+    summary = {
+        "n_sample": len(sampled),
+        "elapsed_sec": round(secs, 1),
+        "score_distribution": {str(k): dist.get(k, 0) for k in [0, 1, 2, 3, 4, 5]},
+        "score_distribution_pct": {str(k): round(dist.get(k, 0) / len(sampled) * 100, 1) for k in [0, 1, 2, 3, 4, 5]},
+        "ge3_count": sum(dist.get(k, 0) for k in [3, 4, 5]),
+        "ge3_pct": round(sum(dist.get(k, 0) for k in [3, 4, 5]) / len(sampled) * 100, 1),
+        "ge4_count": sum(dist.get(k, 0) for k in [4, 5]),
+        "ge4_pct": round(sum(dist.get(k, 0) for k in [4, 5]) / len(sampled) * 100, 1),
+    }
+    Path(args.out_summary).parent.mkdir(parents=True, exist_ok=True)
+    with open(args.out_summary, "w") as f:
+        json.dump(summary, f, indent=2, ensure_ascii=False)
+    with open(args.out_scores, "w") as f:
+        for (i, obj), s, raw in zip(sampled, scores, raw_outs):
+            code = get_assistant_code(obj)
+            f.write(json.dumps({
+                "idx": i,
+                "score": s,
+                "raw_out": raw,
+                "code_head": code[:200],
+            }, ensure_ascii=False) + "\n")
+    print("\n=== summary ===")
+    print(json.dumps(summary, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

code/f3_filtering/reports/F3_stage1_report.json

@@ -0,0 +1,26 @@
+{
+  "src": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean.jsonl",
+  "dst": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage1.jsonl",
+  "n_total": 473465,
+  "n_kept": 437671,
+  "n_dropped": 35794,
+  "drop_rate": 0.0756,
+  "drop_reasons": {
+    "1b_line_repeat": 13506,
+    "1c_too_short": 9295,
+    "1a_ast_fail_python": 5119,
+    "1e_placeholder": 2497,
+    "1d_hardcoded_path": 2396,
+    "1c_too_long": 1621,
+    "1f_debug_pdb": 962,
+    "1f_many_todo": 398
+  },
+  "kept_languages": {
+    "python": 400263,
+    "cpp": 16733,
+    "c": 10995,
+    "java": 6879,
+    "rust": 2577,
+    "go": 224
+  }
+}

code/f3_filtering/reports/F3_stage2_report.json

@@ -0,0 +1,20 @@
+{
+  "src": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage1.jsonl",
+  "dst": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage2.jsonl",
+  "n_total": 437671,
+  "n_kept": 366672,
+  "n_dropped": 70999,
+  "drop_rate": 0.1622,
+  "n_projects": 14408,
+  "n_projects_capped": 217,
+  "n_projects_untouched": 14191,
+  "cap": 300,
+  "d1_target": {
+    "S": 75,
+    "M1": 75,
+    "M2": 75,
+    "L": 75
+  },
+  "rep_limit_per_d1": 5,
+  "seed": 0
+}

code/f3_filtering/reports/F3_stage3_report.json

@@ -0,0 +1,13 @@
+{
+  "src": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage2.jsonl",
+  "dst": "/raid/data/weifeng/Datasets/Instruction-turning/F3_v2_pipeline/F3_stage3.jsonl",
+  "n_total": 366672,
+  "n_kept": 362367,
+  "n_dropped": 4305,
+  "drop_rate": 0.0117,
+  "n_unique_(proj,asst)_groups": 344696,
+  "n_groups_truncated": 1509,
+  "n_dups_dropped": 4305,
+  "K": 3,
+  "seed": 0
+}

code/f3_filtering/reports/F3_stage4_calibration.json

@@ -0,0 +1,24 @@
+{
+  "n_sample": 500,
+  "elapsed_sec": 9.3,
+  "score_distribution": {
+    "0": 0,
+    "1": 19,
+    "2": 180,
+    "3": 233,
+    "4": 66,
+    "5": 2
+  },
+  "score_distribution_pct": {
+    "0": 0.0,
+    "1": 3.8,
+    "2": 36.0,
+    "3": 46.6,
+    "4": 13.2,
+    "5": 0.4
+  },
+  "ge3_count": 301,
+  "ge3_pct": 60.2,
+  "ge4_count": 68,
+  "ge4_pct": 13.6
+}

code/f3_filtering/reports/F3_v3_build.log

@@ -0,0 +1,45 @@
+=== F3_v3 cleaning pipeline ===
+Pass 1: scanning /raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v2.jsonl
+  [50,000]  bad_long=827  bad_line=934  bad_ngram=1382  uniq_hashes=42,720
+  [100,000]  bad_long=1732  bad_line=1875  bad_ngram=2931  uniq_hashes=81,939
+  [150,000]  bad_long=2709  bad_line=3110  bad_ngram=4630  uniq_hashes=119,702
+  [200,000]  bad_long=3624  bad_line=4026  bad_ngram=6034  uniq_hashes=159,121
+Pass 1 done in 39s.  total samples = 207,009
+  exact-content dup clusters: 17,531  covering 61,156 rows
+Pass 2: writing /raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v3.jsonl
+  [50,000]  kept=40523  dropped: long=827, line=708, ngram=1046, dup=6896
+  [100,000]  kept=77467  dropped: long=1732, line=1400, ngram=2224, dup=17177
+  [150,000]  kept=112838  dropped: long=2709, line=2307, ngram=3474, dup=28672
+{
+  "input": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v2.jsonl",
+  "output": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v3.jsonl",
+  "input_size": 207009,
+  "final_size": 154103,
+  "kept_rate": 0.7444,
+  "rules": [
+    "R1: drop if assistant > 8000 chars",
+    "R2: drop if any stripped line repeats >= 10 times in assistant",
+    "R3: drop if any 5-gram repeats >= 20 times in assistant",
+    "R4: drop cross-corpus exact-content duplicates (keep first occurrence)"
+  ],
+  "scan_stats_before_filter": {
+    "samples_with_long_assistant": 3764,
+    "samples_with_line_repeat": 4140,
+    "samples_with_ngram_repeat": 6251,
+    "samples_with_both_line_and_ngram": 652,
+    "samples_with_only_line": 3488,
+    "samples_with_only_ngram": 5599,
+    "exact_dup_clusters": 17531,
+    "exact_dup_rows_total": 61156
+  },
+  "applied_drops": {
+    "R1_too_long": 3764,
+    "R2_line_repeat": 3046,
+    "R3_ngram_repeat": 4677,
+    "R4_dup": 41419,
+    "total_dropped": 52906
+  },
+  "elapsed_sec": 81.8
+}
+
+Wrote /raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v3.jsonl  (0.79 GB, 154,103 samples)

code/f3_filtering/reports/F3_v3_report.json

@@ -0,0 +1,31 @@
+{
+  "input": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v2.jsonl",
+  "output": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v3.jsonl",
+  "input_size": 207009,
+  "final_size": 154103,
+  "kept_rate": 0.7444,
+  "rules": [
+    "R1: drop if assistant > 8000 chars",
+    "R2: drop if any stripped line repeats >= 10 times in assistant",
+    "R3: drop if any 5-gram repeats >= 20 times in assistant",
+    "R4: drop cross-corpus exact-content duplicates (keep first occurrence)"
+  ],
+  "scan_stats_before_filter": {
+    "samples_with_long_assistant": 3764,
+    "samples_with_line_repeat": 4140,
+    "samples_with_ngram_repeat": 6251,
+    "samples_with_both_line_and_ngram": 652,
+    "samples_with_only_line": 3488,
+    "samples_with_only_ngram": 5599,
+    "exact_dup_clusters": 17531,
+    "exact_dup_rows_total": 61156
+  },
+  "applied_drops": {
+    "R1_too_long": 3764,
+    "R2_line_repeat": 3046,
+    "R3_ngram_repeat": 4677,
+    "R4_dup": 41419,
+    "total_dropped": 52906
+  },
+  "elapsed_sec": 81.8
+}

code/f3_filtering/reports/F3_v4_build.log

@@ -0,0 +1,40 @@
+=== F3_v4 cleaning pipeline ===
+Pass 1: scanning + computing features + minhash
+  [20,000]  features computed
+  [40,000]  features computed
+  [60,000]  features computed
+  [80,000]  features computed
+  [100,000]  features computed
+  [120,000]  features computed
+  [140,000]  features computed
+Pass 1 done in 95s.  total = 154,103
+Pass 2: LSH bucketing for near-dup
+  candidate pairs: 28,732
+  near-dup marked: 6,390
+Pass 3: writing /raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v4.jsonl
+{
+  "input": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v3.jsonl",
+  "output": "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v4.jsonl",
+  "input_size": 154103,
+  "final_size": 134775,
+  "kept_rate": 0.8746,
+  "rules": [
+    "S1: MinHash near-dup, Jaccard >= 0.85, bottom-128 shingle-8",
+    "S2: template ctor / setup / main with trivial body (content-based)",
+    "S3: trivial body (< 3 non-trivial statements)",
+    "S4: comment+docstring ratio > 0.7",
+    "S6: import-only / docstring-only assistant",
+    "S8: placeholder residue (TODO, FIXME, XXX, HACK, pdb.set_trace, ...)"
+  ],
+  "applied_drops": {
+    "S1_near_dup": 6390,
+    "S3_trivial": 6376,
+    "S8_placeholder": 3958,
+    "S4_comment_ratio": 2286,
+    "S2_template": 318
+  },
+  "total_dropped": 19328,
+  "elapsed_sec": 101.0
+}
+
+Wrote /raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v4.jsonl  (0.70 GB, 134,775 samples)

code/pretrain_data_prep/analyze_template_repetition.py

@@ -0,0 +1,199 @@
+"""Scan filtered/filtered_*.jsonl, report:
+  1. Top template-style basenames (counts + total bytes + duplicate ratio per file).
+  2. Top high-frequency lines (stripped) across the whole corpus, with sample coverage.
+  3. Top short-phrase n-grams (3-gram and 5-gram over tokens of stripped lines), sampled.
+
+Output: filtered_repetition_report.json + .md
+Pure-stdlib, single-pass over disk, ~10GB.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+from collections import Counter, defaultdict
+from glob import glob
+from pathlib import Path
+
+SRC_DIR = Path("/raid/data/weifeng/Datasets/filtered")
+OUT_JSON = Path("/raid/data/weifeng/Datasets/filtered_repetition_report.json")
+OUT_MD = Path("/raid/data/weifeng/Datasets/filtered_repetition_report.md")
+
+TEMPLATE_BASENAMES = {
+    "__init__.py", "setup.py", "config.py", "conf.py", "settings.py",
+    "constants.py", "version.py", "_version.py", "manage.py",
+    "decorators.py", "exceptions.py", "errors.py", "utils.py",
+    "types.py", "schemas.py", "logging.py", "logger.py",
+    "__main__.py", "wsgi.py", "asgi.py", "urls.py", "apps.py",
+    "models.py", "admin.py",  # not always template, but check
+}
+
+# heuristic: line frequency we care about
+TOP_LINES_KEEP = 2000        # keep top N lines for report
+TOP_BASENAMES_KEEP = 500
+NGRAM_TOP_KEEP = 500
+NGRAM_SAMPLE_EVERY = 20      # only do n-gram counting on every Nth file to save memory
+
+# strip whitespace, ignore very short / very long lines for stats
+MIN_LINE_LEN = 8
+MAX_LINE_LEN = 200
+
+# token regex for n-grams over line text
+TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|[^\s\w]")
+
+
+def main() -> None:
+    files = sorted(glob(str(SRC_DIR / "filtered_*.jsonl")))
+    print(f"found {len(files)} jsonl files", flush=True)
+
+    basename_counter: Counter[str] = Counter()
+    basename_bytes: Counter[str] = Counter()
+    basename_dup_lines: Counter[str] = Counter()  # how many duplicate lines (within file) total per basename
+    line_counter: Counter[str] = Counter()
+    line_sample_coverage: Counter[str] = Counter()  # how many distinct samples contain the line
+    ngram3_counter: Counter[tuple] = Counter()
+    ngram5_counter: Counter[tuple] = Counter()
+
+    total_samples = 0
+    total_bytes = 0
+    py_samples = 0
+
+    for fi, fp in enumerate(files):
+        do_ngram = (fi % NGRAM_SAMPLE_EVERY == 0)
+        try:
+            with open(fp, encoding="utf-8") as f:
+                for line in f:
+                    try:
+                        o = json.loads(line)
+                    except Exception:
+                        continue
+                    path = o.get("path", "") or ""
+                    text = o.get("text", "") or ""
+                    if not text:
+                        continue
+                    total_samples += 1
+                    total_bytes += len(text)
+                    basename = os.path.basename(path) if path else "<unknown>"
+                    basename_counter[basename] += 1
+                    basename_bytes[basename] += len(text)
+                    if path.endswith(".py"):
+                        py_samples += 1
+
+                    # within-file line duplication: how repetitive is THIS file?
+                    file_lines = [ln.strip() for ln in text.split("\n")]
+                    file_line_cnt = Counter(ln for ln in file_lines if MIN_LINE_LEN <= len(ln) <= MAX_LINE_LEN)
+                    dup_here = sum(c - 1 for c in file_line_cnt.values() if c > 1)
+                    basename_dup_lines[basename] += dup_here
+
+                    # cross-corpus line stats
+                    seen_in_this_sample: set[str] = set()
+                    for ln, c in file_line_cnt.items():
+                        line_counter[ln] += c
+                        if ln not in seen_in_this_sample:
+                            line_sample_coverage[ln] += 1
+                            seen_in_this_sample.add(ln)
+
+                    if do_ngram and len(file_lines) > 5:
+                        # take a slice to bound memory
+                        for ln in file_lines[:400]:
+                            toks = TOKEN_RE.findall(ln)
+                            if len(toks) < 3:
+                                continue
+                            for i in range(len(toks) - 2):
+                                ngram3_counter[tuple(toks[i:i + 3])] += 1
+                            if len(toks) >= 5:
+                                for i in range(len(toks) - 4):
+                                    ngram5_counter[tuple(toks[i:i + 5])] += 1
+
+                    # prune to avoid OOM
+                    if total_samples % 100000 == 0:
+                        print(f"  [{total_samples}] processed; "
+                              f"line_counter={len(line_counter):,} ngram3={len(ngram3_counter):,}",
+                              flush=True)
+                        if len(line_counter) > 5_000_000:
+                            # keep top half
+                            cutoff = line_counter.most_common(2_000_000)
+                            line_counter = Counter(dict(cutoff))
+                            line_sample_coverage = Counter({k: line_sample_coverage[k] for k, _ in cutoff})
+                        if len(ngram3_counter) > 3_000_000:
+                            ngram3_counter = Counter(dict(ngram3_counter.most_common(1_000_000)))
+                        if len(ngram5_counter) > 3_000_000:
+                            ngram5_counter = Counter(dict(ngram5_counter.most_common(1_000_000)))
+        except Exception as e:
+            print(f"  ERR {fp}: {e}", file=sys.stderr, flush=True)
+
+    print(f"DONE: {total_samples:,} samples, {total_bytes/1e9:.2f} GB", flush=True)
+
+    # ----- build report -----
+    top_basenames = basename_counter.most_common(TOP_BASENAMES_KEEP)
+    top_lines = line_counter.most_common(TOP_LINES_KEEP)
+    top_ngram3 = ngram3_counter.most_common(NGRAM_TOP_KEEP)
+    top_ngram5 = ngram5_counter.most_common(NGRAM_TOP_KEEP)
+
+    report = {
+        "total_samples": total_samples,
+        "total_bytes": total_bytes,
+        "py_samples": py_samples,
+        "n_files_scanned": len(files),
+        "top_basenames": [
+            {
+                "basename": b,
+                "count": c,
+                "pct_of_corpus": round(c / total_samples * 100, 3),
+                "total_bytes": basename_bytes[b],
+                "dup_lines_within_files": basename_dup_lines[b],
+            }
+            for b, c in top_basenames
+        ],
+        "top_lines_corpus_wide": [
+            {
+                "line": ln,
+                "occurrences": c,
+                "n_samples_containing": line_sample_coverage[ln],
+                "pct_samples": round(line_sample_coverage[ln] / total_samples * 100, 3),
+            }
+            for ln, c in top_lines[:500]
+        ],
+        "top_3grams": [{"tokens": list(t), "count": c} for t, c in top_ngram3],
+        "top_5grams": [{"tokens": list(t), "count": c} for t, c in top_ngram5],
+        "ngram_sampling_note": f"n-gram counted on every {NGRAM_SAMPLE_EVERY}th file, first 400 lines per sample",
+    }
+    OUT_JSON.write_text(json.dumps(report, ensure_ascii=False, indent=2))
+    print(f"wrote {OUT_JSON} ({OUT_JSON.stat().st_size/1e6:.1f} MB)", flush=True)
+
+    # ----- markdown summary -----
+    md = []
+    md.append(f"# Filtered pretrain repetition report\n")
+    md.append(f"- samples: **{total_samples:,}**, bytes: **{total_bytes/1e9:.2f} GB**, .py: {py_samples:,}\n")
+    md.append(f"- files scanned: {len(files)}\n\n")
+
+    md.append("## Top template basenames (by sample count)\n\n")
+    md.append("| basename | count | % corpus | total bytes | dup lines (within-file) |\n|---|---:|---:|---:|---:|\n")
+    for b, c in top_basenames[:60]:
+        md.append(f"| `{b}` | {c} | {c/total_samples*100:.2f}% | {basename_bytes[b]/1e6:.1f} MB | {basename_dup_lines[b]} |\n")
+
+    md.append("\n## Top corpus-wide lines (frequency)\n\n")
+    md.append("| line | occurrences | samples containing | % samples |\n|---|---:|---:|---:|\n")
+    for ln, c in top_lines[:80]:
+        disp = ln.replace("|", "\\|")[:120]
+        md.append(f"| `{disp}` | {c} | {line_sample_coverage[ln]} | {line_sample_coverage[ln]/total_samples*100:.2f}% |\n")
+
+    md.append("\n## Top 5-grams (token-level)\n\n")
+    md.append("| 5-gram | count |\n|---|---:|\n")
+    for t, c in top_ngram5[:60]:
+        disp = " ".join(t).replace("|", "\\|")[:120]
+        md.append(f"| `{disp}` | {c} |\n")
+
+    md.append("\n## Top 3-grams (token-level)\n\n")
+    md.append("| 3-gram | count |\n|---|---:|\n")
+    for t, c in top_ngram3[:60]:
+        disp = " ".join(t).replace("|", "\\|")[:120]
+        md.append(f"| `{disp}` | {c} |\n")
+
+    OUT_MD.write_text("".join(md))
+    print(f"wrote {OUT_MD}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

code/pretrain_data_prep/build_filtered_pretrain.py

@@ -0,0 +1,76 @@
+"""Merge filtered/*.jsonl into swift pre-train format and shuffle.
+
+Output: code_messages_pre_train_filtered.jsonl
+Format per line: {"messages": [{"role": "assistant", "content": <text>}]}
+"""
+from __future__ import annotations
+
+import json
+import random
+from collections import Counter
+from pathlib import Path
+
+SRC_DIR = Path("/raid/data/weifeng/Datasets/filtered")
+DST = Path("/raid/data/weifeng/Datasets/code_messages_pre_train_filtered.jsonl")
+REPORT = Path("/raid/data/weifeng/Datasets/code_messages_pre_train_filtered_report.json")
+SEED = 0
+
+
+def main() -> None:
+    rng = random.Random(SEED)
+    files = sorted(SRC_DIR.glob("filtered_*.jsonl"))
+    print(f"found {len(files)} domain files")
+
+    out_lines: list[str] = []
+    by_domain: Counter[str] = Counter()
+    n_total = n_kept = n_empty = n_bad = 0
+    bad_by_file: Counter[str] = Counter()
+
+    for fp in files:
+        domain_kept = 0
+        with open(fp, encoding="utf-8", errors="replace") as f:
+            for line_no, line in enumerate(f, 1):
+                n_total += 1
+                try:
+                    obj = json.loads(line)
+                except Exception:
+                    n_bad += 1
+                    bad_by_file[fp.name] += 1
+                    continue
+                text = obj.get("text", "")
+                if not isinstance(text, str) or not text.strip():
+                    n_empty += 1
+                    continue
+                record = {"messages": [{"role": "assistant", "content": text}]}
+                out_lines.append(json.dumps(record, ensure_ascii=False))
+                domain_kept += 1
+                n_kept += 1
+        by_domain[fp.stem.replace("filtered_", "")] = domain_kept
+        print(f"  {fp.name}: kept {domain_kept} (bad rows: {bad_by_file[fp.name]})")
+
+    print(f"\nshuffling {len(out_lines)} records (seed={SEED})...")
+    rng.shuffle(out_lines)
+
+    print(f"writing → {DST}")
+    with open(DST, "w", encoding="utf-8") as fout:
+        for line in out_lines:
+            fout.write(line + "\n")
+
+    report = {
+        "src_dir": str(SRC_DIR),
+        "dst": str(DST),
+        "seed": SEED,
+        "n_total_input": n_total,
+        "n_empty_dropped": n_empty,
+        "n_bad_rows_dropped": n_bad,
+        "bad_rows_by_file": dict(bad_by_file),
+        "n_kept": n_kept,
+        "per_domain_kept": dict(by_domain.most_common()),
+        "total_bytes": DST.stat().st_size,
+    }
+    REPORT.write_text(json.dumps(report, indent=2, ensure_ascii=False))
+    print(json.dumps(report, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

code/pretrain_data_prep/dryrun_filter_repetition.py

@@ -0,0 +1,166 @@
+"""Dry-run apply the 5 filter rules over filtered/filtered_*.jsonl. Count only,
+don't write output.
+
+Rules:
+  1) drop by basename (CHANGELOG/NEWS/CONTRIBUTING/version.py/etc)
+  2) cap per basename N=200 (random sample via reservoir)
+  3) collapse long runs of same char inside a line (modifies text, doesn't drop)
+  4) drop sample if any 5-token n-gram repeats >=30 times
+  5) drop sample if any short line repeats >=20 times
+
+Reports: how many samples remain, how much each rule dropped.
+"""
+from __future__ import annotations
+
+import json
+import os
+import random
+import re
+from collections import Counter, defaultdict
+from glob import glob
+from pathlib import Path
+
+SRC_DIR = Path("/raid/data/weifeng/Datasets/filtered")
+REPORT = Path("/raid/data/weifeng/Datasets/dryrun_filter_report.json")
+
+DROP_BASENAMES = {
+    "CHANGELOG.md", "NEWS.md", "CONTRIBUTING.md", "SKILL.md",
+    "package-info.java", "index.md", "version.py", "_version.py",
+}
+CAPPED_BASENAMES = {
+    "__init__.py", "utils.py", "setup.py", "conftest.py", "conf.py",
+    "config.py", "settings.py", "constants.py", "__main__.py",
+}
+CAP = 200
+
+RUN_SAME_CHAR = re.compile(r"^(.)\1{40,}$")
+RUN_PUNCT = re.compile(r"^([^\w\s])\1{10,}$")
+TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|[^\s\w]")
+
+NGRAM_N = 5
+NGRAM_THRESH = 30
+LINE_REP_THRESH = 20
+LINE_MAX_LEN = 80
+SEED = 0
+
+
+def main() -> None:
+    rng = random.Random(SEED)
+    files = sorted(glob(str(SRC_DIR / "filtered_*.jsonl")))
+
+    total = 0
+    dropped_rule1 = 0
+    dropped_rule2 = 0
+    dropped_rule4 = 0
+    dropped_rule5 = 0
+    modified_rule3 = 0
+    kept = 0
+    bytes_kept = 0
+    bytes_total = 0
+
+    # rule2 reservoir: for capped basenames, decide kept indices by reservoir sampling
+    # easier: stream twice OR random.random() < CAP/seen_count threshold. We do
+    # the simpler "see first then decide". Pass 1 counts. Pass 2 applies. To save
+    # time on dry-run we accept slight bias: simple random.random()<keep_prob with
+    # estimated counts from prior repetition report.
+    from json import load as jload
+    prior = jload(open("/raid/data/weifeng/Datasets/filtered_repetition_report.json"))
+    basename_total = {x["basename"]: x["count"] for x in prior["top_basenames"]}
+    keep_prob = {b: min(1.0, CAP / basename_total[b]) for b in CAPPED_BASENAMES if b in basename_total}
+    print("keep_prob (cap 200):", keep_prob, flush=True)
+
+    drop_by_basename: Counter[str] = Counter()
+
+    for fi, fp in enumerate(files):
+        with open(fp, encoding="utf-8") as f:
+            for line in f:
+                try:
+                    o = json.loads(line)
+                except Exception:
+                    continue
+                total += 1
+                path = o.get("path", "") or ""
+                text = o.get("text", "") or ""
+                if not text:
+                    continue
+                bytes_total += len(text)
+                bname = os.path.basename(path)
+
+                # rule 1
+                if bname in DROP_BASENAMES:
+                    dropped_rule1 += 1
+                    drop_by_basename[bname] += 1
+                    continue
+                # rule 2
+                if bname in keep_prob and rng.random() > keep_prob[bname]:
+                    dropped_rule2 += 1
+                    drop_by_basename[bname] += 1
+                    continue
+
+                # rule 3: in-place collapse
+                lines_in = text.split("\n")
+                modified = False
+                new_lines = []
+                for ln in lines_in:
+                    if RUN_SAME_CHAR.match(ln) or RUN_PUNCT.match(ln):
+                        new_lines.append(ln[:5])
+                        modified = True
+                    else:
+                        new_lines.append(ln)
+                if modified:
+                    modified_rule3 += 1
+                    text = "\n".join(new_lines)
+
+                # rule 5: short line repeat
+                stripped = [ln.strip() for ln in new_lines]
+                cnt = Counter(ln for ln in stripped if 0 < len(ln) <= LINE_MAX_LEN)
+                if cnt and cnt.most_common(1)[0][1] >= LINE_REP_THRESH:
+                    dropped_rule5 += 1
+                    continue
+
+                # rule 4: 5-gram repeat (skip for very small samples)
+                tokens = TOKEN_RE.findall(text)
+                if len(tokens) >= 100:
+                    # only check head + tail slices to bound cost
+                    sample_toks = tokens[:5000] if len(tokens) > 5000 else tokens
+                    ngrams = Counter()
+                    for i in range(len(sample_toks) - NGRAM_N + 1):
+                        ngrams[tuple(sample_toks[i:i + NGRAM_N])] += 1
+                    if ngrams and ngrams.most_common(1)[0][1] >= NGRAM_THRESH:
+                        dropped_rule4 += 1
+                        continue
+
+                kept += 1
+                bytes_kept += len(text)
+
+                if total % 100000 == 0:
+                    print(f"  [{total}] kept={kept} r1={dropped_rule1} r2={dropped_rule2} "
+                          f"r4={dropped_rule4} r5={dropped_rule5} r3mod={modified_rule3}", flush=True)
+
+    report = {
+        "total": total,
+        "kept": kept,
+        "kept_pct": round(kept / total * 100, 2),
+        "dropped_rule1_basename_drop": dropped_rule1,
+        "dropped_rule2_basename_cap": dropped_rule2,
+        "dropped_rule4_ngram_repeat": dropped_rule4,
+        "dropped_rule5_line_repeat": dropped_rule5,
+        "rule3_in_place_collapsed_samples": modified_rule3,
+        "bytes_total": bytes_total,
+        "bytes_kept": bytes_kept,
+        "bytes_kept_pct": round(bytes_kept / bytes_total * 100, 2),
+        "top_dropped_basenames": drop_by_basename.most_common(40),
+        "thresholds": {
+            "CAP": CAP,
+            "NGRAM_N": NGRAM_N,
+            "NGRAM_THRESH": NGRAM_THRESH,
+            "LINE_REP_THRESH": LINE_REP_THRESH,
+            "LINE_MAX_LEN": LINE_MAX_LEN,
+        },
+    }
+    REPORT.write_text(json.dumps(report, ensure_ascii=False, indent=2))
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

code/pretrain_data_prep/reports/code_messages_pre_train_filtered_report.json

@@ -0,0 +1,191 @@
+{
+  "src_dir": "/raid/data/weifeng/Datasets/filtered",
+  "dst": "/raid/data/weifeng/Datasets/code_messages_pre_train_filtered.jsonl",
+  "seed": 0,
+  "n_total_input": 1656628,
+  "n_empty_dropped": 0,
+  "n_bad_rows_dropped": 0,
+  "bad_rows_by_file": {},
+  "n_kept": 1656628,
+  "per_domain_kept": {
+    "Human": 241767,
+    "System_biology": 113218,
+    "Potential": 112912,
+    "Folding": 80374,
+    "3D": 69104,
+    "Activation": 68866,
+    "Networks": 67309,
+    "Single_cell": 54945,
+    "Generate": 47990,
+    "Imaging": 46115,
+    "Ligand": 41856,
+    "Protein": 39974,
+    "Bioinformatics": 36252,
+    "Gene_editing": 29170,
+    "Biomedical": 24735,
+    "Next_generation_sequencing": 24553,
+    "Quantum_mechanics": 24443,
+    "Score": 23368,
+    "Sequencing": 21806,
+    "Biologics": 20203,
+    "Modeling": 18449,
+    "Computational_Materials": 17899,
+    "Computational_Chemistry": 15152,
+    "Heterogeneity": 15150,
+    "Cycle": 14545,
+    "Integration": 14539,
+    "Mechanism": 14384,
+    "GAN": 13513,
+    "Molecular_biology": 13031,
+    "Kinetics": 12501,
+    "Ion": 12365,
+    "Diagnostics": 12150,
+    "Stochastic_modeling": 11911,
+    "Pharmacometrics": 11079,
+    "Design": 10790,
+    "Genotype": 10392,
+    "Screening": 10379,
+    "Genomics": 9262,
+    "DFT": 8520,
+    "Antibody": 7986,
+    "RNA": 7886,
+    "Proteomics": 7827,
+    "Coupled_cluster": 7771,
+    "Gene": 6981,
+    "High_throughput": 6934,
+    "Transformer": 6788,
+    "Dynamics": 6577,
+    "Protein_Protein_Interactions": 5893,
+    "Profiling": 5645,
+    "Cheminformatics": 5497,
+    "Chemistry": 5483,
+    "Diffusion": 5159,
+    "Crystal": 5120,
+    "Biology": 4777,
+    "Material": 4718,
+    "Evolution": 4649,
+    "Physiology": 4624,
+    "Hartree_Fock": 4540,
+    "Autoregressive": 4413,
+    "Electronic_structure": 3949,
+    "Flow_matching": 3815,
+    "Ab_initio": 3757,
+    "Lipid": 3718,
+    "Finite_element_method": 3697,
+    "Biotechnology": 3410,
+    "Phenotype": 3278,
+    "Monte_Carlo": 3244,
+    "Compartmental_model": 2834,
+    "Agent_based_model": 2744,
+    "Chemical_space": 2297,
+    "Microbial": 2198,
+    "Bio": 2191,
+    "Translation": 2157,
+    "Pathology": 2135,
+    "Metabolomics": 2090,
+    "Spectrometry": 2053,
+    "Spatial_Transcriptomics": 1951,
+    "Drug_repurpose": 1931,
+    "Pathway": 1798,
+    "De_novo": 1768,
+    "Reinforcement_learning": 1731,
+    "Drug": 1702,
+    "Immunology": 1676,
+    "Absorption": 1659,
+    "2D": 1573,
+    "Redox": 1537,
+    "Anomaly_detection": 1452,
+    "Pharmacology": 1374,
+    "Spatial_biology": 1357,
+    "Reproduction_number": 1336,
+    "First_principles_based_DFT": 1300,
+    "Antigen": 1293,
+    "DNA": 1277,
+    "AMR": 1124,
+    "Signaling": 1078,
+    "QM_MM": 974,
+    "Hydrophobic": 937,
+    "Prediction": 903,
+    "Free_energy_perturbation": 819,
+    "Pharmacokinetics": 777,
+    "Reaction": 751,
+    "Pharmaceutics": 742,
+    "Quantum_biology": 728,
+    "Multi_omics": 721,
+    "Lead_optimization": 719,
+    "Enzyme": 693,
+    "Metagenomics": 660,
+    "Generative": 653,
+    "Medicine": 617,
+    "Porous": 611,
+    "Rosettafold": 573,
+    "Phase_field_technique": 517,
+    "Transcriptomics": 459,
+    "Docking": 449,
+    "Biomarker": 439,
+    "Allele": 431,
+    "Discovery": 429,
+    "Microbiology": 411,
+    "Marker": 402,
+    "Lead_discovery": 390,
+    "Pharmacogenetics": 387,
+    "Pandemic": 386,
+    "Variational_autoencoders": 377,
+    "Transcription": 346,
+    "Nucleic_acids": 345,
+    "Pathogens": 314,
+    "Neurology": 313,
+    "Multi_scale_modeling": 310,
+    "Sensors": 301,
+    "CRISPR": 280,
+    "Glycomics": 270,
+    "Disease": 220,
+    "Quantum_Chemistry": 219,
+    "Substrate": 217,
+    "Assay": 201,
+    "Viral": 201,
+    "Biophysics": 200,
+    "VAE": 199,
+    "Omics": 198,
+    "Conformation": 194,
+    "Bio_foundation_model": 176,
+    "Ensemble": 170,
+    "Cell_biology": 169,
+    "Metabolic": 157,
+    "In_Silico": 150,
+    "Metamaterial": 144,
+    "Computational_Biochemistry": 137,
+    "Personalized": 137,
+    "Bioengineering": 132,
+    "AlphaFold": 99,
+    "Explainable_AI": 99,
+    "Genesis": 99,
+    "Nowcasting": 97,
+    "QSAR": 96,
+    "Pharmacogenomics": 93,
+    "Antagonist": 88,
+    "Codon": 76,
+    "ADMET": 75,
+    "Epigenetics": 71,
+    "Conjugate": 66,
+    "Toxicology": 43,
+    "RNN": 34,
+    "Vaccine": 33,
+    "PBPK_model": 31,
+    "Hydrophilic": 26,
+    "Protein_protein": 23,
+    "Hit_to_lead": 21,
+    "Homo_Lumo": 19,
+    "Homeostasis": 17,
+    "Receptor": 13,
+    "Biosensors": 9,
+    "Inhibition": 8,
+    "Organoid": 7,
+    "Lipidomics": 4,
+    "Biochemistry": 2,
+    "Digital_twin": 1,
+    "Cell_atlas": 0,
+    "Pseudotime_analysis": 0
+  },
+  "total_bytes": 9992235548
+}

code/pretrain_data_prep/reports/code_messages_pre_train_filtered_stages_report.json

@@ -0,0 +1,19 @@
+{
+  "src": "/raid/data/weifeng/Datasets/code_messages_pre_train_filtered.jsonl",
+  "n_total": 1656628,
+  "stage1_path": "/raid/data/weifeng/Datasets/code_messages_pre_train_filtered_stage1.jsonl",
+  "stage1_count": 902115,
+  "stage1_pct": 0.5445489270976949,
+  "stage2_path": "/raid/data/weifeng/Datasets/code_messages_pre_train_filtered_stage2.jsonl",
+  "stage2_count": 605249,
+  "stage2_pct": 0.3653499759752944,
+  "stage3_path": "/raid/data/weifeng/Datasets/code_messages_pre_train_filtered_stage3.jsonl",
+  "stage3_count": 149264,
+  "stage3_pct": 0.09010109692701077,
+  "thresholds": [
+    1024,
+    4096
+  ],
+  "tokenizer": "Qwen/Qwen2.5-Coder-0.5B",
+  "elapsed_min": 10.5
+}

code/pretrain_data_prep/reports/dryrun_filter_report.json

@@ -0,0 +1,90 @@
+{
+  "total": 1656628,
+  "kept": 1046224,
+  "kept_pct": 63.15,
+  "dropped_rule1_basename_drop": 8994,
+  "dropped_rule2_basename_cap": 66713,
+  "dropped_rule4_ngram_repeat": 275296,
+  "dropped_rule5_line_repeat": 259401,
+  "rule3_in_place_collapsed_samples": 78338,
+  "bytes_total": 9490697379,
+  "bytes_kept": 3693108225,
+  "bytes_kept_pct": 38.91,
+  "top_dropped_basenames": [
+    [
+      "__init__.py",
+      41168
+    ],
+    [
+      "utils.py",
+      10610
+    ],
+    [
+      "setup.py",
+      6890
+    ],
+    [
+      "index.md",
+      2856
+    ],
+    [
+      "config.py",
+      2122
+    ],
+    [
+      "conftest.py",
+      1905
+    ],
+    [
+      "CHANGELOG.md",
+      1790
+    ],
+    [
+      "conf.py",
+      1781
+    ],
+    [
+      "CONTRIBUTING.md",
+      1552
+    ],
+    [
+      "__main__.py",
+      1076
+    ],
+    [
+      "constants.py",
+      884
+    ],
+    [
+      "NEWS.md",
+      878
+    ],
+    [
+      "package-info.java",
+      609
+    ],
+    [
+      "version.py",
+      561
+    ],
+    [
+      "SKILL.md",
+      497
+    ],
+    [
+      "settings.py",
+      277
+    ],
+    [
+      "_version.py",
+      251
+    ]
+  ],
+  "thresholds": {
+    "CAP": 200,
+    "NGRAM_N": 5,
+    "NGRAM_THRESH": 30,
+    "LINE_REP_THRESH": 20,
+    "LINE_MAX_LEN": 80
+  }
+}

code/pretrain_data_prep/split_filtered_by_token_length.py

@@ -0,0 +1,86 @@
+"""Split code_messages_pre_train_filtered.jsonl into 3 token-length stages.
+
+Stage 1: <=1024 tokens
+Stage 2: 1025-4096
+Stage 3: >4096
+
+Tokenizer: Qwen2.5-Coder-0.5B.
+Uses multiprocessing for speed.
+"""
+from __future__ import annotations
+
+import json
+import multiprocessing as mp
+from pathlib import Path
+
+SRC = Path("/raid/data/weifeng/Datasets/code_messages_pre_train_filtered.jsonl")
+DST_DIR = Path("/raid/data/weifeng/Datasets")
+DST_S1 = DST_DIR / "code_messages_pre_train_filtered_stage1.jsonl"   # <=1024
+DST_S2 = DST_DIR / "code_messages_pre_train_filtered_stage2.jsonl"   # 1025-4096
+DST_S3 = DST_DIR / "code_messages_pre_train_filtered_stage3.jsonl"   # >4096
+REPORT = DST_DIR / "code_messages_pre_train_filtered_stages_report.json"
+
+T1 = 1024
+T2 = 4096
+N_WORKERS = 8
+CHUNK = 1000
+
+# Worker setup
+_tokenizer = None
+def init_worker():
+    global _tokenizer
+    from transformers import AutoTokenizer
+    _tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-Coder-0.5B", trust_remote_code=True)
+
+def count_tokens(line: str) -> tuple[int, str]:
+    """Return (n_tokens, original_line) so we can keep the line."""
+    o = json.loads(line)
+    text = o["messages"][0]["content"]
+    n = len(_tokenizer.encode(text, add_special_tokens=False))
+    return n, line
+
+
+def main() -> None:
+    # Read all lines first (cheap, ~10GB)
+    print(f"loading {SRC} ...")
+    with open(SRC, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+    print(f"loaded {len(lines):,} lines")
+
+    out_s1 = open(DST_S1, "w", encoding="utf-8")
+    out_s2 = open(DST_S2, "w", encoding="utf-8")
+    out_s3 = open(DST_S3, "w", encoding="utf-8")
+
+    n_s1 = n_s2 = n_s3 = 0
+    import time
+    t0 = time.time()
+    with mp.Pool(N_WORKERS, initializer=init_worker) as pool:
+        for i, (n_tok, line) in enumerate(pool.imap_unordered(count_tokens, lines, chunksize=CHUNK)):
+            if n_tok <= T1:
+                out_s1.write(line); n_s1 += 1
+            elif n_tok <= T2:
+                out_s2.write(line); n_s2 += 1
+            else:
+                out_s3.write(line); n_s3 += 1
+            if (i + 1) % 50000 == 0:
+                elapsed = time.time() - t0
+                rate = (i + 1) / max(elapsed, 1e-9)
+                eta = (len(lines) - i - 1) / max(rate, 1e-9)
+                print(f"  [{i+1:>8,}/{len(lines):,}]  s1={n_s1:,}  s2={n_s2:,}  s3={n_s3:,}  rate={rate:.0f}/s  eta={eta/60:.1f}min", flush=True)
+
+    out_s1.close(); out_s2.close(); out_s3.close()
+    report = {
+        "src": str(SRC), "n_total": len(lines),
+        "stage1_path": str(DST_S1), "stage1_count": n_s1, "stage1_pct": n_s1/len(lines),
+        "stage2_path": str(DST_S2), "stage2_count": n_s2, "stage2_pct": n_s2/len(lines),
+        "stage3_path": str(DST_S3), "stage3_count": n_s3, "stage3_pct": n_s3/len(lines),
+        "thresholds": [T1, T2],
+        "tokenizer": "Qwen/Qwen2.5-Coder-0.5B",
+        "elapsed_min": round((time.time() - t0)/60, 1),
+    }
+    REPORT.write_text(json.dumps(report, indent=2, ensure_ascii=False))
+    print(json.dumps(report, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

data/f3/intermediate/F3_stage1.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:167d17cd03fbbdbca40e3ef289c81898120107ba13f975db2ce23e3d6516c1a3
+size 2276490009

data/f3/intermediate/F3_stage2.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97353d440b3b0d292ff443cf2e705140ff5fa25ebbc2a65163d240b1eff03432
+size 1956575594

data/f3/intermediate/F3_stage3.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c54871e7896bb83e49bed1f0fc07cef2021a8c7ddb5053c1f2c7d5eed8ddcb56
+size 1935884904

environment/KNOWN_LIMITATIONS.md

@@ -0,0 +1,54 @@
+# Environment Record and Known Reproducibility Limitations
+
+## Best Available Training-Environment Evidence
+
+The only nearby training run for which a complete environment snapshot was captured was a
+text-only continued-pre-training run on 13 April 2026. It used the following key versions:
+
+| Component | Version |
+|---|---|
+| ms-swift | 4.0.2 |
+| PyTorch | 2.5.1+cu121 |
+| Transformers | 5.3.0 |
+| Accelerate | 1.13.0 |
+| Datasets | 3.6.0 |
+| PEFT | 0.18.1 |
+| DeepSpeed | 0.18.8 |
+| FlashAttention | 2.8.3 (CUDA 12 / PyTorch 2.5 / CPython 3.11 wheel) |
+| CUDA / driver | 12.2 / 535.183.06 |
+
+This environment is contextual evidence only. It is not a run-specific snapshot for either
+checkpoint in this package.
+
+## Known Gap
+
+The following delivered runs were launched directly and do not have their own `pip freeze`
+or complete system snapshot:
+
+- `models/pretrained/stage1_cpt` (14–15 May 2026)
+- `models/sft/sft_f3_refined_instruct` (8 May 2026)
+
+Each checkpoint includes `args.json`, which records the effective ms-swift training
+configuration, and `trainer_state.json`, which records the observed steps and loss history.
+The exact datasets and historical launch scripts are also included. These artefacts support
+procedural reproduction, but not a claim of bit-for-bit numerical reproduction.
+
+The April environment is the best known approximation because the training container was
+not intentionally changed between the April and May runs. This is an inference rather than
+a contemporaneous record and must not be presented as confirmed run-specific evidence.
+
+## Delivery Validation Environment
+
+Both delivered checkpoints were loaded and executed successfully using:
+
+| Component | Version |
+|---|---|
+| Python | 3.12 |
+| PyTorch | 2.10.0+cu128 |
+| Transformers | 5.7.0 |
+| Safetensors | 0.7.0 |
+
+`tools/smoke_test.py` confirmed that both checkpoints produce finite, non-degenerate logits.
+The weights can therefore be loaded with the newer delivery environment. Rerunning the
+historical training scripts requires a compatible ms-swift environment; version 4.0.2 is
+the best available reference.

evidence/eval/f3_refined_9p1_summary.json

@@ -0,0 +1,27 @@
+{
+  "dataset": "/raid/data/weifeng/Runable_Benchmark/dataset/alignment_humaneval_runnable_benchmark_reviewed_primary.jsonl",
+  "provider": "hf",
+  "model": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/checkpoint-6435",
+  "num_tasks": 200,
+  "samples_per_task": 5,
+  "pass@1": 0.09100000000000001,
+  "pass@5": 0.13,
+  "reasons": {
+    "test_runtime_error": 721,
+    "ok": 91,
+    "solution_syntax_error": 105,
+    "timeout": 2,
+    "solution_exec_error": 62,
+    "entry_point_not_found": 19
+  },
+  "elapsed_sec": 2452.127,
+  "attempts_file": "/raid/data/weifeng/Runable_Benchmark/outputs/attempts_hf__raid_data_weifeng_Pre_train_output_user_qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu_v0-20260508-164342_checkpoint-6435_20260508_191037.jsonl",
+  "generation_limit": {
+    "reasoning_model": false,
+    "max_new_tokens": 2048,
+    "batch_size": 5,
+    "temperature": 0.8
+  },
+  "num_attempts": 1000,
+  "resumed": true
+}

evidence/instruction_recipe_results.md

@@ -0,0 +1,43 @@
+# Instruction Tuning Recipe Results
+
+Evaluation protocol for the main HF results:
+- Dataset: runnable alignment benchmark, 200 tasks
+- Samples per task: 5
+- Attempts: 1000
+- Metric: pass@1 / pass@5
+
+## Same Instruct Initialization: F3 / F4 Data Recipe
+
+| Setup | Data Recipe | Approx. Train Samples | pass@1 | pass@5 | Notes |
+|---|---|---:|---:|---:|---|
+| Qwen2.5-Coder-0.5B-Instruct + SFT | F3 clean | 473K | 9.1% | 13.0% | Strongest F3 baseline |
+| Qwen2.5-Coder-0.5B-Instruct + SFT | F3 refined | 207K | 9.1% | 13.0% | Similar performance with fewer samples |
+| Qwen2.5-Coder-0.5B-Instruct + SFT | F3 high-quality | 135K | 8.3% | 13.0% | Slight pass@1 drop after stronger filtering |
+| Qwen2.5-Coder-0.5B-Instruct + SFT | F3 + F4 clean | 150K | 5.7% | 11.5% | Mixed recipe underperforms F3-only |
+| Qwen2.5-Coder-0.5B-Instruct + SFT | F4 clean | 17.5K | 0.9% | 2.5% | Too small / narrow for this benchmark |
+
+## Evol / Sequential Instruction Tuning
+
+| Setup | Initialization | Data Recipe | Approx. Train Samples | pass@1 | pass@5 | Notes |
+|---|---|---|---:|---:|---:|---|
+| Stage2 CPT + SFT | Stage2 CPT | Evol + self-OSS | 80.7K | 1.7% | 5.5% | General instruction data alone is weak for the target benchmark |
+| Stage2 CPT + SFT + SFT | Stage2 CPT + Evol/self-OSS SFT | F3 refined | 207K | 1.9% | 6.5% | F3 recovery is limited after weak Evol-stage alignment |
+| Magicoder-style SFT | Qwen2.5-Coder Base | F3 refined, 3 epochs | 207K | 0.6% | 2.5% | F3 alone from base is weak under this setup |
+| Magicoder-style SFT | Qwen2.5-Coder Base | Evol, 3 epochs | 30K | 1.7% | 5.0% | Evol improves over F3-only base setup |
+| Magicoder-style SFT | Qwen2.5-Coder Base + Evol SFT | F3 refined, 1 epoch | 207K | 4.1% | 10.5% | Sequential Evol -> F3 gives the best result in this group |
+
+## Initialization Reference
+
+| Setup | Initialization | Data Recipe | pass@1 | pass@5 | Notes |
+|---|---|---|---:|---:|---|
+| Base + SFT | Qwen2.5-Coder-0.5B Base | F3 high-quality | 0.4% | 2.0% | Raw base lacks instruction-following prior |
+| CPT + SFT | Continued-pretrained checkpoint | F3 clean | 2.8% | 8.5% | CPT helps but does not match instruct initialization |
+| Stage1 CPT + SFT | Stage1 CPT | F3 high-quality | 4.9% | 10.5% | Stage1 CPT is better than raw base |
+| Instruct + SFT | Qwen2.5-Coder-0.5B-Instruct | F3 high-quality | 8.3% | 13.0% | Instruct initialization remains strongest |
+
+## PPT Takeaways
+
+1. F3-style instruction data is the most reliable recipe for the runnable benchmark.
+2. Stronger cleaning can reduce data size substantially with only small pass@1 loss, but overly small F4-style data loses coverage.
+3. Evol-style general instruction data helps in sequential settings, especially Evol -> F3, but Evol alone does not solve the target benchmark.
+4. Model initialization is a major confound: instruction-aligned initialization is much stronger than raw base initialization.

evidence/smoke_test_results.json

@@ -0,0 +1,38 @@
+[
+  {
+    "model": "pretrained/stage1_cpt",
+    "path": "models/pretrained/stage1_cpt",
+    "ok": true,
+    "architecture": "Qwen2ForCausalLM",
+    "hidden_size": 896,
+    "layers": 24,
+    "vocab_size": 151936,
+    "tokenizer": "Qwen2Tokenizer",
+    "parameters_million": 494.0,
+    "logits_shape": [
+      1,
+      14,
+      151936
+    ],
+    "finite": true,
+    "top1_zscore": 13.02
+  },
+  {
+    "model": "sft/sft_f3_refined_instruct",
+    "path": "models/sft/sft_f3_refined_instruct",
+    "ok": true,
+    "architecture": "Qwen2ForCausalLM",
+    "hidden_size": 896,
+    "layers": 24,
+    "vocab_size": 151936,
+    "tokenizer": "Qwen2Tokenizer",
+    "parameters_million": 494.0,
+    "logits_shape": [
+      1,
+      14,
+      151936
+    ],
+    "finite": true,
+    "top1_zscore": 12.84
+  }
+]

models/pretrained/stage1_cpt/args.json

@@ -0,0 +1,344 @@
+{
+  "output_dir": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114",
+  "per_device_train_batch_size": 8,
+  "num_train_epochs": 1.0,
+  "max_steps": -1,
+  "learning_rate": 5e-05,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_steps": 0,
+  "optim": "adamw_torch_fused",
+  "optim_args": null,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "optim_target_modules": null,
+  "gradient_accumulation_steps": 4,
+  "average_tokens_across_devices": true,
+  "max_grad_norm": 1.0,
+  "label_smoothing_factor": 0.0,
+  "bf16": true,
+  "fp16": false,
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "use_cache": false,
+  "neftune_noise_alpha": null,
+  "torch_empty_cache_steps": null,
+  "auto_find_batch_size": false,
+  "logging_strategy": "steps",
+  "logging_steps": 50,
+  "logging_first_step": true,
+  "log_on_each_node": true,
+  "logging_nan_inf_filter": true,
+  "include_num_input_tokens_seen": false,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "disable_tqdm": null,
+  "report_to": [
+    "tensorboard"
+  ],
+  "run_name": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114",
+  "project": "huggingface",
+  "trackio_space_id": null,
+  "trackio_bucket_id": null,
+  "trackio_static_space_id": null,
+  "eval_strategy": "no",
+  "eval_steps": 2000.0,
+  "eval_delay": 0,
+  "per_device_eval_batch_size": 1,
+  "prediction_loss_only": false,
+  "eval_on_start": false,
+  "eval_do_concat_batches": true,
+  "eval_use_gather_object": false,
+  "eval_accumulation_steps": null,
+  "include_for_metrics": [],
+  "batch_eval_metrics": false,
+  "save_only_model": false,
+  "save_strategy": "steps",
+  "save_steps": 2000.0,
+  "save_on_each_node": false,
+  "save_total_limit": 3,
+  "enable_jit_checkpoint": false,
+  "push_to_hub": false,
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_always_push": false,
+  "hub_revision": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "restore_callback_states_from_checkpoint": false,
+  "full_determinism": false,
+  "seed": 42,
+  "data_seed": 42,
+  "use_cpu": false,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "parallelism_config": null,
+  "dataloader_drop_last": false,
+  "dataloader_num_workers": 8,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "dataloader_prefetch_factor": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "train_sampling_strategy": "random",
+  "length_column_name": "length",
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "ddp_static_graph": null,
+  "ddp_backend": null,
+  "ddp_timeout": 18000000,
+  "fsdp": [],
+  "fsdp_config": null,
+  "deepspeed": null,
+  "debug": null,
+  "skip_memory_metrics": true,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "resume_from_checkpoint": null,
+  "warmup_ratio": 0.03,
+  "logging_dir": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114/runs",
+  "local_rank": 0,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": false,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "safe_serialization": true,
+  "max_shard_size": "5GB",
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "group_by_length": false,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "eval_metric": null,
+  "callbacks": [],
+  "early_stop_interval": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "tuner_type": "full",
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "use_flash_ckpt": false,
+  "use_ray": false,
+  "ray_exp_name": null,
+  "device_groups": null,
+  "model": "Qwen/Qwen2.5-Coder-0.5B",
+  "model_type": "qwen2",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": "flash_attn",
+  "experts_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5",
+  "system": null,
+  "max_length": 1024,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": false,
+  "padding_side": "right",
+  "padding_free": false,
+  "loss_scale": "all",
+  "sequence_parallel_size": 1,
+  "template_backend": "swift",
+  "response_prefix": null,
+  "enable_thinking": null,
+  "add_non_thinking_prefix": true,
+  "dataset": [
+    "/raid/data/weifeng/Datasets/code_messages_pre_train_filtered_stage1.jsonl"
+  ],
+  "val_dataset": [],
+  "cached_dataset": [],
+  "cached_val_dataset": [],
+  "split_dataset_ratio": 0.0,
+  "dataset_num_proc": 8,
+  "load_from_cache_file": false,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "structured_outputs_regex": null,
+  "adapters": [],
+  "external_plugins": [],
+  "custom_register_path": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "packing_num_proc": 1,
+  "lazy_tokenize": false,
+  "use_hf": true,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": "ms-swift",
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_notification_method": null,
+  "swanlab_webhook_url": null,
+  "swanlab_secret": null,
+  "swanlab_sender_email": null,
+  "swanlab_receiver_email": null,
+  "swanlab_smtp_server": null,
+  "swanlab_smtp_port": null,
+  "swanlab_email_language": "zh",
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "swift_version": "4.1.3",
+  "ckpt_dir": null,
+  "rank": 0,
+  "global_world_size": 1,
+  "local_world_size": 1,
+  "model_suffix": "Qwen2.5-Coder-0.5B",
+  "model_info": "ModelInfo(model_type='qwen2', model_dir='/home/weifeng/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-0.5B/snapshots/8123ea2e9354afb7ffcc6c8641d1b2f5ecf18301', torch_dtype=torch.bfloat16, max_model_len=32768, quant_method=None, quant_bits=None, rope_scaling={'rope_theta': 1000000.0, 'rope_type': 'default'}, is_moe_model=False, is_multimodal=False, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat', hf_model_id='Qwen/Qwen1.5-0.5B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat', hf_model_id='Qwen/Qwen1.5-1.8B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat', hf_model_id='Qwen/Qwen1.5-4B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat', hf_model_id='Qwen/Qwen1.5-7B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat', hf_model_id='Qwen/Qwen1.5-14B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat', hf_model_id='Qwen/Qwen1.5-32B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat', hf_model_id='Qwen/Qwen1.5-72B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat', hf_model_id='Qwen/Qwen1.5-110B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B', hf_model_id='Qwen/Qwen1.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B', hf_model_id='Qwen/Qwen1.5-1.8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B', hf_model_id='Qwen/Qwen1.5-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B', hf_model_id='Qwen/Qwen1.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B', hf_model_id='Qwen/Qwen1.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B', hf_model_id='Qwen/Qwen1.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B', hf_model_id='Qwen/Qwen1.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B', hf_model_id='Qwen/Qwen1.5-110B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-4B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-7B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-14B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-32B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-72B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-110B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/CodeQwen1.5-7B', hf_model_id='Qwen/CodeQwen1.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/CodeQwen1.5-7B-Chat', hf_model_id='Qwen/CodeQwen1.5-7B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/CodeQwen1.5-7B-Chat-AWQ', hf_model_id='Qwen/CodeQwen1.5-7B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct', hf_model_id='Qwen/Qwen2-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct', hf_model_id='Qwen/Qwen2-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct', hf_model_id='Qwen/Qwen2-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct', hf_model_id='Qwen/Qwen2-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B', hf_model_id='Qwen/Qwen2-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B', hf_model_id='Qwen/Qwen2-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B', hf_model_id='Qwen/Qwen2-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B', hf_model_id='Qwen/Qwen2-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2-Math-1.5B-Instruct', hf_model_id='Qwen/Qwen2-Math-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-7B-Instruct', hf_model_id='Qwen/Qwen2-Math-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-72B-Instruct', hf_model_id='Qwen/Qwen2-Math-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-1.5B', hf_model_id='Qwen/Qwen2-Math-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-7B', hf_model_id='Qwen/Qwen2-Math-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-72B', hf_model_id='Qwen/Qwen2-Math-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=['math']), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-1M', hf_model_id='Qwen/Qwen2.5-7B-Instruct-1M', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-1M', hf_model_id='Qwen/Qwen2.5-14B-Instruct-1M', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='PowerInfer/SmallThinker-3B-Preview', hf_model_id='PowerInfer/SmallThinker-3B-Preview', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='moonshotai/Kimi-Dev-72B', hf_model_id='moonshotai/Kimi-Dev-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Math-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-72B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-1.5B', hf_model_id='Qwen/Qwen2.5-Math-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-7B', hf_model_id='Qwen/Qwen2.5-Math-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-72B', hf_model_id='Qwen/Qwen2.5-Math-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5_math', ignore_patterns=None, requires=None, tags=['math']), ModelGroup(models=[Model(ms_model_id='AIDC-AI/Marco-o1', hf_model_id='AIDC-AI/Marco-o1', model_path=None, ms_revision=None, hf_revision=None)], template='marco_o1', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/QwQ-32B-Preview', hf_model_id='Qwen/QwQ-32B-Preview', model_path=None, ms_revision=None, hf_revision=None)], template='qwq_preview', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/QwQ-32B', hf_model_id='Qwen/QwQ-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/QwQ-32B-AWQ', hf_model_id='Qwen/QwQ-32B-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwq', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='iic/QwenLong-L1-32B', hf_model_id='Tongyi-Zhiwen/QwenLong-L1-32B', model_path=None, ms_revision=None, hf_revision=None)], template='deepseek_r1', ignore_patterns=None, requires=None, tags=[])], loader=<class 'swift.model.register.ModelLoader'>, template='qwen2_5', model_arch=ModelKeys(arch_name='llama', embedding='model.embed_tokens', module_list='model.layers', lm_head='lm_head', q_proj='model.layers.{}.self_attn.q_proj', k_proj='model.layers.{}.self_attn.k_proj', v_proj='model.layers.{}.self_attn.v_proj', o_proj='model.layers.{}.self_attn.o_proj', attention='model.layers.{}.self_attn', mlp='model.layers.{}.mlp', down_proj='model.layers.{}.mlp.down_proj', qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None), mcore_model_type=None, architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37'], tags=['coding'])",
+  "model_dir": "/home/weifeng/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-0.5B/snapshots/8123ea2e9354afb7ffcc6c8641d1b2f5ecf18301",
+  "template_meta": "Qwen2_5TemplateMeta(template_type='qwen2_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=<class 'swift.template.base.Template'>, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system='You are Qwen, created by Alibaba Cloud. You are a helpful assistant.', auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='', non_thinking_prefix='', history_thinking_prefix='')",
+  "_val_dataset_exists": false,
+  "hub": "<class 'swift.hub.hub.HFHub'>",
+  "evaluation_strategy": "steps",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114', per_device_train_batch_size=8, num_train_epochs=1.0, max_steps=-1, learning_rate=5e-05, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_steps=0.03, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, optim_target_modules=None, gradient_accumulation_steps=4, average_tokens_across_devices=None, max_grad_norm=1.0, label_smoothing_factor=0.0, bf16=True, fp16=False, bf16_full_eval=False, fp16_full_eval=False, tf32=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, use_liger_kernel=False, liger_kernel_config=None, use_cache=False, neftune_noise_alpha=None, torch_empty_cache_steps=None, auto_find_batch_size=False, logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_steps=50, logging_first_step=True, log_on_each_node=True, logging_nan_inf_filter=True, include_num_input_tokens_seen=None, log_level='passive', log_level_replica='warning', disable_tqdm=False, report_to=['tensorboard'], run_name='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114', project='huggingface', trackio_space_id=None, trackio_bucket_id=None, trackio_static_space_id=None, eval_strategy=<IntervalStrategy.NO: 'no'>, eval_steps=2000.0, eval_delay=0, per_device_eval_batch_size=1, prediction_loss_only=False, eval_on_start=False, eval_do_concat_batches=True, eval_use_gather_object=False, eval_accumulation_steps=None, include_for_metrics=[], batch_eval_metrics=False, save_only_model=False, save_strategy=<SaveStrategy.STEPS: 'steps'>, save_steps=2000, save_on_each_node=False, save_total_limit=3, enable_jit_checkpoint=False, push_to_hub=False, hub_token=None, hub_private_repo=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_always_push=False, hub_revision=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, restore_callback_states_from_checkpoint=False, full_determinism=False, seed=42, data_seed=42, use_cpu=False, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, dataloader_drop_last=False, dataloader_num_workers=8, dataloader_pin_memory=True, dataloader_persistent_workers=False, dataloader_prefetch_factor=2, remove_unused_columns=False, label_names=None, train_sampling_strategy='random', length_column_name='length', ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, ddp_static_graph=None, ddp_backend=None, ddp_timeout=18000000, fsdp=[], fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, deepspeed=None, debug=[], skip_memory_metrics=True, do_train=False, do_eval=False, do_predict=False, resume_from_checkpoint=None, warmup_ratio=0.03, logging_dir='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114/runs', local_rank=0, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=False, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, safe_serialization=True, max_shard_size='5GB', check_model=True, acc_strategy='token', train_dataloader_shuffle=True, group_by_length=False, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, eval_metric=None, callbacks=[], early_stop_interval=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, tuner_type='full', use_galore=False, galore_target_modules=None, galore_rank=128, galore_update_proj_gap=50, galore_scale=1.0, galore_proj_type='std', galore_optim_per_parameter=False, galore_with_embedding=False, galore_quantization=False, galore_proj_quant=False, galore_proj_bits=4, galore_proj_group_size=256, galore_cos_threshold=0.4, galore_gamma_proj=2, galore_queue_size=5, lisa_activated_layers=0, lisa_step_interval=20, use_flash_ckpt=False)"
+}

models/pretrained/stage1_cpt/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

models/pretrained/stage1_cpt/config.json

@@ -0,0 +1,57 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 1000000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.6.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

models/pretrained/stage1_cpt/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": false,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "temperature": 1.0,
+  "top_k": 50,
+  "top_p": 1.0,
+  "transformers_version": "5.6.2"
+}

models/pretrained/stage1_cpt/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

models/sft/sft_f3_refined_instruct/args.json

@@ -0,0 +1,344 @@
+{
+  "output_dir": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342",
+  "per_device_train_batch_size": 4,
+  "num_train_epochs": 1.0,
+  "max_steps": -1,
+  "learning_rate": 2e-05,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_steps": 0,
+  "optim": "adamw_torch_fused",
+  "optim_args": null,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "optim_target_modules": null,
+  "gradient_accumulation_steps": 8,
+  "average_tokens_across_devices": true,
+  "max_grad_norm": 1.0,
+  "label_smoothing_factor": 0.0,
+  "bf16": true,
+  "fp16": false,
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "use_cache": false,
+  "neftune_noise_alpha": null,
+  "torch_empty_cache_steps": null,
+  "auto_find_batch_size": false,
+  "logging_strategy": "steps",
+  "logging_steps": 10,
+  "logging_first_step": true,
+  "log_on_each_node": true,
+  "logging_nan_inf_filter": true,
+  "include_num_input_tokens_seen": false,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "disable_tqdm": null,
+  "report_to": [
+    "tensorboard"
+  ],
+  "run_name": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342",
+  "project": "huggingface",
+  "trackio_space_id": null,
+  "trackio_bucket_id": null,
+  "trackio_static_space_id": null,
+  "eval_strategy": "no",
+  "eval_steps": 500.0,
+  "eval_delay": 0,
+  "per_device_eval_batch_size": 1,
+  "prediction_loss_only": false,
+  "eval_on_start": false,
+  "eval_do_concat_batches": true,
+  "eval_use_gather_object": false,
+  "eval_accumulation_steps": null,
+  "include_for_metrics": [],
+  "batch_eval_metrics": false,
+  "save_only_model": false,
+  "save_strategy": "steps",
+  "save_steps": 500.0,
+  "save_on_each_node": false,
+  "save_total_limit": 5,
+  "enable_jit_checkpoint": false,
+  "push_to_hub": false,
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_always_push": false,
+  "hub_revision": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "restore_callback_states_from_checkpoint": false,
+  "full_determinism": false,
+  "seed": 42,
+  "data_seed": 42,
+  "use_cpu": false,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "parallelism_config": null,
+  "dataloader_drop_last": false,
+  "dataloader_num_workers": 8,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "dataloader_prefetch_factor": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "train_sampling_strategy": "random",
+  "length_column_name": "length",
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "ddp_static_graph": null,
+  "ddp_backend": null,
+  "ddp_timeout": 18000000,
+  "fsdp": [],
+  "fsdp_config": null,
+  "deepspeed": null,
+  "debug": null,
+  "skip_memory_metrics": true,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "resume_from_checkpoint": null,
+  "warmup_ratio": 0.03,
+  "logging_dir": "/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/runs",
+  "local_rank": 0,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": false,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "safe_serialization": true,
+  "max_shard_size": "5GB",
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "group_by_length": false,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "eval_metric": null,
+  "callbacks": [],
+  "early_stop_interval": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "tuner_type": "full",
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "use_flash_ckpt": false,
+  "use_ray": false,
+  "ray_exp_name": null,
+  "device_groups": null,
+  "model": "Qwen/Qwen2.5-Coder-0.5B-Instruct",
+  "model_type": "qwen2",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": "flash_attn",
+  "experts_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5",
+  "system": null,
+  "max_length": 4096,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_side": "right",
+  "padding_free": false,
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "template_backend": "swift",
+  "response_prefix": null,
+  "enable_thinking": null,
+  "add_non_thinking_prefix": true,
+  "dataset": [
+    "/raid/data/weifeng/Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v2.jsonl"
+  ],
+  "val_dataset": [],
+  "cached_dataset": [],
+  "cached_val_dataset": [],
+  "split_dataset_ratio": 0.0,
+  "dataset_num_proc": 8,
+  "load_from_cache_file": false,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "structured_outputs_regex": null,
+  "adapters": [],
+  "external_plugins": [],
+  "custom_register_path": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "packing_num_proc": 1,
+  "lazy_tokenize": false,
+  "use_hf": true,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": "ms-swift",
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_notification_method": null,
+  "swanlab_webhook_url": null,
+  "swanlab_secret": null,
+  "swanlab_sender_email": null,
+  "swanlab_receiver_email": null,
+  "swanlab_smtp_server": null,
+  "swanlab_smtp_port": null,
+  "swanlab_email_language": "zh",
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "swift_version": "4.1.3",
+  "ckpt_dir": null,
+  "rank": 0,
+  "global_world_size": 1,
+  "local_world_size": 1,
+  "model_suffix": "Qwen2.5-Coder-0.5B-Instruct",
+  "model_info": "ModelInfo(model_type='qwen2', model_dir='/home/weifeng/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-0.5B-Instruct/snapshots/ea3f2471cf1b1f0db85067f1ef93848e38e88c25', torch_dtype=torch.bfloat16, max_model_len=32768, quant_method=None, quant_bits=None, rope_scaling={'rope_theta': 1000000.0, 'rope_type': 'default'}, is_moe_model=False, is_multimodal=False, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat', hf_model_id='Qwen/Qwen1.5-0.5B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat', hf_model_id='Qwen/Qwen1.5-1.8B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat', hf_model_id='Qwen/Qwen1.5-4B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat', hf_model_id='Qwen/Qwen1.5-7B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat', hf_model_id='Qwen/Qwen1.5-14B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat', hf_model_id='Qwen/Qwen1.5-32B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat', hf_model_id='Qwen/Qwen1.5-72B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat', hf_model_id='Qwen/Qwen1.5-110B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B', hf_model_id='Qwen/Qwen1.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B', hf_model_id='Qwen/Qwen1.5-1.8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B', hf_model_id='Qwen/Qwen1.5-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B', hf_model_id='Qwen/Qwen1.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B', hf_model_id='Qwen/Qwen1.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B', hf_model_id='Qwen/Qwen1.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B', hf_model_id='Qwen/Qwen1.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B', hf_model_id='Qwen/Qwen1.5-110B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', hf_model_id='Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', hf_model_id='Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-0.5B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-0.5B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-1.8B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-1.8B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-4B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-4B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-7B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-7B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-14B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-14B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-32B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-32B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-72B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-72B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen1.5-110B-Chat-AWQ', hf_model_id='Qwen/Qwen1.5-110B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/CodeQwen1.5-7B', hf_model_id='Qwen/CodeQwen1.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/CodeQwen1.5-7B-Chat', hf_model_id='Qwen/CodeQwen1.5-7B-Chat', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/CodeQwen1.5-7B-Chat-AWQ', hf_model_id='Qwen/CodeQwen1.5-7B-Chat-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct', hf_model_id='Qwen/Qwen2-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct', hf_model_id='Qwen/Qwen2-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct', hf_model_id='Qwen/Qwen2-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct', hf_model_id='Qwen/Qwen2-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B', hf_model_id='Qwen/Qwen2-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B', hf_model_id='Qwen/Qwen2-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B', hf_model_id='Qwen/Qwen2-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B', hf_model_id='Qwen/Qwen2-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2-Math-1.5B-Instruct', hf_model_id='Qwen/Qwen2-Math-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-7B-Instruct', hf_model_id='Qwen/Qwen2-Math-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-72B-Instruct', hf_model_id='Qwen/Qwen2-Math-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-1.5B', hf_model_id='Qwen/Qwen2-Math-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-7B', hf_model_id='Qwen/Qwen2-Math-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2-Math-72B', hf_model_id='Qwen/Qwen2-Math-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=['math']), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-1M', hf_model_id='Qwen/Qwen2.5-7B-Instruct-1M', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-1M', hf_model_id='Qwen/Qwen2.5-14B-Instruct-1M', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='PowerInfer/SmallThinker-3B-Preview', hf_model_id='PowerInfer/SmallThinker-3B-Preview', model_path=None, ms_revision=None, hf_revision=None)], template='qwen', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='moonshotai/Kimi-Dev-72B', hf_model_id='moonshotai/Kimi-Dev-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Math-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-72B-Instruct', hf_model_id='Qwen/Qwen2.5-Math-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-1.5B', hf_model_id='Qwen/Qwen2.5-Math-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-7B', hf_model_id='Qwen/Qwen2.5-Math-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Math-72B', hf_model_id='Qwen/Qwen2.5-Math-72B', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5_math', ignore_patterns=None, requires=None, tags=['math']), ModelGroup(models=[Model(ms_model_id='AIDC-AI/Marco-o1', hf_model_id='AIDC-AI/Marco-o1', model_path=None, ms_revision=None, hf_revision=None)], template='marco_o1', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/QwQ-32B-Preview', hf_model_id='Qwen/QwQ-32B-Preview', model_path=None, ms_revision=None, hf_revision=None)], template='qwq_preview', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/QwQ-32B', hf_model_id='Qwen/QwQ-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/QwQ-32B-AWQ', hf_model_id='Qwen/QwQ-32B-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwq', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', hf_model_id='deepseek-ai/DeepSeek-R1-Distill-Qwen-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='iic/QwenLong-L1-32B', hf_model_id='Tongyi-Zhiwen/QwenLong-L1-32B', model_path=None, ms_revision=None, hf_revision=None)], template='deepseek_r1', ignore_patterns=None, requires=None, tags=[])], loader=<class 'swift.model.register.ModelLoader'>, template='qwen2_5', model_arch=ModelKeys(arch_name='llama', embedding='model.embed_tokens', module_list='model.layers', lm_head='lm_head', q_proj='model.layers.{}.self_attn.q_proj', k_proj='model.layers.{}.self_attn.k_proj', v_proj='model.layers.{}.self_attn.v_proj', o_proj='model.layers.{}.self_attn.o_proj', attention='model.layers.{}.self_attn', mlp='model.layers.{}.mlp', down_proj='model.layers.{}.mlp.down_proj', qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None), mcore_model_type=None, architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37'], tags=['coding'])",
+  "model_dir": "/home/weifeng/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-0.5B-Instruct/snapshots/ea3f2471cf1b1f0db85067f1ef93848e38e88c25",
+  "template_meta": "Qwen2_5TemplateMeta(template_type='qwen2_5', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=<class 'swift.template.base.Template'>, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system='You are Qwen, created by Alibaba Cloud. You are a helpful assistant.', auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='', non_thinking_prefix='', history_thinking_prefix='')",
+  "_val_dataset_exists": false,
+  "hub": "<class 'swift.hub.hub.HFHub'>",
+  "evaluation_strategy": "steps",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342', per_device_train_batch_size=4, num_train_epochs=1.0, max_steps=-1, learning_rate=2e-05, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_steps=0.03, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, optim_target_modules=None, gradient_accumulation_steps=8, average_tokens_across_devices=None, max_grad_norm=1.0, label_smoothing_factor=0.0, bf16=True, fp16=False, bf16_full_eval=False, fp16_full_eval=False, tf32=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, use_liger_kernel=False, liger_kernel_config=None, use_cache=False, neftune_noise_alpha=None, torch_empty_cache_steps=None, auto_find_batch_size=False, logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_steps=10, logging_first_step=True, log_on_each_node=True, logging_nan_inf_filter=True, include_num_input_tokens_seen=None, log_level='passive', log_level_replica='warning', disable_tqdm=False, report_to=['tensorboard'], run_name='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342', project='huggingface', trackio_space_id=None, trackio_bucket_id=None, trackio_static_space_id=None, eval_strategy=<IntervalStrategy.NO: 'no'>, eval_steps=500.0, eval_delay=0, per_device_eval_batch_size=1, prediction_loss_only=False, eval_on_start=False, eval_do_concat_batches=True, eval_use_gather_object=False, eval_accumulation_steps=None, include_for_metrics=[], batch_eval_metrics=False, save_only_model=False, save_strategy=<SaveStrategy.STEPS: 'steps'>, save_steps=500, save_on_each_node=False, save_total_limit=5, enable_jit_checkpoint=False, push_to_hub=False, hub_token=None, hub_private_repo=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_always_push=False, hub_revision=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, restore_callback_states_from_checkpoint=False, full_determinism=False, seed=42, data_seed=42, use_cpu=False, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, dataloader_drop_last=False, dataloader_num_workers=8, dataloader_pin_memory=True, dataloader_persistent_workers=False, dataloader_prefetch_factor=2, remove_unused_columns=False, label_names=None, train_sampling_strategy='random', length_column_name='length', ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, ddp_static_graph=None, ddp_backend=None, ddp_timeout=18000000, fsdp=[], fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, deepspeed=None, debug=[], skip_memory_metrics=True, do_train=False, do_eval=False, do_predict=False, resume_from_checkpoint=None, warmup_ratio=0.03, logging_dir='/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/runs', local_rank=0, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=False, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, safe_serialization=True, max_shard_size='5GB', check_model=True, acc_strategy='token', train_dataloader_shuffle=True, group_by_length=False, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, eval_metric=None, callbacks=[], early_stop_interval=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, tuner_type='full', use_galore=False, galore_target_modules=None, galore_rank=128, galore_update_proj_gap=50, galore_scale=1.0, galore_proj_type='std', galore_optim_per_parameter=False, galore_with_embedding=False, galore_quantization=False, galore_proj_quant=False, galore_proj_bits=4, galore_proj_group_size=256, galore_cos_threshold=0.4, galore_gamma_proj=2, galore_queue_size=5, lisa_activated_layers=0, lisa_step_interval=20, use_flash_ckpt=False)"
+}

models/sft/sft_f3_refined_instruct/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

models/sft/sft_f3_refined_instruct/config.json

@@ -0,0 +1,57 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 1000000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.6.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

models/sft/sft_f3_refined_instruct/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "5.6.2"
+}

models/sft/sft_f3_refined_instruct/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

tools/build_manifest.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Generate a provenance manifest and SHA256SUMS for this delivery directory."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+WORKSPACE = Path(os.environ.get("DELIVERY_SOURCE_ROOT", "/raid/data/weifeng"))
+
+SOURCE_PREFIXES = {
+    "models/pretrained/stage1_cpt": WORKSPACE / "Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1/v1-20260514-232114/checkpoint-28175",
+    "models/sft/sft_f3_refined_instruct": WORKSPACE / "Pre_train/output_user/qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu/v0-20260508-164342/checkpoint-6435",
+    "code/f3_filtering": WORKSPACE / "Datasets/Instruction-turning/F3_v2_pipeline",
+    "code/pretrain_data_prep": WORKSPACE / "Datasets",
+}
+
+SOURCE_FILES = {
+    "data/pretrain/code_messages_pre_train_filtered_stage1.jsonl": WORKSPACE / "Datasets/code_messages_pre_train_filtered_stage1.jsonl",
+    "data/f3/input/scicode_sft_swift_F3_clean.jsonl": WORKSPACE / "Datasets/Instruction-turning/scicode_sft_swift_F3_clean.jsonl",
+    "data/f3/intermediate/F3_stage1.jsonl": WORKSPACE / "Datasets/Instruction-turning/F3_v2_pipeline/F3_stage1.jsonl",
+    "data/f3/intermediate/F3_stage2.jsonl": WORKSPACE / "Datasets/Instruction-turning/F3_v2_pipeline/F3_stage2.jsonl",
+    "data/f3/intermediate/F3_stage3.jsonl": WORKSPACE / "Datasets/Instruction-turning/F3_v2_pipeline/F3_stage3.jsonl",
+    "data/f3/intermediate/F3_stage4_scores.jsonl": WORKSPACE / "Datasets/Instruction-turning/F3_v2_pipeline/F3_stage4_scores.jsonl",
+    "data/f3/intermediate/F3_stage4_calibration_scores.jsonl": WORKSPACE / "Datasets/Instruction-turning/F3_v2_pipeline/F3_stage4_calibration_scores.jsonl",
+    "data/f3/output/scicode_sft_swift_F3_clean_v2.jsonl": WORKSPACE / "Datasets/Instruction-turning/scicode_sft_swift_F3_clean_v2.jsonl",
+    "training_scripts/pretrain/stage1_short.sh": WORKSPACE / "Pre_train/swift_runs_curriculum_filtered_0p5b/stage1_short.sh",
+    "training_scripts/sft/run_qwen25_coder_0p5b_instruct_F3_v2_1gpu.sh": WORKSPACE / "Pre_train/swift_runs_F3_v2_0p5b_instruct/run_qwen25_coder_0p5b_instruct_F3_v2_1gpu.sh",
+    "evidence/eval/f3_refined_9p1_summary.json": WORKSPACE / "DELIVERY_0p5B_CPT_SFT/eval/summaries/summary_hf__raid_data_weifeng_Pre_train_output_user_qwen25_coder_0p5b_instruct_F3_v2_1epoch_1gpu_v0-20260508-164342_checkpoint-6435_20260508_191037.json",
+    "evidence/instruction_recipe_results.md": WORKSPACE / "DELIVERY_0p5B_CPT_SFT/reports/instruction_recipe_results.md",
+}
+
+
+def source_for(relative: str) -> Path | None:
+    if relative in SOURCE_FILES:
+        return SOURCE_FILES[relative]
+    # Reports were grouped into a delivery-only ``reports/`` directory.  In
+    # the historical source trees they live beside the corresponding scripts.
+    if relative.startswith("code/f3_filtering/reports/"):
+        return (
+            WORKSPACE
+            / "Datasets/Instruction-turning/F3_v2_pipeline"
+            / Path(relative).name
+        )
+    if relative.startswith("code/pretrain_data_prep/reports/"):
+        return WORKSPACE / "Datasets" / Path(relative).name
+    for prefix, source_root in sorted(SOURCE_PREFIXES.items(), key=lambda item: -len(item[0])):
+        if relative == prefix or relative.startswith(prefix + "/"):
+            return source_root / Path(relative).relative_to(prefix)
+    return None
+
+
+def hash_and_count(path: Path) -> tuple[str, int | None]:
+    digest = hashlib.sha256()
+    lines = 0 if path.suffix == ".jsonl" else None
+    with path.open("rb") as handle:
+        while chunk := handle.read(8 * 1024 * 1024):
+            digest.update(chunk)
+            if lines is not None:
+                lines += chunk.count(b"\n")
+    return digest.hexdigest(), lines
+
+
+def role_for(relative: str) -> str:
+    return relative.split("/", 1)[0]
+
+
+def main() -> None:
+    excluded = {"MANIFEST.json", "SHA256SUMS"}
+    paths = sorted(
+        path for path in ROOT.rglob("*")
+        if path.is_file() and path.relative_to(ROOT).as_posix() not in excluded
+    )
+    files = []
+    copied_bytes = 0
+    hardlinked_bytes = 0
+
+    for path in paths:
+        relative = path.relative_to(ROOT).as_posix()
+        source = source_for(relative)
+        sha256, records = hash_and_count(path)
+        mode = "authored"
+        source_sha256_matches = None
+        if source is not None:
+            if not source.is_file():
+                raise FileNotFoundError(f"missing provenance source: {source}")
+            same_inode = os.path.samefile(path, source)
+            mode = "hardlink" if same_inode else "copy"
+            source_sha256, _ = hash_and_count(source) if not same_inode else (sha256, None)
+            source_sha256_matches = source_sha256 == sha256
+            if not source_sha256_matches:
+                raise RuntimeError(f"delivery/source mismatch: {relative}")
+
+        size = path.stat().st_size
+        if mode == "hardlink":
+            hardlinked_bytes += size
+        else:
+            copied_bytes += size
+        record = {
+            "path": relative,
+            "role": role_for(relative),
+            "bytes": size,
+            "sha256": sha256,
+            "mode": mode,
+            "source": str(source) if source is not None else None,
+            "source_sha256_matches": source_sha256_matches,
+        }
+        if records is not None:
+            record["records"] = records
+        files.append(record)
+
+    manifest = {
+        "schema_version": "delivery-0p5b-pretrain-f3refine-v1",
+        "generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        "delivery_root": str(ROOT),
+        "headline": {
+            "pretrained_model": "models/pretrained/stage1_cpt",
+            "instruction_tuned_model": "models/sft/sft_f3_refined_instruct",
+            "reported_pass_at_1_pct": 9.1,
+            "reported_pass_at_5_pct": 13.0,
+            "evaluation": "Runnable alignment benchmark; 200 tasks; 5 samples/task; 1000 attempts",
+        },
+        "exclusions": [
+            "optimizer.pt, scheduler.pt, rng_state.pth and training_args.bin are not included",
+            "upstream Qwen base/instruct checkpoints are not included",
+            "the exact May training environment snapshot was not captured",
+        ],
+        "storage": {
+            "hardlinked_bytes": hardlinked_bytes,
+            "copied_or_authored_bytes": copied_bytes,
+            "note": "Hard links save local disk; tar/zip/upload reads complete file bytes.",
+        },
+        "file_count": len(files),
+        "files": files,
+    }
+    manifest_path = ROOT / "MANIFEST.json"
+    manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+    checksum_paths = sorted(path for path in ROOT.rglob("*") if path.is_file() and path.name != "SHA256SUMS")
+    checksum_lines = []
+    for path in checksum_paths:
+        digest, _ = hash_and_count(path)
+        checksum_lines.append(f"{digest}  {path.relative_to(ROOT).as_posix()}")
+    (ROOT / "SHA256SUMS").write_text("\n".join(checksum_lines) + "\n", encoding="utf-8")
+
+    print(json.dumps({
+        "root": str(ROOT),
+        "files": len(files),
+        "hardlinked_bytes": hardlinked_bytes,
+        "copied_or_authored_bytes": copied_bytes,
+    }, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

tools/smoke_test.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Load each delivered checkpoint on CPU and verify finite logits."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import traceback
+from pathlib import Path
+
+
+THREADS = "4"
+for variable in (
+    "OMP_NUM_THREADS",
+    "MKL_NUM_THREADS",
+    "OPENBLAS_NUM_THREADS",
+    "NUMEXPR_NUM_THREADS",
+    "VECLIB_MAXIMUM_THREADS",
+    "BLIS_NUM_THREADS",
+):
+    os.environ.setdefault(variable, THREADS)
+os.environ.setdefault("CUDA_VISIBLE_DEVICES", "")
+
+import torch  # noqa: E402
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer  # noqa: E402
+
+
+ROOT = Path(__file__).resolve().parents[1]
+PROMPT = 'def rmsd(a, b):\n    """Root-mean-square deviation."""\n'
+TARGETS = {
+    "pretrained/stage1_cpt": ROOT / "models/pretrained/stage1_cpt",
+    "sft/sft_f3_refined_instruct": ROOT / "models/sft/sft_f3_refined_instruct",
+}
+
+
+def check(name: str, path: Path) -> dict:
+    result = {"model": name, "path": str(path.relative_to(ROOT)), "ok": False}
+    try:
+        config = AutoConfig.from_pretrained(path)
+        result.update(
+            architecture=(config.architectures or [type(config).__name__])[0],
+            hidden_size=getattr(config, "hidden_size", None),
+            layers=getattr(config, "num_hidden_layers", None),
+            vocab_size=getattr(config, "vocab_size", None),
+        )
+        tokenizer = AutoTokenizer.from_pretrained(path)
+        model = AutoModelForCausalLM.from_pretrained(path, dtype=torch.float32)
+        model.eval()
+        inputs = tokenizer(PROMPT, return_tensors="pt")
+        with torch.no_grad():
+            logits = model(**inputs).logits
+        last = logits[0, -1].float()
+        result.update(
+            tokenizer=type(tokenizer).__name__,
+            parameters_million=round(sum(p.numel() for p in model.parameters()) / 1e6, 1),
+            logits_shape=list(logits.shape),
+            finite=bool(torch.isfinite(logits).all()),
+            top1_zscore=round(((last.max() - last.mean()) / last.std()).item(), 2),
+        )
+        result["ok"] = result["finite"] and result["top1_zscore"] > 3.0
+        del model
+    except Exception as error:  # noqa: BLE001
+        result["error"] = f"{type(error).__name__}: {error}"
+        result["traceback"] = traceback.format_exc(limit=3)
+    return result
+
+
+def main() -> int:
+    torch.set_num_threads(int(THREADS))
+    results = [check(name, path) for name, path in TARGETS.items()]
+    output = ROOT / "evidence/smoke_test_results.json"
+    output.write_text(json.dumps(results, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    for result in results:
+        status = "PASS" if result["ok"] else "FAIL"
+        detail = f"{result.get('parameters_million', '?')}M, z={result.get('top1_zscore', '?')}"
+        print(f"{status}: {result['model']} ({detail})")
+    passed = sum(result["ok"] for result in results)
+    print(f"{passed}/{len(results)} checkpoints passed; results: {output}")
+    return 0 if passed == len(results) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

training_scripts/README.md

@@ -0,0 +1,11 @@
+# Historical Training Scripts
+
+This directory contains the original launch scripts for the two principal training runs.
+Machine-specific absolute paths have intentionally not been rewritten.
+
+- `pretrain/stage1_short.sh`: Stage-1 short-sequence continued pre-training.
+- `sft/run_qwen25_coder_0p5b_instruct_F3_v2_1gpu.sh`: full-parameter F3-refined SFT.
+
+Before rerunning either script, review `CUDA_VISIBLE_DEVICES`, `DATASET`, `OUTPUT_DIR`, cache
+directories, ports, and the FlashAttention/ms-swift environment. Use each checkpoint's
+`args.json` to verify the effective parameters item by item.

training_scripts/pretrain/stage1_short.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Curriculum Stage 1 (short, token <= 1024)
+# Start: Qwen2.5-Coder-0.5B (base)
+# Data:  902,115 samples
+# lr:    5e-5, cosine
+
+export CUDA_VISIBLE_DEVICES=3
+export NPROC_PER_NODE=1
+export NNODES=1
+export NODE_RANK=0
+export MASTER_ADDR=127.0.0.1
+export MASTER_PORT=29540
+
+export USE_HF=1
+export HF_HOME=/home/weifeng/.cache/huggingface
+export HUGGINGFACE_HUB_CACHE=/home/weifeng/.cache/huggingface/hub
+export TRANSFORMERS_CACHE=/home/weifeng/.cache/huggingface/transformers
+export MODELSCOPE_CACHE=/home/weifeng/.cache/modelscope
+
+mkdir -p "${HF_HOME}" "${HUGGINGFACE_HUB_CACHE}" "${TRANSFORMERS_CACHE}" "${MODELSCOPE_CACHE}"
+
+DATASET="/raid/data/weifeng/Datasets/code_messages_pre_train_filtered_stage1.jsonl"
+OUTPUT_DIR="/raid/data/weifeng/Pre_train/output_user/qwen25_coder_0p5b_curriculum_filtered_stage1"
+
+swift pt \
+  --model Qwen/Qwen2.5-Coder-0.5B \
+  --tuner_type full \
+  --dataset "${DATASET}" \
+  --torch_dtype bfloat16 \
+  --attn_impl flash_attn \
+  --per_device_train_batch_size 8 \
+  --learning_rate 5e-5 \
+  --gradient_accumulation_steps 4 \
+  --save_steps 2000 \
+  --save_total_limit 3 \
+  --logging_steps 50 \
+  --max_length 1024 \
+  --num_train_epochs 1 \
+  --warmup_ratio 0.03 \
+  --lr_scheduler_type cosine \
+  --dataloader_num_workers 8 \
+  --dataset_num_proc 8 \
+  --use_hf true \
+  --save_only_model false \
+  --output_dir "${OUTPUT_DIR}"