Upload asvd_llm/heal/llama2-13b-base/provenance.json with huggingface_hub

asvd_llm/heal/llama2-13b-base/provenance.json

@@ -0,0 +1,200 @@
+{
+  "method": "KD healing: alpha*T^2*KL(teacher||student) + (1-alpha)*CE",
+  "student_init": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/asvd_base_13b/tr3fmt/state_dict_fp16.pt",
+  "teacher": "NousResearch/Llama-2-13b-hf",
+  "alpha": 1.0,
+  "temperature": 1.0,
+  "lr": 2e-05,
+  "tokens_seen": 59998208,
+  "epochs": 0.228,
+  "global_batch_tokens": 32768,
+  "dataset_mixture_pre_exclusion": {
+    "alpaca": 50000,
+    "openhermes": 200000,
+    "ultrachat": 100000
+  },
+  "dataset_revisions": {
+    "tatsu-lab/alpaca": "dce01c9b08f87459cf36a430d809084718273017",
+    "teknium/OpenHermes-2.5": "b82037821055c377bed0d495e72e46de3bc72e84",
+    "HuggingFaceH4/ultrachat_200k": "8049631c405ae6576f93f445c6b8166f76f5505a"
+  },
+  "canonical_pool_excluded_texts": 10000,
+  "train_texts_after_exclusion": 356000,
+  "chat_texts": 340000,
+  "mmlu_aux_texts_kept": 16000,
+  "mmlu_aux_split": "cais/mmlu auxiliary_train (official TRAIN split, disjoint from the evaluated test split; audit verified 0/99842 exact overlap)",
+  "mmlu_aux_stats": {
+    "requested_groups": 16000,
+    "rendered_groups": 16000,
+    "shots": 5,
+    "rows_total": 99842,
+    "rows_dropped_by_decontam": 71,
+    "rows_clean": 99771,
+    "revision": "c30699e8356da336a370243923dbaf21066bb9fe",
+    "format": "MMLU-shaped (NOT byte-identical to lm-eval; auxiliary_train has no subject)"
+  },
+  "gold_label_supervision": false,
+  "attribution_note": "alpha=1.0 => pure KD, no gold-label CE, so any MMLU change is attributable to distillation on in-domain text. alpha<1 adds CE on text containing answers = labeled MCQ training, which is a DIFFERENT and weaker claim.",
+  "train_tokens_total": 263112704,
+  "decontamination": {
+    "removed": {
+      "alpaca": 11,
+      "openhermes": 120,
+      "ultrachat": 33
+    }
+  },
+  "start_kl": 3.904147833585739,
+  "final_kl": 0.162364493124187,
+  "start_mmlu": 0.25219298245614036,
+  "final_mmlu": 0.43991228070175437,
+  "history": [
+    {
+      "tokens_m": 5.01,
+      "step": 153,
+      "held_out_kl": 0.24505719915032387,
+      "mmlu": 0.3557017543859649
+    },
+    {
+      "tokens_m": 10.03,
+      "step": 306,
+      "held_out_kl": 0.2154273558408022,
+      "mmlu": 0.4100877192982456
+    },
+    {
+      "tokens_m": 15.01,
+      "step": 458,
+      "held_out_kl": 0.1983913779258728,
+      "mmlu": 0.425
+    },
+    {
+      "tokens_m": 20.02,
+      "step": 611,
+      "held_out_kl": 0.1859402358531952,
+      "mmlu": 0.4276315789473684
+    },
+    {
+      "tokens_m": 25.0,
+      "step": 763,
+      "held_out_kl": 0.17727956734597683,
+      "mmlu": 0.43991228070175437
+    },
+    {
+      "tokens_m": 30.02,
+      "step": 916,
+      "held_out_kl": 0.17239027563482523,
+      "mmlu": 0.42719298245614035
+    },
+    {
+      "tokens_m": 35.03,
+      "step": 1069,
+      "held_out_kl": 0.16772183869034052,
+      "mmlu": 0.43508771929824563
+    },
+    {
+      "tokens_m": 40.01,
+      "step": 1221,
+      "held_out_kl": 0.1655239937826991,
+      "mmlu": 0.4342105263157895
+    },
+    {
+      "tokens_m": 45.02,
+      "step": 1374,
+      "held_out_kl": 0.16350429691374302,
+      "mmlu": 0.4394736842105263
+    },
+    {
+      "tokens_m": 50.0,
+      "step": 1526,
+      "held_out_kl": 0.16269836202263832,
+      "mmlu": 0.43640350877192985
+    },
+    {
+      "tokens_m": 55.02,
+      "step": 1679,
+      "held_out_kl": 0.16245183907449245,
+      "mmlu": 0.4412280701754386
+    },
+    {
+      "tokens_m": 60.0,
+      "step": 1831,
+      "held_out_kl": 0.162364493124187,
+      "mmlu": 0.43991228070175437,
+      "final": true
+    }
+  ],
+  "teacher_mmlu_same_limit": 0.5425438596491228,
+  "checkpoint": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_asvd_base_13b_h1/state_dict_fp16.pt",
+  "sha256": "e3d477e17f4a2f7dc17adc980e34866f1f6134e7b482d47f2835c3ce06189940",
+  "seed": 1234,
+  "mmlu_limit_per_subtask": 40,
+  "teacher_revision": "4b54e9fdd2ebd8db0901d37b98d3a53bbfaa4503",
+  "student_init_sha256": "c44d7dc63dd23f1eb09843d73abebe48d31b732457d9b5054670c70ca55b98ac",
+  "ranks_file": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/asvd_base_13b/tr3fmt/ranks.json",
+  "ranks_sha256": "fef5d275a2ac880abaf4b6f833b17c876cca823ad5634b949f575393c72b9d2e",
+  "input_identity_gate": {
+    "student_ckpt": {
+      "key": "tr3fmt",
+      "sha256": "c44d7dc63dd23f1eb09843d73abebe48d31b732457d9b5054670c70ca55b98ac",
+      "status": "unregistered_allowed"
+    },
+    "ranks": {
+      "key": "tr3fmt",
+      "sha256": "fef5d275a2ac880abaf4b6f833b17c876cca823ad5634b949f575393c72b9d2e",
+      "status": "unregistered_allowed"
+    }
+  },
+  "train_seed": 1234,
+  "mmlu_seed": 1234,
+  "dynamic_batch": false,
+  "dbl": null,
+  "snapshot_milestones": true,
+  "snapshots": [
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_asvd_base_13b_h1_snap15m'), 15.007744)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_asvd_base_13b_h1_snap30m'), 30.015488)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_asvd_base_13b_h1_snap45m'), 45.023232)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_asvd_base_13b_h1_snap50m'), 50.003968)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_asvd_base_13b_h1_snap55m'), 55.017472)"
+  ],
+  "stream_skip_tokens_m": 0.0,
+  "stream_blocks_consumed": [
+    0,
+    58592
+  ],
+  "stream_note": "the block order is a seeded permutation consumed from index 0; stages sharing a --seed with stream_skip=0 re-train on the SAME blocks. [start, end] block indices are recorded so any continuation can be checked for overlap with what earlier stages consumed.",
+  "student_master_dtype": "bf16",
+  "eval_dtype": "bf16",
+  "fake_quant": null,
+  "rebalance_report": null,
+  "student_param_count": 7746790400,
+  "micro": 8,
+  "grad_accum": 4,
+  "seq_len": 1024,
+  "held_out_source": "canonical: mixture 2000/4000/4000, seed 1234, seq 1024, blocks 256:264 (identical to the compression gates)",
+  "decontam_stats": {
+    "mmlu": {
+      "items": 14042,
+      "new_ngrams": 764261,
+      "revision": "c30699e8356da336a370243923dbaf21066bb9fe"
+    },
+    "hellaswag": {
+      "items": 10042,
+      "new_ngrams": 1334302,
+      "revision": "218ec52e09a7e7462a5400043bb9a69a41d06b76"
+    },
+    "boolq": {
+      "items": 3270,
+      "new_ngrams": 276806,
+      "revision": "3de24cf8022e94f4ee4b9d55a6f539891524d646"
+    },
+    "triviaqa": {
+      "items": 17944,
+      "new_ngrams": 32608,
+      "revision": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f"
+    },
+    "gsm8k": {
+      "items": 1319,
+      "new_ngrams": 46281,
+      "revision": "740312add88f781978c0658806c59bc2815b9866"
+    }
+  }
+}