Upload svd_llm/heal/llama2-7b-base/provenance.json with huggingface_hub

svd_llm/heal/llama2-7b-base/provenance.json

@@ -0,0 +1,199 @@
+{
+  "method": "KD healing: alpha*T^2*KL(teacher||student) + (1-alpha)*CE",
+  "student_init": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/svdllm1_base_7b/tr3fmt/state_dict_fp16.pt",
+  "teacher": "NousResearch/Llama-2-7b-hf",
+  "alpha": 1.0,
+  "temperature": 1.0,
+  "lr": 2e-05,
+  "tokens_seen": 59998208,
+  "epochs": 0.228,
+  "global_batch_tokens": 32768,
+  "dataset_mixture_pre_exclusion": {
+    "alpaca": 50000,
+    "openhermes": 200000,
+    "ultrachat": 100000
+  },
+  "dataset_revisions": {
+    "tatsu-lab/alpaca": "dce01c9b08f87459cf36a430d809084718273017",
+    "teknium/OpenHermes-2.5": "b82037821055c377bed0d495e72e46de3bc72e84",
+    "HuggingFaceH4/ultrachat_200k": "8049631c405ae6576f93f445c6b8166f76f5505a"
+  },
+  "canonical_pool_excluded_texts": 10000,
+  "train_texts_after_exclusion": 356000,
+  "chat_texts": 340000,
+  "mmlu_aux_texts_kept": 16000,
+  "mmlu_aux_split": "cais/mmlu auxiliary_train (official TRAIN split, disjoint from the evaluated test split; audit verified 0/99842 exact overlap)",
+  "mmlu_aux_stats": {
+    "requested_groups": 16000,
+    "rendered_groups": 16000,
+    "shots": 5,
+    "rows_total": 99842,
+    "rows_dropped_by_decontam": 71,
+    "rows_clean": 99771,
+    "revision": "c30699e8356da336a370243923dbaf21066bb9fe",
+    "format": "MMLU-shaped (NOT byte-identical to lm-eval; auxiliary_train has no subject)"
+  },
+  "gold_label_supervision": false,
+  "attribution_note": "alpha=1.0 => pure KD, no gold-label CE, so any MMLU change is attributable to distillation on in-domain text. alpha<1 adds CE on text containing answers = labeled MCQ training, which is a DIFFERENT and weaker claim.",
+  "train_tokens_total": 263112704,
+  "decontamination": {
+    "removed": {
+      "alpaca": 11,
+      "openhermes": 120,
+      "ultrachat": 33
+    }
+  },
+  "start_kl": 2.04391610622406,
+  "final_kl": 0.20887502655386925,
+  "start_mmlu": 0.256140350877193,
+  "final_mmlu": 0.3412280701754386,
+  "history": [
+    {
+      "tokens_m": 5.01,
+      "step": 153,
+      "held_out_kl": 0.5083311274647713,
+      "mmlu": 0.23728070175438595
+    },
+    {
+      "tokens_m": 10.03,
+      "step": 306,
+      "held_out_kl": 0.41946782171726227,
+      "mmlu": 0.27631578947368424
+    },
+    {
+      "tokens_m": 15.01,
+      "step": 458,
+      "held_out_kl": 0.3989887274801731,
+      "mmlu": 0.2657894736842105
+    },
+    {
+      "tokens_m": 20.02,
+      "step": 611,
+      "held_out_kl": 0.3655772991478443,
+      "mmlu": 0.2710526315789474
+    },
+    {
+      "tokens_m": 25.0,
+      "step": 763,
+      "held_out_kl": 0.3386868014931679,
+      "mmlu": 0.2881578947368421
+    },
+    {
+      "tokens_m": 30.02,
+      "step": 916,
+      "held_out_kl": 0.30176149867475033,
+      "mmlu": 0.29736842105263156
+    },
+    {
+      "tokens_m": 35.03,
+      "step": 1069,
+      "held_out_kl": 0.2661347910761833,
+      "mmlu": 0.3131578947368421
+    },
+    {
+      "tokens_m": 40.01,
+      "step": 1221,
+      "held_out_kl": 0.24369513802230358,
+      "mmlu": 0.33728070175438596
+    },
+    {
+      "tokens_m": 45.02,
+      "step": 1374,
+      "held_out_kl": 0.22949891351163387,
+      "mmlu": 0.3412280701754386
+    },
+    {
+      "tokens_m": 50.0,
+      "step": 1526,
+      "held_out_kl": 0.218830781057477,
+      "mmlu": 0.3412280701754386
+    },
+    {
+      "tokens_m": 55.02,
+      "step": 1679,
+      "held_out_kl": 0.21270246990025043,
+      "mmlu": 0.33421052631578946
+    },
+    {
+      "tokens_m": 60.0,
+      "step": 1831,
+      "held_out_kl": 0.20887502655386925,
+      "mmlu": 0.3412280701754386,
+      "final": true
+    }
+  ],
+  "teacher_mmlu_same_limit": 0.47412280701754383,
+  "checkpoint": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm1_base_7b_h1/state_dict_fp16.pt",
+  "sha256": "3e59b1f75dc3bae94827c89da86606511a786785f967b85941c439762ebe5ad6",
+  "seed": 1234,
+  "mmlu_limit_per_subtask": 40,
+  "teacher_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+  "student_init_sha256": "1009aa0e0a873b4e4957ffd9df855027ff0f695ab2ef0f0ac7b0f4fd91295a69",
+  "ranks_file": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/svdllm1_base_7b/tr3fmt/ranks.json",
+  "ranks_sha256": "715d09b26b58bbc4052eb425dc041e5d8b3acb5cc6cef729e2ca34c76c0e2943",
+  "input_identity_gate": {
+    "student_ckpt": {
+      "key": "tr3fmt",
+      "sha256": "1009aa0e0a873b4e4957ffd9df855027ff0f695ab2ef0f0ac7b0f4fd91295a69",
+      "status": "unregistered_allowed"
+    },
+    "ranks": {
+      "key": "tr3fmt",
+      "sha256": "715d09b26b58bbc4052eb425dc041e5d8b3acb5cc6cef729e2ca34c76c0e2943",
+      "status": "unregistered_allowed"
+    }
+  },
+  "train_seed": 1234,
+  "mmlu_seed": 1234,
+  "dynamic_batch": false,
+  "dbl": null,
+  "snapshot_milestones": true,
+  "snapshots": [
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm1_base_7b_h1_snap15m'), 15.007744)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm1_base_7b_h1_snap30m'), 30.015488)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm1_base_7b_h1_snap45m'), 45.023232)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm1_base_7b_h1_snap50m'), 50.003968)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm1_base_7b_h1_snap55m'), 55.017472)"
+  ],
+  "stream_skip_tokens_m": 0.0,
+  "stream_blocks_consumed": [
+    0,
+    58592
+  ],
+  "stream_note": "the block order is a seeded permutation consumed from index 0; stages sharing a --seed with stream_skip=0 re-train on the SAME blocks. [start, end] block indices are recorded so any continuation can be checked for overlap with what earlier stages consumed.",
+  "student_master_dtype": "bf16",
+  "fake_quant": null,
+  "rebalance_report": null,
+  "student_param_count": 4021989376,
+  "micro": 16,
+  "grad_accum": 2,
+  "seq_len": 1024,
+  "held_out_source": "canonical: mixture 2000/4000/4000, seed 1234, seq 1024, blocks 256:264 (identical to the compression gates)",
+  "decontam_stats": {
+    "mmlu": {
+      "items": 14042,
+      "new_ngrams": 764261,
+      "revision": "c30699e8356da336a370243923dbaf21066bb9fe"
+    },
+    "hellaswag": {
+      "items": 10042,
+      "new_ngrams": 1334302,
+      "revision": "218ec52e09a7e7462a5400043bb9a69a41d06b76"
+    },
+    "boolq": {
+      "items": 3270,
+      "new_ngrams": 276806,
+      "revision": "3de24cf8022e94f4ee4b9d55a6f539891524d646"
+    },
+    "triviaqa": {
+      "items": 17944,
+      "new_ngrams": 32608,
+      "revision": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f"
+    },
+    "gsm8k": {
+      "items": 1319,
+      "new_ngrams": 46281,
+      "revision": "740312add88f781978c0658806c59bc2815b9866"
+    }
+  }
+}