Upload asvd_llm/heal/llama2-7b-base/provenance.json with huggingface_hub

asvd_llm/heal/llama2-7b-base/provenance.json

@@ -0,0 +1,200 @@
+{
+  "method": "KD healing: alpha*T^2*KL(teacher||student) + (1-alpha)*CE",
+  "student_init": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/asvd_base_7b/tr3fmt/state_dict_fp16.pt",
+  "teacher": "NousResearch/Llama-2-7b-hf",
+  "alpha": 1.0,
+  "temperature": 1.0,
+  "lr": 2e-05,
+  "tokens_seen": 59998208,
+  "epochs": 0.228,
+  "global_batch_tokens": 32768,
+  "dataset_mixture_pre_exclusion": {
+    "alpaca": 50000,
+    "openhermes": 200000,
+    "ultrachat": 100000
+  },
+  "dataset_revisions": {
+    "tatsu-lab/alpaca": "dce01c9b08f87459cf36a430d809084718273017",
+    "teknium/OpenHermes-2.5": "b82037821055c377bed0d495e72e46de3bc72e84",
+    "HuggingFaceH4/ultrachat_200k": "8049631c405ae6576f93f445c6b8166f76f5505a"
+  },
+  "canonical_pool_excluded_texts": 10000,
+  "train_texts_after_exclusion": 356000,
+  "chat_texts": 340000,
+  "mmlu_aux_texts_kept": 16000,
+  "mmlu_aux_split": "cais/mmlu auxiliary_train (official TRAIN split, disjoint from the evaluated test split; audit verified 0/99842 exact overlap)",
+  "mmlu_aux_stats": {
+    "requested_groups": 16000,
+    "rendered_groups": 16000,
+    "shots": 5,
+    "rows_total": 99842,
+    "rows_dropped_by_decontam": 71,
+    "rows_clean": 99771,
+    "revision": "c30699e8356da336a370243923dbaf21066bb9fe",
+    "format": "MMLU-shaped (NOT byte-identical to lm-eval; auxiliary_train has no subject)"
+  },
+  "gold_label_supervision": false,
+  "attribution_note": "alpha=1.0 => pure KD, no gold-label CE, so any MMLU change is attributable to distillation on in-domain text. alpha<1 adds CE on text containing answers = labeled MCQ training, which is a DIFFERENT and weaker claim.",
+  "train_tokens_total": 263112704,
+  "decontamination": {
+    "removed": {
+      "alpaca": 11,
+      "openhermes": 120,
+      "ultrachat": 33
+    }
+  },
+  "start_kl": 7.296980082988739,
+  "final_kl": 0.22557126358151436,
+  "start_mmlu": 0.23421052631578948,
+  "final_mmlu": 0.262719298245614,
+  "history": [
+    {
+      "tokens_m": 5.01,
+      "step": 153,
+      "held_out_kl": 0.3589833080768585,
+      "mmlu": 0.24780701754385964
+    },
+    {
+      "tokens_m": 10.03,
+      "step": 306,
+      "held_out_kl": 0.30126900784671307,
+      "mmlu": 0.2469298245614035
+    },
+    {
+      "tokens_m": 15.01,
+      "step": 458,
+      "held_out_kl": 0.2774391770362854,
+      "mmlu": 0.2535087719298246
+    },
+    {
+      "tokens_m": 20.02,
+      "step": 611,
+      "held_out_kl": 0.26040532626211643,
+      "mmlu": 0.24956140350877193
+    },
+    {
+      "tokens_m": 25.0,
+      "step": 763,
+      "held_out_kl": 0.24820629693567753,
+      "mmlu": 0.2543859649122807
+    },
+    {
+      "tokens_m": 30.02,
+      "step": 916,
+      "held_out_kl": 0.23963410779833794,
+      "mmlu": 0.25570175438596493
+    },
+    {
+      "tokens_m": 35.03,
+      "step": 1069,
+      "held_out_kl": 0.23328605107963085,
+      "mmlu": 0.2548245614035088
+    },
+    {
+      "tokens_m": 40.01,
+      "step": 1221,
+      "held_out_kl": 0.2291983012109995,
+      "mmlu": 0.25263157894736843
+    },
+    {
+      "tokens_m": 45.02,
+      "step": 1374,
+      "held_out_kl": 0.22688163630664349,
+      "mmlu": 0.25526315789473686
+    },
+    {
+      "tokens_m": 50.0,
+      "step": 1526,
+      "held_out_kl": 0.22644959017634392,
+      "mmlu": 0.2600877192982456
+    },
+    {
+      "tokens_m": 55.02,
+      "step": 1679,
+      "held_out_kl": 0.2259067613631487,
+      "mmlu": 0.2631578947368421
+    },
+    {
+      "tokens_m": 60.0,
+      "step": 1831,
+      "held_out_kl": 0.22557126358151436,
+      "mmlu": 0.262719298245614,
+      "final": true
+    }
+  ],
+  "teacher_mmlu_same_limit": 0.47412280701754383,
+  "checkpoint": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_asvd_base_7b_h1/state_dict_fp16.pt",
+  "sha256": "fb2bbeeb648f6bed993d66ac7999af6c751860f2d0edb2cb9f38fc9d20f9efb5",
+  "seed": 1234,
+  "mmlu_limit_per_subtask": 40,
+  "teacher_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+  "student_init_sha256": "cc8a3ede167248b34b28fe9ecb94215f841c33cbcd8336b311cfae4e46852f00",
+  "ranks_file": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/asvd_base_7b/tr3fmt/ranks.json",
+  "ranks_sha256": "5836ebdb4a54ccfd0e40c2d7b5abf5f828efbd508f7efb28b27e89aee06e69e4",
+  "input_identity_gate": {
+    "student_ckpt": {
+      "key": "tr3fmt",
+      "sha256": "cc8a3ede167248b34b28fe9ecb94215f841c33cbcd8336b311cfae4e46852f00",
+      "status": "unregistered_allowed"
+    },
+    "ranks": {
+      "key": "tr3fmt",
+      "sha256": "5836ebdb4a54ccfd0e40c2d7b5abf5f828efbd508f7efb28b27e89aee06e69e4",
+      "status": "unregistered_allowed"
+    }
+  },
+  "train_seed": 1234,
+  "mmlu_seed": 1234,
+  "dynamic_batch": false,
+  "dbl": null,
+  "snapshot_milestones": true,
+  "snapshots": [
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_asvd_base_7b_h1_snap15m'), 15.007744)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_asvd_base_7b_h1_snap30m'), 30.015488)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_asvd_base_7b_h1_snap45m'), 45.023232)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_asvd_base_7b_h1_snap50m'), 50.003968)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_asvd_base_7b_h1_snap55m'), 55.017472)"
+  ],
+  "stream_skip_tokens_m": 0.0,
+  "stream_blocks_consumed": [
+    0,
+    58592
+  ],
+  "stream_note": "the block order is a seeded permutation consumed from index 0; stages sharing a --seed with stream_skip=0 re-train on the SAME blocks. [start, end] block indices are recorded so any continuation can be checked for overlap with what earlier stages consumed.",
+  "student_master_dtype": "bf16",
+  "eval_dtype": "bf16",
+  "fake_quant": null,
+  "rebalance_report": null,
+  "student_param_count": 4020677888,
+  "micro": 16,
+  "grad_accum": 2,
+  "seq_len": 1024,
+  "held_out_source": "canonical: mixture 2000/4000/4000, seed 1234, seq 1024, blocks 256:264 (identical to the compression gates)",
+  "decontam_stats": {
+    "mmlu": {
+      "items": 14042,
+      "new_ngrams": 764261,
+      "revision": "c30699e8356da336a370243923dbaf21066bb9fe"
+    },
+    "hellaswag": {
+      "items": 10042,
+      "new_ngrams": 1334302,
+      "revision": "218ec52e09a7e7462a5400043bb9a69a41d06b76"
+    },
+    "boolq": {
+      "items": 3270,
+      "new_ngrams": 276806,
+      "revision": "3de24cf8022e94f4ee4b9d55a6f539891524d646"
+    },
+    "triviaqa": {
+      "items": 17944,
+      "new_ngrams": 32608,
+      "revision": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f"
+    },
+    "gsm8k": {
+      "items": 1319,
+      "new_ngrams": 46281,
+      "revision": "740312add88f781978c0658806c59bc2815b9866"
+    }
+  }
+}