Upload svd_llm/heal/llama2-13b-base/provenance.json with huggingface_hub

svd_llm/heal/llama2-13b-base/provenance.json

@@ -0,0 +1,199 @@
+{
+  "method": "KD healing: alpha*T^2*KL(teacher||student) + (1-alpha)*CE",
+  "student_init": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/svdllm1_base_13b/tr3fmt/state_dict_fp16.pt",
+  "teacher": "NousResearch/Llama-2-13b-hf",
+  "alpha": 1.0,
+  "temperature": 1.0,
+  "lr": 2e-05,
+  "tokens_seen": 59998208,
+  "epochs": 0.228,
+  "global_batch_tokens": 32768,
+  "dataset_mixture_pre_exclusion": {
+    "alpaca": 50000,
+    "openhermes": 200000,
+    "ultrachat": 100000
+  },
+  "dataset_revisions": {
+    "tatsu-lab/alpaca": "dce01c9b08f87459cf36a430d809084718273017",
+    "teknium/OpenHermes-2.5": "b82037821055c377bed0d495e72e46de3bc72e84",
+    "HuggingFaceH4/ultrachat_200k": "8049631c405ae6576f93f445c6b8166f76f5505a"
+  },
+  "canonical_pool_excluded_texts": 10000,
+  "train_texts_after_exclusion": 356000,
+  "chat_texts": 340000,
+  "mmlu_aux_texts_kept": 16000,
+  "mmlu_aux_split": "cais/mmlu auxiliary_train (official TRAIN split, disjoint from the evaluated test split; audit verified 0/99842 exact overlap)",
+  "mmlu_aux_stats": {
+    "requested_groups": 16000,
+    "rendered_groups": 16000,
+    "shots": 5,
+    "rows_total": 99842,
+    "rows_dropped_by_decontam": 71,
+    "rows_clean": 99771,
+    "revision": "c30699e8356da336a370243923dbaf21066bb9fe",
+    "format": "MMLU-shaped (NOT byte-identical to lm-eval; auxiliary_train has no subject)"
+  },
+  "gold_label_supervision": false,
+  "attribution_note": "alpha=1.0 => pure KD, no gold-label CE, so any MMLU change is attributable to distillation on in-domain text. alpha<1 adds CE on text containing answers = labeled MCQ training, which is a DIFFERENT and weaker claim.",
+  "train_tokens_total": 263112704,
+  "decontamination": {
+    "removed": {
+      "alpaca": 11,
+      "openhermes": 120,
+      "ultrachat": 33
+    }
+  },
+  "start_kl": 1.6616289168596268,
+  "final_kl": 0.19117177929729223,
+  "start_mmlu": 0.2574561403508772,
+  "final_mmlu": 0.43333333333333335,
+  "history": [
+    {
+      "tokens_m": 5.01,
+      "step": 153,
+      "held_out_kl": 0.4570828378200531,
+      "mmlu": 0.37105263157894736
+    },
+    {
+      "tokens_m": 10.03,
+      "step": 306,
+      "held_out_kl": 0.39639149978756905,
+      "mmlu": 0.3973684210526316
+    },
+    {
+      "tokens_m": 15.01,
+      "step": 458,
+      "held_out_kl": 0.3739243634045124,
+      "mmlu": 0.4
+    },
+    {
+      "tokens_m": 20.02,
+      "step": 611,
+      "held_out_kl": 0.3440779075026512,
+      "mmlu": 0.38903508771929823
+    },
+    {
+      "tokens_m": 25.0,
+      "step": 763,
+      "held_out_kl": 0.33168373070657253,
+      "mmlu": 0.4043859649122807
+    },
+    {
+      "tokens_m": 30.02,
+      "step": 916,
+      "held_out_kl": 0.28602365776896477,
+      "mmlu": 0.40921052631578947
+    },
+    {
+      "tokens_m": 35.03,
+      "step": 1069,
+      "held_out_kl": 0.2575125452131033,
+      "mmlu": 0.4118421052631579
+    },
+    {
+      "tokens_m": 40.01,
+      "step": 1221,
+      "held_out_kl": 0.23124912194907665,
+      "mmlu": 0.4307017543859649
+    },
+    {
+      "tokens_m": 45.02,
+      "step": 1374,
+      "held_out_kl": 0.21070933435112238,
+      "mmlu": 0.43991228070175437
+    },
+    {
+      "tokens_m": 50.0,
+      "step": 1526,
+      "held_out_kl": 0.20029953494668007,
+      "mmlu": 0.4412280701754386
+    },
+    {
+      "tokens_m": 55.02,
+      "step": 1679,
+      "held_out_kl": 0.19521544314920902,
+      "mmlu": 0.43508771929824563
+    },
+    {
+      "tokens_m": 60.0,
+      "step": 1831,
+      "held_out_kl": 0.19117177929729223,
+      "mmlu": 0.43333333333333335,
+      "final": true
+    }
+  ],
+  "teacher_mmlu_same_limit": 0.5425438596491228,
+  "checkpoint": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm1_base_13b_h1/state_dict_fp16.pt",
+  "sha256": "1543670e3bc39f0656c4522aa70315ebbc5caadb0bf4ab04577da853cb04e97f",
+  "seed": 1234,
+  "mmlu_limit_per_subtask": 40,
+  "teacher_revision": "4b54e9fdd2ebd8db0901d37b98d3a53bbfaa4503",
+  "student_init_sha256": "bcf2f34e1c7ba25c796057ee555bb3863d65e88f8f75fef35ca4477583901efc",
+  "ranks_file": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/svdllm1_base_13b/tr3fmt/ranks.json",
+  "ranks_sha256": "5c9ed0db91c9c81c51e62e6f82c7c2772478cc353c6f28040164843e64219b55",
+  "input_identity_gate": {
+    "student_ckpt": {
+      "key": "tr3fmt",
+      "sha256": "bcf2f34e1c7ba25c796057ee555bb3863d65e88f8f75fef35ca4477583901efc",
+      "status": "unregistered_allowed"
+    },
+    "ranks": {
+      "key": "tr3fmt",
+      "sha256": "5c9ed0db91c9c81c51e62e6f82c7c2772478cc353c6f28040164843e64219b55",
+      "status": "unregistered_allowed"
+    }
+  },
+  "train_seed": 1234,
+  "mmlu_seed": 1234,
+  "dynamic_batch": false,
+  "dbl": null,
+  "snapshot_milestones": true,
+  "snapshots": [
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm1_base_13b_h1_snap15m'), 15.007744)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm1_base_13b_h1_snap30m'), 30.015488)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm1_base_13b_h1_snap45m'), 45.023232)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm1_base_13b_h1_snap50m'), 50.003968)",
+    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm1_base_13b_h1_snap55m'), 55.017472)"
+  ],
+  "stream_skip_tokens_m": 0.0,
+  "stream_blocks_consumed": [
+    0,
+    58592
+  ],
+  "stream_note": "the block order is a seeded permutation consumed from index 0; stages sharing a --seed with stream_skip=0 re-train on the SAME blocks. [start, end] block indices are recorded so any continuation can be checked for overlap with what earlier stages consumed.",
+  "student_master_dtype": "bf16",
+  "fake_quant": null,
+  "rebalance_report": null,
+  "student_param_count": 7737799680,
+  "micro": 8,
+  "grad_accum": 4,
+  "seq_len": 1024,
+  "held_out_source": "canonical: mixture 2000/4000/4000, seed 1234, seq 1024, blocks 256:264 (identical to the compression gates)",
+  "decontam_stats": {
+    "mmlu": {
+      "items": 14042,
+      "new_ngrams": 764261,
+      "revision": "c30699e8356da336a370243923dbaf21066bb9fe"
+    },
+    "hellaswag": {
+      "items": 10042,
+      "new_ngrams": 1334302,
+      "revision": "218ec52e09a7e7462a5400043bb9a69a41d06b76"
+    },
+    "boolq": {
+      "items": 3270,
+      "new_ngrams": 276806,
+      "revision": "3de24cf8022e94f4ee4b9d55a6f539891524d646"
+    },
+    "triviaqa": {
+      "items": 17944,
+      "new_ngrams": 32608,
+      "revision": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f"
+    },
+    "gsm8k": {
+      "items": 1319,
+      "new_ngrams": 46281,
+      "revision": "740312add88f781978c0658806c59bc2815b9866"
+    }
+  }
+}