| { |
| "adaptation": { |
| "adapter_compute": "float32", |
| "alpha": 64, |
| "base_compute": "bfloat16", |
| "dropout": 0.0, |
| "expected_trainable_parameters": 3785856, |
| "layers": "all", |
| "method": "lora", |
| "rank": 32, |
| "targets": [ |
| "attn.qkv_proj", |
| "attn.proj" |
| ], |
| "trainable_token_ids": [ |
| 2, |
| 3 |
| ] |
| }, |
| "answer_max_tokens": { |
| "constraints": 320, |
| "dialogue": 256, |
| "general": 256, |
| "grounded": 160, |
| "knowledge": 80, |
| "rewrite": 320, |
| "skills": 160, |
| "social": 128 |
| }, |
| "authored_holdouts": { |
| "constraints": 128, |
| "dialogue": 128, |
| "skills": 192 |
| }, |
| "authored_quotas": { |
| "constraints": 4200, |
| "dialogue": 6800, |
| "skills": 12000 |
| }, |
| "baseline_cache": "${WORKSPACE}/pretraining/prepared/chat_adapter_search_20261108/baseline_cache", |
| "batches": [ |
| 8 |
| ], |
| "betas": [ |
| 0.9, |
| 0.95 |
| ], |
| "checked_components": [ |
| "knowledge", |
| "grounded", |
| "skills", |
| "constraints", |
| "dialogue" |
| ], |
| "comparison_checkpoint": "${WORKSPACE}/reports/benchmarks/deep44-reliable-chat-167-fp32-20261106/choices/checkpoint", |
| "component_quotas": { |
| "constraints": 800, |
| "dialogue": 2000, |
| "general": 2400, |
| "grounded": 2000, |
| "skills": 800 |
| }, |
| "components": [ |
| "general", |
| "knowledge", |
| "grounded", |
| "skills", |
| "constraints", |
| "dialogue", |
| "social", |
| "rewrite" |
| ], |
| "epochs": 1, |
| "eps": 1e-08, |
| "eval_every": 64, |
| "eval_examples_per_component": 64, |
| "execution": "single-mac", |
| "final_lr": 5e-06, |
| "format": "opensml-verified-chat-v2", |
| "fresh_mixture": { |
| "constraints": 0.4, |
| "dialogue": 0.1, |
| "grounded": 0.1, |
| "skills": 0.4 |
| }, |
| "fresh_weight": 0.5, |
| "generated_eval_per_component": 64, |
| "grad_accum": 8, |
| "grad_clip": 1.0, |
| "initial_checkpoint": "${WORKSPACE}/pretraining/runs/deep44_adapter_trial_09_replica_20261108/step_0002000_f3d1ffd185f9", |
| "keep_checkpoints": 6, |
| "kl_weight": 0.5, |
| "log_every": 10, |
| "max_cumulative_retention_loss_increase": 0.03, |
| "max_messages": 8, |
| "max_repetition_rate": 0.03333333333333333, |
| "max_retention_loss_increase": 0.003, |
| "max_sequence_length": 2048, |
| "max_steps": 256, |
| "method": "Editing and instruction skills with preservation", |
| "microbatch_rows": 4, |
| "min_domain_pass_cases": {}, |
| "min_natural_stop": 0.9833333333333333, |
| "min_task_gain_cases": 4, |
| "minimum_source_tokens": 42000000000, |
| "original_pretraining_retention_loss": 2.8353193145245315, |
| "output": "${WORKSPACE}/pretraining/runs/deep44_parent2000_generalize_05_replica_20261109", |
| "peak_lr": 5e-05, |
| "prepared": "${WORKSPACE}/pretraining/runs/deep44_parent2000_generalize_05_replica_20261109/input/corpus", |
| "public_holdouts": { |
| "constraints_smol": 32, |
| "dialogue_smol": 64, |
| "general_smol": 64, |
| "grounded_no": 32, |
| "grounded_yes": 96, |
| "knowledge": 128, |
| "rewrite_smol": 32, |
| "social": 32 |
| }, |
| "public_quotas": { |
| "constraints_smol": 2000, |
| "dialogue_smol": 1200, |
| "general_smol": 4800, |
| "grounded_no": 2750, |
| "grounded_yes": 8250, |
| "knowledge": 4000, |
| "rewrite_smol": 1000, |
| "social": 1000 |
| }, |
| "qualitative_eval_per_component": 8, |
| "regression_patience": 2, |
| "replay_length": 512, |
| "replay_mixture": { |
| "constraints": 0.125, |
| "dialogue": 0.125, |
| "general": 0.1875, |
| "grounded": 0.125, |
| "knowledge": 0.1875, |
| "rewrite": 0.0625, |
| "skills": 0.125, |
| "social": 0.0625 |
| }, |
| "replay_objective_weight": 0.2, |
| "replay_policy": "prose-only", |
| "replay_rows_per_update": 16, |
| "retention": "${WORKSPACE}/pretraining/data/deep44_pretraining_validation_20261102", |
| "rows_per_group": 16, |
| "seed": 202611097, |
| "selection_reference": "Parent 2000: +4/320 development, +8/704 final, no domain/control regressions; fresh 256-case check must not regress; full-suite guard after private pass", |
| "sft_objective_weight": 0.8, |
| "sft_rows_per_update": 48, |
| "sft_stage": 2, |
| "source_checkpoint": "${WORKSPACE}/pretraining/runs/deep44_cooldown_2b_20261107/step_0320445_08b689895ed2", |
| "source_run": "${WORKSPACE}/pretraining/runs/deep44_cooldown_2b_20261107", |
| "sources": [ |
| { |
| "file": "train.jsonl.gz", |
| "label": "helpsteer2", |
| "license": "cc-by-4.0", |
| "repo": "nvidia/HelpSteer2", |
| "revision": "990b2711a36180dd19d9c94b8627844866f8982a" |
| }, |
| { |
| "file": "data/train-00000-of-00001-b42a775f407cee45.parquet", |
| "label": "oasst1", |
| "license": "apache-2.0", |
| "repo": "OpenAssistant/oasst1", |
| "revision": "fdf72ae0827c1cda404aff25b6603abec9e3399b" |
| }, |
| { |
| "files": [ |
| "data/smol-magpie-ultra/train-00000-of-00006.parquet", |
| "data/smol-magpie-ultra/train-00001-of-00006.parquet", |
| "data/smol-magpie-ultra/train-00002-of-00006.parquet", |
| "data/smol-magpie-ultra/train-00003-of-00006.parquet", |
| "data/smol-magpie-ultra/train-00004-of-00006.parquet", |
| "data/smol-magpie-ultra/train-00005-of-00006.parquet" |
| ], |
| "label": "smol_magpie_ultra", |
| "license": "apache-2.0", |
| "repo": "HuggingFaceTB/smoltalk", |
| "revision": "5feaf2fd3ffca7c237fc38d1861bc30365d48ffa" |
| }, |
| { |
| "label": "squad", |
| "license": "cc-by-sa-4.0", |
| "parquet_urls": [ |
| "https://huggingface.co/datasets/rajpurkar/squad_v2/resolve/3ffb306f725f7d2ce8394bc1873b24868140c412/squad_v2/train-00000-of-00001.parquet" |
| ], |
| "repo": "rajpurkar/squad_v2", |
| "revision": "3ffb306f725f7d2ce8394bc1873b24868140c412", |
| "split": "train" |
| }, |
| { |
| "label": "squad_validation", |
| "license": "cc-by-sa-4.0", |
| "parquet_urls": [ |
| "https://huggingface.co/datasets/rajpurkar/squad_v2/resolve/3ffb306f725f7d2ce8394bc1873b24868140c412/squad_v2/validation-00000-of-00001.parquet" |
| ], |
| "repo": "rajpurkar/squad_v2", |
| "revision": "3ffb306f725f7d2ce8394bc1873b24868140c412", |
| "split": "validation" |
| } |
| ], |
| "strategy": "Fresh computed instructions and context tasks plus broad original-training replay; optional forward KL to the frozen parent on replay assistant tokens; all attempts restart exactly from parent2000 with fresh moments", |
| "tokenizer": "${WORKSPACE}/reference/opensml-150m/tokenizer/bytebpe32k_v1", |
| "warmup_fraction": 0.1, |
| "weight_decay": 0.0 |
| } |
|
|
