LM1B small baseline step 1: checkpoint and evaluation

lm1b-ncp-small-released12-c60-0125-v1/checkpoint-iter-1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:693c552b8974cfc21eae70c7c4a30f38274151bd131646e66028605746363718
+size 1166859775

lm1b-ncp-small-released12-c60-0125-v1/config.yaml

@@ -0,0 +1,194 @@
+task: next-concept
+experiment: lm1b-ncp-small-released12-c60-0125-v1
+experiment_dir: /home/ubuntu/mstok-results/lm1b-ncp-small-released12-c60-0125-v1
+model:
+  n_layer: 12
+  n_head: 12
+  bias: true
+  dropout: 0.1
+  n_embd: null
+  context_length: 129
+  prefix_len: 1
+  raw_token_prefix: true
+  prefix_pad_token_id: 0
+  vocab_size:
+  - 8192
+  - 8192
+  - 8192
+  - 8192
+  - 8192
+  - 8192
+  - 8192
+  - 8192
+  - 8192
+  - 8192
+  - 8192
+  - 8192
+  predict_level0: false
+  query_level0: true
+  attn_pattern: block_diagonal
+  use_positional_encoding: true
+  use_level_encoding: true
+  use_rope: true
+  rope_base: 10000
+  use_qk_norm: true
+  use_flex_attention: false
+  ctx_encoder:
+    enabled: false
+    n_layer: 4
+    vocab_size: 30528
+  post_upsample_conv:
+    enabled: true
+    kernel_size: 3
+  shared_output_head: true
+  shared_head_per_level_bias: true
+tokenizer:
+  pre_tokenizer:
+    name: hf
+    tokenizer: hf
+    model_id: /home/ubuntu/.cache/huggingface/hub/models--google-bert--bert-base-uncased/snapshots/86b5e0934494bd15c9632b12f734a8a67f723594
+    special_tokens:
+      eos_token: '[CLS]'
+      pad_token: '[PAD]'
+  vqvae:
+    n_layers: 6
+    embed_dim: 384
+    in_vocab_size: 30528
+    context_length: 128
+    compression_factor: 1
+    dropout: 0.1
+    use_qk_norm: true
+    pre_quant_groupnorm: 4
+    pre_quant_dropout: 0.0
+    enc_positional: false
+    dec_positional: true
+    dec_layers: 4
+    vqvae_vocab_size:
+    - 8192
+    - 8192
+    - 8192
+    - 8192
+    - 8192
+    - 8192
+    - 8192
+    - 8192
+    - 8192
+    - 8192
+    - 8192
+    - 8192
+    checkpoint_path: /home/ubuntu/.cache/huggingface/hub/models--kumboh--ncp/snapshots/28ef3928fe91cf3fd28ebf5b4bef36516fe53f92/lm1b-dev/vqvae-12-level
+    vector_quantizer_config:
+      clss: multiscale_residual_vector_quantizer
+      decay: 0.99
+      epsilon: 1.0e-05
+      commitment_cost: 0.25
+      learned_l1_sampling: true
+      learned_l1_cascade: false
+      learned_l1_cascade_base: 4
+      learned_all_sampling: false
+      quant_resi:
+        enabled: true
+        ratio: 0.5
+        share_mode: 0
+        learnable_ratio: false
+      levels:
+        use_manual_levels: true
+        manual_levels:
+        - 1
+        - 8
+        - 16
+        - 24
+        - 32
+        - 40
+        - 52
+        - 64
+        - 76
+        - 92
+        - 108
+        - 128
+      aux:
+        fine_drop:
+          dual: true
+          prob: 1.0
+          min_keep: 1
+          w_full: 0.7
+          w_trunc: 0.3
+        code_corruption:
+          enabled: true
+          mode: uniform
+          random_rate: true
+          low: 0.0
+          high: 1.0
+          skip_level0: true
+dataset:
+  train_source: /home/ubuntu/data/lm1b-reference/train.bin
+  validate_source: /home/ubuntu/data/lm1b-reference/validation-prefix129.bin
+  null_prefix_len: 1
+  null_prefix_doc_aware: true
+  null_prefix_head: 101
+wandb:
+  project: ncp-lm1b
+  group: released12-baselines
+  enable: true
+  resume: allow
+  gradients_and_params:
+    enable: false
+    log: all
+    log_freq: 1000
+optimization:
+  lr: 0.0003
+  reset_lr: false
+  warmup_iters: 1000
+  lr_decay_iters: null
+  min_lr: 3.0e-05
+  beta_1: 0.9
+  beta_2: 0.95
+  weight_decay: 0.1
+  max_grad_norm: 1.0
+  grad_accumulation_steps: 1
+  decay_param_groups: true
+  token_corruption_prob: 0.35
+  token_corruption_prob_per_level:
+  - 0.0
+  - 0.6
+  - 0.5525
+  - 0.505
+  - 0.4575
+  - 0.41
+  - 0.3625
+  - 0.315
+  - 0.2675
+  - 0.22
+  - 0.1725
+  - 0.125
+  corruption_neighbour: false
+  level_loss_alpha: 0.25
+  stochastic_encode:
+    enabled: false
+    temperature: 1.0
+    top_k: 0
+    top_p: 1.0
+  ema_decay: 0.9999
+training:
+  seed: 42
+  log_interval: 50
+  eval_interval: 1000
+  checkpoint_interval: 1000
+  val_iters: 50
+  n_epochs: 6
+  total_iters: null
+  tqdm_interval: 100
+  batch_size: 128
+  eval_batch_size: 16
+  demo_batch_size: 4
+  keep_last: 100
+torch_compile:
+  enable: true
+  mode: default
+  dynamic: false
+  fullgraph: false
+  backend: inductor
+mixed_precision:
+  enable: true
+  opt_level: O1
+  loss_scale: dynamic

lm1b-ncp-small-released12-c60-0125-v1/ema-iter-1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a931fe61b18a418a21e65afb9f067bb25af89909b12f18cffa4302fd5aecdbf9
+size 390019666

lm1b-ncp-small-released12-c60-0125-v1/frozen-codec.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aab3d012916a412950f99308c8474d20f524ccd4ff27b91f7c2b83a9cd3225fb
+size 597859731

lm1b-ncp-small-released12-c60-0125-v1/launch-identity.json

@@ -0,0 +1,4 @@
+{
+  "config_sha256": "1ec75fc45056534f0285dcd9443522fe6da18c9915fb30b1386d411e2a78af47",
+  "data_manifest_sha256": "857aafa66c7e3cc2dc3d72b69a10632a711354c48e882206e0eda3d6c2ab09a1"
+}

lm1b-ncp-small-released12-c60-0125-v1/manifest.json

@@ -0,0 +1,62 @@
+{
+  "ncm_reference_commit": "29163c8fa391fe980400e6a5e2338b001e4cff31",
+  "codec_repo": "kumboh/ncp",
+  "codec_revision": "28ef3928fe91cf3fd28ebf5b4bef36516fe53f92",
+  "codec_file": "lm1b-dev/vqvae-12-level/checkpoint-iter-25957.pt",
+  "codec_step": 25957,
+  "codec_sha256": "aab3d012916a412950f99308c8474d20f524ccd4ff27b91f7c2b83a9cd3225fb",
+  "data_manifest_sha256": "857aafa66c7e3cc2dc3d72b69a10632a711354c48e882206e0eda3d6c2ab09a1",
+  "train_blocks": 7087759,
+  "global_batch": 1024,
+  "raw_positions_per_update": 131072,
+  "reference_budget_positions_per_update_including_null_prefix": 132096,
+  "n_epochs": 6,
+  "levels": [
+    1,
+    8,
+    16,
+    24,
+    32,
+    40,
+    52,
+    64,
+    76,
+    92,
+    108,
+    128
+  ],
+  "corruption": [
+    0.0,
+    0.6,
+    0.5525,
+    0.505,
+    0.4575,
+    0.41,
+    0.3625,
+    0.315,
+    0.2675,
+    0.22,
+    0.1725,
+    0.125
+  ],
+  "note": "Fresh generator; released codec differs from original rc1 ladder. No claim to reproduce the released 105.74 score.",
+  "source_commit": "4b543af901d1302f1fea25274d43c855688608b7",
+  "total_updates": 41208,
+  "total_target_positions": 5401214976,
+  "scoring_protocol": "flm-lm1b-128, with separate clean-text diagnostic",
+  "generator_parameters": 97065600,
+  "derived_data": {
+    "train_offsets.npy": {
+      "bytes": 56702208,
+      "sha256": "454624ab4bbc856a4dde582f8b3e3521ac1b03b4ee565f4ceae8b9011947d80a"
+    },
+    "validation-prefix129.bin": {
+      "bytes": 18530850,
+      "sha256": "2b053f0b46bf6badb2992ba2ac05a31aa2f306117aa1d11e89707f3803dd410f"
+    },
+    "validation-prefix129_offsets.npy": {
+      "bytes": 574736,
+      "sha256": "016c7f2e8e6d72c461edb965b88aee9c6bc98c81616d55ac02bce5d8d3bf67c4"
+    }
+  }
+}

lm1b-ncp-small-released12-c60-0125-v1/source-identity.json

@@ -0,0 +1,4 @@
+{
+  "sha256": "95ef1c1ecda53f04dd39c105b318804ccf5e95a743b31af4d4d092c2f2112b2e",
+  "commit": "4b543af901d1302f1fea25274d43c855688608b7"
+}

lm1b-ncp-small-released12-c60-0125-v1/source.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:158848e39557ee522b6c669ecc13d02284d94ebefd25f15fbf9c4a44f686329c
+size 10645022