Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,4 @@
+{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

initial_latent_z.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e90527caee390bad07cd8394d4b8604bc0ebb4f0c95f0891e2443a2c16db66b
+size 4194392

metrics.json

@@ -0,0 +1,76 @@
+{
+  "phase_results": [
+    {
+      "phase": "simple_multiturn_transcript",
+      "best_selection_metric_value": 5.384137900958771,
+      "best_metrics": {
+        "validation/loss_total": 1.408942354669146,
+        "validation/loss_response": 2.149438410067044,
+        "validation/loss_current_user_reconstruction": 1.2758943989573013,
+        "validation/loss_probe": 1.2539331475071445,
+        "validation/loss_past_response_contrastive": 0.05162567709699089,
+        "validation/response_first_token_exact_match": 0.21810699588477367,
+        "validation/current-user_reconstruction_first_token_exact_match": 0.6152263374485597,
+        "validation/probe_first_token_exact_match": 1.0,
+        "validation/supervised_turn_count": 486.0,
+        "validation/response_exact_match": 0.0,
+        "validation/response_similarity": 0.10392940422656348,
+        "validation/response_token_f1": 0.20785880845312696,
+        "validation/response_line_recall": 0.0,
+        "validation/response_reconstruction_similarity": 0.4982011305164052,
+        "validation/response_reconstruction_exact_match": 0.025806451612903226,
+        "validation/response_reconstruction_token_f1": 0.4982011305164052,
+        "validation/probe_exact_match": 0.0,
+        "validation/probe_transcript_similarity": 0.2633016615969853,
+        "validation/probe_token_f1": 0.5266033231939706,
+        "validation/probe_line_recall": 0.0,
+        "validation/response_similarity_by_turn/turn_2": 0.11559769549843006,
+        "validation/response_reconstruction_similarity_by_turn/turn_2": 0.6686679695190779,
+        "validation/probe_transcript_similarity_by_turn/turn_2": 0.29072483292965323,
+        "validation/response_similarity_by_turn/turn_3": 0.10607868284934467,
+        "validation/response_reconstruction_similarity_by_turn/turn_3": 0.5215039721757743,
+        "validation/probe_transcript_similarity_by_turn/turn_3": 0.26205390129767436,
+        "validation/response_similarity_by_turn/turn_4": 0.09581244011492175,
+        "validation/response_reconstruction_similarity_by_turn/turn_4": 0.3903211526872822,
+        "validation/probe_transcript_similarity_by_turn/turn_4": 0.24065981727738503,
+        "validation/response_similarity_by_turn/turn_5": 0.08826680527651742,
+        "validation/response_reconstruction_similarity_by_turn/turn_5": 0.2735804094022488,
+        "validation/probe_transcript_similarity_by_turn/turn_5": 0.24492614677816982,
+        "validation/response_similarity_by_turn/turn_6": 0.09125961673419455,
+        "validation/response_reconstruction_similarity_by_turn/turn_6": 0.3105487240823803,
+        "validation/probe_transcript_similarity_by_turn/turn_6": 0.2404807978040937,
+        "validation/goal_loss": 5.384137900958771
+      },
+      "global_step": 805,
+      "train_dataset": {
+        "example_count": 7696,
+        "pair_count_mean": 4.2548076923076925,
+        "pair_count_max": 6,
+        "response_target_tokens_mean_turn2_plus": 126.23921517671518,
+        "response_target_tokens_max_turn2_plus": 1643,
+        "probe_target_tokens_mean_turn2_plus": 471.31613825363826,
+        "probe_target_tokens_max_turn2_plus": 3285,
+        "dataset_counts": {
+          "chatalpaca_multiturn_enriched": 7696
+        },
+        "probe_question_text": "What is everything we have talked about so far? Give exact conversation transcript verbatim in following format: [User 1]: X [Assistant 1]: Y [User 2]: A etc",
+        "current_user_reconstruction_question_text": "What did me (the user) just ask you? give verbatim message I just previously sent"
+      },
+      "validation_dataset": {
+        "example_count": 151,
+        "pair_count_mean": 4.139072847682119,
+        "pair_count_max": 6,
+        "response_target_tokens_mean_turn2_plus": 123.21854304635761,
+        "response_target_tokens_max_turn2_plus": 889,
+        "probe_target_tokens_mean_turn2_plus": 448.5298013245033,
+        "probe_target_tokens_max_turn2_plus": 1545,
+        "dataset_counts": {
+          "chatalpaca_multiturn_enriched": 151
+        },
+        "probe_question_text": "What is everything we have talked about so far? Give exact conversation transcript verbatim in following format: [User 1]: X [Assistant 1]: Y [User 2]: A etc",
+        "current_user_reconstruction_question_text": "What did me (the user) just ask you? give verbatim message I just previously sent"
+      },
+      "wandb_enabled": true
+    }
+  ]
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d2252e8af791fcaa666ea08fafd5a71bc70a9ab0144780db4e00404974b5c0b
+size 2521506792

sft_b200.yaml

@@ -0,0 +1,132 @@
+datasets:
+  - name: chatalpaca_multiturn_enriched
+    repo_id: BRlkl/chatalpaca-multiturn-enriched
+    source_split: train
+    format: messages_all_turns
+    validation_ratio: 0.02
+    split_seed: 17
+    min_turns: 2
+    max_turns: 6
+    max_message_chars: 6000
+    use_base_chat_template: true
+
+model:
+  base_model_name: google/t5gemma-l-l-prefixlm-it
+  initial_model_path: BRlkl/test_multiturn_simple_transcript1024_3
+  dtype: bfloat16
+  attn_implementation: sdpa
+  disable_cudnn_sdp: true
+  disable_mha_fastpath: true
+  magicnorm_eps: 1.0e-6
+  z_slots: 1024
+  num_time_tokens: 0
+  use_explicit_time_features: false
+  gate_attention_heads: 4
+  max_observation_tokens: 1024
+  max_decoder_tokens: 1024
+  thought_loop_proposal_mode: observation_hidden_compression
+  preserve_observation_encoder_manifold: true
+  observation_encoder_use_state_context: true
+  latent_attention_mask_mode: full
+  hard_state_replace: true
+
+training:
+  seed: 17
+  num_workers: 2
+  gradient_checkpointing: true
+  mixed_precision: bf16
+  max_grad_norm: 1.0
+  weight_decay: 0.01
+  backbone_learning_rate: 5.0e-6
+  new_module_learning_rate: 1.0e-4
+  adam_beta1: 0.9
+  adam_beta2: 0.95
+  adam_epsilon: 1.0e-8
+  fused_adamw: true
+  freeze_gate_head: true
+  assistant_feedback_mode: teacher_forced
+  log_every_steps: 1
+  eval_every_steps: 100
+  checkpoint_every_steps: 500
+  eval_max_batches: 16
+  validation_behavior_max_batches: 4
+  max_train_examples:
+  max_validation_examples:
+
+  response_loss_weight: 0.3
+  current_user_reconstruction_loss_weight: 0.3
+  probe_loss_weight: 0.3
+  past_response_contrastive_loss_weight: 0.1
+
+  enable_past_response_contrastive_loss: true
+  past_response_contrastive_margin: 0.1
+  past_response_contrastive_pool:
+    include_previous_assistant_responses: true
+    include_current_user_reconstruction_target: true
+    include_previous_user_reconstruction_targets: false
+    include_current_transcript_probe_target: true
+    include_previous_transcript_probe_targets: false
+  probe_question_text: "What is everything we have talked about so far? Give exact conversation transcript verbatim in following format: [User 1]: X [Assistant 1]: Y [User 2]: A etc"
+  current_user_reconstruction_question_text: "What did me (the user) just ask you? give verbatim message I just previously sent"
+
+  feedback_generation_max_new_tokens: 1024
+  feedback_generation_extra_new_tokens: 16
+  validation_response_max_new_tokens: 1024
+  validation_response_extra_new_tokens: 16
+  validation_probe_max_new_tokens: 1024
+  validation_probe_extra_new_tokens: 16
+
+  wandb_train_metric_keys:
+    - train/loss_total
+    - train/loss_response
+    - train/loss_current_user_reconstruction
+    - train/loss_probe
+    - train/loss_past_response_contrastive
+    - train/response_first_token_exact_match
+    - train/current-user_reconstruction_first_token_exact_match
+    - train/probe_first_token_exact_match
+  wandb_validation_metric_keys:
+    - validation/loss_total
+    - validation/loss_response
+    - validation/loss_current_user_reconstruction
+    - validation/loss_probe
+    - validation/loss_past_response_contrastive
+    - validation/goal_loss
+    - validation/response_similarity
+    - validation/response_reconstruction_similarity
+    - validation/probe_transcript_similarity
+
+  checkpoint_selection_metric: validation/goal_loss
+  checkpoint_selection_mode: min
+  validation_response_exact_miss_penalty: 1.0
+  validation_reconstruction_similarity_miss_penalty: 1.0
+  validation_probe_exact_miss_penalty: 1.0
+  validation_probe_similarity_miss_penalty: 2.0
+
+phase:
+  micro_batch_size: 6
+  eval_batch_size: 6
+  gradient_accumulation_steps: 4
+  num_train_epochs: 5
+  warmup_ratio: 0.03
+  shuffle_train: true
+
+cache:
+  preprocessed_root: cache/preprocessed_pre_sft_multiturn_simple_transcript
+
+paths:
+  run_root: runs_pre_sft_multiturn_simple_transcript
+  export_root: exports_multiturn_simple_transcript
+
+inference:
+  format: predictive_state_multiturn
+  use_base_chat_template: true
+
+wandb:
+  enabled: true
+  project: samantha-pre-sft
+  run_name: t5gemma2-thoughtloop-pre-sft-simple-transcript
+
+hub:
+  model_repo_id: BRlkl/test_multiturn_simple_transcript1024_4
+  private: false

sft_config.json

@@ -0,0 +1,139 @@
+{
+  "datasets": [
+    {
+      "name": "chatalpaca_multiturn_enriched",
+      "repo_id": "BRlkl/chatalpaca-multiturn-enriched",
+      "source_split": "train",
+      "format": "messages_all_turns",
+      "validation_ratio": 0.02,
+      "split_seed": 17,
+      "min_turns": 2,
+      "max_turns": 6,
+      "max_message_chars": 6000,
+      "use_base_chat_template": true
+    }
+  ],
+  "model": {
+    "base_model_name": "google/t5gemma-l-l-prefixlm-it",
+    "initial_model_path": "BRlkl/test_multiturn_simple_transcript1024_3",
+    "dtype": "bfloat16",
+    "attn_implementation": "sdpa",
+    "disable_cudnn_sdp": true,
+    "disable_mha_fastpath": true,
+    "magicnorm_eps": 1e-06,
+    "z_slots": 1024,
+    "num_time_tokens": 0,
+    "use_explicit_time_features": false,
+    "gate_attention_heads": 4,
+    "max_observation_tokens": 1024,
+    "max_decoder_tokens": 1024,
+    "thought_loop_proposal_mode": "observation_hidden_compression",
+    "preserve_observation_encoder_manifold": true,
+    "observation_encoder_use_state_context": true,
+    "latent_attention_mask_mode": "full",
+    "hard_state_replace": true,
+    "gate_decision_temperature": 1.0,
+    "initial_gate_decision_threshold_logit": 0.0
+  },
+  "training": {
+    "seed": 17,
+    "num_workers": 2,
+    "gradient_checkpointing": true,
+    "mixed_precision": "bf16",
+    "max_grad_norm": 1.0,
+    "weight_decay": 0.01,
+    "backbone_learning_rate": 5e-06,
+    "new_module_learning_rate": 0.0001,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.95,
+    "adam_epsilon": 1e-08,
+    "fused_adamw": true,
+    "freeze_gate_head": true,
+    "assistant_feedback_mode": "teacher_forced",
+    "log_every_steps": 1,
+    "eval_every_steps": 100,
+    "checkpoint_every_steps": 500,
+    "eval_max_batches": 16,
+    "validation_behavior_max_batches": 4,
+    "max_train_examples": null,
+    "max_validation_examples": null,
+    "response_loss_weight": 0.3,
+    "current_user_reconstruction_loss_weight": 0.3,
+    "probe_loss_weight": 0.3,
+    "past_response_contrastive_loss_weight": 0.1,
+    "enable_past_response_contrastive_loss": true,
+    "past_response_contrastive_margin": 0.1,
+    "past_response_contrastive_pool": {
+      "include_previous_assistant_responses": false,
+      "include_current_user_reconstruction_target": true,
+      "include_previous_user_reconstruction_targets": false,
+      "include_current_transcript_probe_target": true,
+      "include_previous_transcript_probe_targets": false
+    },
+    "probe_question_text": "What is everything we have talked about so far? Give exact conversation transcript verbatim in following format: [User 1]: X [Assistant 1]: Y [User 2]: A etc",
+    "current_user_reconstruction_question_text": "What did me (the user) just ask you? give verbatim message I just previously sent",
+    "feedback_generation_max_new_tokens": 1024,
+    "feedback_generation_extra_new_tokens": 16,
+    "validation_response_max_new_tokens": 1024,
+    "validation_response_extra_new_tokens": 16,
+    "validation_probe_max_new_tokens": 1024,
+    "validation_probe_extra_new_tokens": 16,
+    "wandb_train_metric_keys": [
+      "train/loss_total",
+      "train/loss_response",
+      "train/loss_current_user_reconstruction",
+      "train/loss_probe",
+      "train/loss_past_response_contrastive",
+      "train/response_first_token_exact_match",
+      "train/current-user_reconstruction_first_token_exact_match",
+      "train/probe_first_token_exact_match"
+    ],
+    "wandb_validation_metric_keys": [
+      "validation/loss_total",
+      "validation/loss_response",
+      "validation/loss_current_user_reconstruction",
+      "validation/loss_probe",
+      "validation/loss_past_response_contrastive",
+      "validation/goal_loss",
+      "validation/response_similarity",
+      "validation/response_reconstruction_similarity",
+      "validation/probe_transcript_similarity"
+    ],
+    "checkpoint_selection_metric": "validation/goal_loss",
+    "checkpoint_selection_mode": "min",
+    "validation_response_exact_miss_penalty": 1.0,
+    "validation_reconstruction_similarity_miss_penalty": 1.0,
+    "validation_probe_exact_miss_penalty": 1.0,
+    "validation_probe_similarity_miss_penalty": 2.0
+  },
+  "phase": {
+    "micro_batch_size": 6,
+    "eval_batch_size": 6,
+    "gradient_accumulation_steps": 4,
+    "num_train_epochs": 5,
+    "warmup_ratio": 0.03,
+    "shuffle_train": true
+  },
+  "cache": {
+    "preprocessed_root": "cache/preprocessed_pre_sft_multiturn_simple_transcript"
+  },
+  "paths": {
+    "run_root": "runs_pre_sft_multiturn_simple_transcript",
+    "export_root": "exports_multiturn_simple_transcript"
+  },
+  "inference": {
+    "format": "predictive_state_multiturn",
+    "use_base_chat_template": true,
+    "gate_probability_threshold": 0.5,
+    "learned_gate_decision_threshold_logit": 0.0
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "samantha-pre-sft",
+    "run_name": "t5gemma2-thoughtloop-pre-sft-simple-transcript"
+  },
+  "hub": {
+    "model_repo_id": "BRlkl/test_multiturn_simple_transcript1024_4",
+    "private": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:561e31602d41b5cf3a4a41197db54761c3b898ac3147cbcb2a9c5094b668cbba
+size 34362527

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}