Buckets:
| {"model": {"d_model": 896, "vocab_size": 16384, "n_layers": 18, "block": {"sequence_mixer": {"name": "default", "n_heads": 28, "n_kv_heads": 7, "head_dim": 32, "bias": false, "gate": {"granularity": "headwise", "full_precision": true, "_CLASS_": "lobotomy.modeling.nn.attention.GateConfig"}, "rope": {"name": "default", "theta": 10000, "full_precision": true, "no_global_rope": false, "hope": {"training_sequence_length": 2048, "_CLASS_": "lobotomy.modeling.nn.rope.HoPEConfig"}, "partial_rotary_factor": 1.0, "_CLASS_": "lobotomy.modeling.nn.rope.RoPEConfig"}, "qk_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "bfloat16", "_CLASS_": "lobotomy.modeling.nn.layer_norm.LayerNormConfig"}, "backend": "flex", "dtype": "bfloat16", "sliding_window": {"pattern": [128, 128, -1], "force_full_attention_on_first_layer": false, "force_full_attention_on_last_layer": true, "_CLASS_": "lobotomy.modeling.nn.attention.SlidingWindowAttentionConfig"}, "use_head_qk_norm": false, "use_attention_sink": true, "_CLASS_": "lobotomy.modeling.nn.attention.AttentionConfig", "type": "attention"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "bfloat16", "_CLASS_": "lobotomy.modeling.nn.layer_norm.LayerNormConfig"}, "feed_forward": {"hidden_size": 6144, "name": "default", "bias": false, "dtype": "bfloat16", "activation": "silu", "_CLASS_": "lobotomy.modeling.nn.feed_forward.FeedForwardConfig"}, "name": "default", "_CLASS_": "lobotomy.modeling.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "bfloat16", "_CLASS_": "lobotomy.modeling.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "bfloat16", "loss_implementation": "fused_linear", "_CLASS_": "lobotomy.modeling.nn.lm_head.LMHeadConfig"}, "name": "default", "dtype": "bfloat16", "residual_dtype": "bfloat16", "init_method": "normal", "init_seed": 0, "init_std": 0.02, "tie_word_embeddings": true, "_CLASS_": "lobotomy.modeling.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 16384, "eos_token_id": 16001, "pad_token_id": 16002, "identifier": "glossia/Rambley-Tokenizer-v1-16k", "_CLASS_": "lobotomy.modeling.tokenizer.TokenizerConfig"}, "mix": "rambley_16k/instruct-no-tools-v1", "mix_base_dir": "./work/rambley-v2-350m/data", "mix_override_roots": true, "expand_glob": false, "include_instance_metadata": true, "instance_filter_config": {"repetition_max_period": 13, "repetition_min_period": 1, "repetition_max_count": 32, "_CLASS_": "lobotomy.data.numpy_dataset.InstanceFilterConfig"}, "work_dir": "./work/rambley-v2-350m", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "max_target_sequence_length": 2048, "generate_doc_lengths": false, "_CLASS_": "lobotomy.data.numpy_dataset.NumpyFSLDatasetConfig"}, "data_loader": {"global_batch_size": 262144, "seed": 34521, "num_workers": 8, "prefetch_factor": 2048, "ignore_fingerprint_mismatch": false, "_CLASS_": "lobotomy.data.data_loader.NumpyDataLoaderConfig", "type": "numpy"}, "train_module": {"rank_microbatch_size": 65536, "max_sequence_length": 2048, "optim": {"compile": false, "fixed_fields": ["initial_lr"], "lr": 0.01, "aux_lr": 0.001, "mu": 0.95, "betas": [0.9, 0.95], "weight_decay": 0.0, "embedding_weight_decay": 0.0, "weight_decay_mode": "hyperball", "adam_weight_decay_mode": "none", "ns_steps": 5, "ns_method": "gram", "epsilon": 1e-08, "nesterov": false, "adjust_lr": "rms_norm", "_CLASS_": "lobotomy.train.optim.muon.MuonConfig", "type": "muon"}, "max_grad_norm": 1.0, "sfplus_diagnostics_interval": 0, "sfplus_diagnostics_prediction_tokens": 0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "units": "steps", "warmup_fraction": 0.1, "decay_fraction": 0.2, "warmup_min_lr": 0.0, "decay_min_lr": 0.0, "_CLASS_": "lobotomy.train.optim.scheduler.WSD", "type": "wsd"}, "compile_model": true, "compile_mode": "default", "float8_config": {"ao_recipe": "tensorwise", "enabled": true, "_CLASS_": "lobotomy.modeling.float8.Float8Config"}, "z_loss_multiplier": 1e-05, "label_ignore_index": -100, "_CLASS_": "lobotomy.train.train_module.transformer.config.TransformerTrainModuleConfig"}, "trainer": {"save_folder": "hf://buckets/glossia/checkpoints/350m-instruct", "work_dir": "./work/rambley-v2-350m", "load_path": "hf://buckets/glossia/checkpoints/350m-v2.1-midtrain4/step6500", "load_strategy": "always", "load_optim_state": false, "load_trainer_state": false, "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "lobotomy.cli.checkpoint.CheckpointerConfig"}, "save_overwrite": true, "max_duration": {"value": 1, "unit": "epochs", "_CLASS_": "lobotomy.cli.common.Duration"}, "cancel_check_interval": 10, "metrics_collect_interval": 10, "callbacks": {"config_saver": {"fname": "config.json", "_CLASS_": "lobotomy.cli.callbacks.config_saver.ConfigSaverCallback"}, "profiler": {"skip_first": 30, "wait": 1, "warmup": 2, "active": 3, "repeat": 1, "with_stack": false, "profile_memory": false, "enable_cuda_sync_events": false, "enabled": true, "_CLASS_": "lobotomy.cli.callbacks.profiler.ProfilerCallback"}, "checkpointer": {"save_interval": 500, "pre_train_checkpoint": false, "save_async": true, "remove": "ephemeral_only", "max_checkpoints": 3, "enabled": true, "_CLASS_": "lobotomy.cli.callbacks.checkpointer.CheckpointerCallback"}, "comet": {"enabled": false, "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "notifications": "none", "failure_tag": "failed", "auto_resume": false, "_CLASS_": "lobotomy.cli.callbacks.comet.CometCallback"}, "wandb": {"enabled": true, "name": "350m-instruct", "project": "rambley-v2-350m", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "_CLASS_": "lobotomy.cli.callbacks.wandb.WandBCallback"}, "runtime_report": {"expected_parameter_dtype": "torch.bfloat16", "_CLASS_": "lobotomy.cli.scripts.fizz.fizz_training.reporting.RuntimeReportCallback"}}, "bookkeeping_soft_timeout": 30, "no_checkpoints": false, "no_evals": false, "_CLASS_": "lobotomy.cli.config.TrainerConfig"}, "init_seed": 12536, "recipe": {"model_shape": "350m", "attention": "128swa", "positional_encoding": "hope", "precision": "bf16-native", "parallelism": "auto", "load_mode": "weights", "context": {"scaling": "none", "beta_fast": 32, "beta_slow": 1, "full_attn_layers_only": true, "_CLASS_": "lobotomy.cli.scripts.fizz.fizz_training.recipe.ContextRecipe"}, "data": {"mix": "rambley_16k_instruct_no_tools_v1", "root": "https://olmo-data.org", "mode": "chunk", "_CLASS_": "lobotomy.cli.scripts.fizz.fizz_training.recipe.DataRecipe"}, "download": {"train": true, "eval": true, "workers": 16, "batch_size": 256, "_CLASS_": "lobotomy.cli.scripts.fizz.fizz_training.recipe.DownloadRecipe"}, "eval": {"ppl": false, "downstream": false, "_CLASS_": "lobotomy.cli.scripts.fizz.fizz_training.recipe.EvalRecipe"}, "engram": {"_CLASS_": "lobotomy.cli.scripts.fizz.fizz_training.recipe.EngramRecipe"}, "patch": {"size": 4, "frac": 0.0, "_CLASS_": "lobotomy.cli.scripts.fizz.fizz_training.recipe.PatchRecipe"}, "_CLASS_": "lobotomy.cli.scripts.fizz.fizz_training.recipe.Recipe"}, "_CLASS_": "lobotomy.cli.scripts.fizz.fizz_training.config.FizzConfig"} |
Xet Storage Details
- Size:
- 7.24 kB
- Xet hash:
- a3136a6e191938952c1b6c6ee502df2378756b251a08ac8f8a9cc5b3330ef8d9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.