| { |
| "seed": 42, |
| "dataset": "roneneldan/TinyStories", |
| "sample_documents": 100000, |
| "split_document_counts": { |
| "train": 90000, |
| "validation": 5000, |
| "test": 5000 |
| }, |
| "sample_sha256": "86725db033dd757ef19af2bc4156f4a85309ccd59d23542553ecb0380bf5d35c", |
| "train_tokenizer_only": true, |
| "parameters": 73461120, |
| "architecture": { |
| "vocab_size": 8192, |
| "max_position_embeddings": 512, |
| "hidden_size": 640, |
| "intermediate_size": 1728, |
| "num_hidden_layers": 15, |
| "num_attention_heads": 10, |
| "num_key_value_heads": 5, |
| "hidden_act": "silu", |
| "initializer_range": 0.02, |
| "rms_norm_eps": 1e-05, |
| "pretraining_tp": 1, |
| "use_cache": false, |
| "rope_theta": 10000.0, |
| "rope_scaling": null, |
| "attention_bias": false, |
| "attention_dropout": 0.0, |
| "mlp_bias": false, |
| "head_dim": 64, |
| "return_dict": true, |
| "output_hidden_states": false, |
| "output_attentions": false, |
| "torchscript": false, |
| "torch_dtype": "float32", |
| "use_bfloat16": false, |
| "tf_legacy_loss": false, |
| "pruned_heads": {}, |
| "tie_word_embeddings": true, |
| "chunk_size_feed_forward": 0, |
| "is_encoder_decoder": false, |
| "is_decoder": false, |
| "cross_attention_hidden_size": null, |
| "add_cross_attention": false, |
| "tie_encoder_decoder": false, |
| "max_length": 20, |
| "min_length": 0, |
| "do_sample": false, |
| "early_stopping": false, |
| "num_beams": 1, |
| "num_beam_groups": 1, |
| "diversity_penalty": 0.0, |
| "temperature": 1.0, |
| "top_k": 50, |
| "top_p": 1.0, |
| "typical_p": 1.0, |
| "repetition_penalty": 1.0, |
| "length_penalty": 1.0, |
| "no_repeat_ngram_size": 0, |
| "encoder_no_repeat_ngram_size": 0, |
| "bad_words_ids": null, |
| "num_return_sequences": 1, |
| "output_scores": false, |
| "return_dict_in_generate": false, |
| "forced_bos_token_id": null, |
| "forced_eos_token_id": null, |
| "remove_invalid_values": false, |
| "exponential_decay_length_penalty": null, |
| "suppress_tokens": null, |
| "begin_suppress_tokens": null, |
| "architectures": [ |
| "LlamaForCausalLM" |
| ], |
| "finetuning_task": null, |
| "id2label": { |
| "0": "LABEL_0", |
| "1": "LABEL_1" |
| }, |
| "label2id": { |
| "LABEL_0": 0, |
| "LABEL_1": 1 |
| }, |
| "tokenizer_class": null, |
| "prefix": null, |
| "bos_token_id": 1, |
| "pad_token_id": 0, |
| "eos_token_id": 2, |
| "sep_token_id": null, |
| "decoder_start_token_id": null, |
| "task_specific_params": null, |
| "problem_type": null, |
| "_name_or_path": "", |
| "_attn_implementation_autoset": false, |
| "transformers_version": "4.46.3", |
| "model_type": "llama" |
| }, |
| "context_length": 256, |
| "micro_batch_size": 4, |
| "gradient_accumulation": 4, |
| "successful_updates": 5000, |
| "input_tokens_seen": 20480000, |
| "targets_seen": 20400000, |
| "full_target_tokens": 1469222400, |
| "learning_rate": 0.0003, |
| "warmup_steps": 250, |
| "weight_decay": 0.1, |
| "adam_betas": [ |
| 0.9, |
| 0.95 |
| ], |
| "baseline_validation_subset": { |
| "loss": 9.106077814102173, |
| "perplexity": 9009.887046841124, |
| "targets": 20400 |
| }, |
| "final_validation": { |
| "loss": 2.0363285289724113, |
| "perplexity": 7.662425120194374, |
| "targets": 1062330 |
| }, |
| "final_test": { |
| "loss": 2.0470112522395434, |
| "perplexity": 7.7447194685618435, |
| "targets": 1062075 |
| }, |
| "before": "Once upon a time, a little robot found a book.oupoupoup Abi Abi Abi Abi Abi Abi Abi Abi Abi Abi Abi Abi noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise noise", |
| "after": "Once upon a time, a little robot found a book. The book was very pretty and had a picture of a princess on it. The robot was very happy and played with the book all day long. One day, the robot saw a little girl who was sad. The robot wanted to help the little girl. The robot asked the little girl, \"Can", |
| "versions": { |
| "torch": "2.14.1+cu130", |
| "transformers": "4.46.3", |
| "datasets": "3.1.0", |
| "tokenizers": "0.20.3" |
| } |
| } |
