| --- |
| base_model: google/gemma-2b |
| pipeline_tag: text-generation |
| library_name: peft |
| --- |
| |
| # Model Card for Model ID |
|
|
| Fine-tuned on python |
|
|
|
|
| ## Model Details |
|
|
| ### Model Description |
|
|
|
|
| ### Model Sources [optional] |
| Gemma-2b trained on python-oasst dataset |
|
|
| ## Uses |
|
|
|
|
| ## Training Details |
|
|
| ### Training Data |
|
|
| { |
| "_timestamp": 1711018613.433522, |
| "train/grad_norm": 0.1240904619429259, |
| "train/global_step": 232, |
| "eval/steps_per_second": 2.894, |
| "_step": 232, |
| "_runtime": 2545.4226660728455, |
| "eval/loss": 1.189491629600525, |
| "eval/runtime": 1805.8574, |
| "train/learning_rate": 0.000014800637958532697, |
| "eval/samples_per_second": 23.152, |
| "_wandb.runtime": 2547, |
| "train/loss": 1.0436, |
| "train/epoch": 0.01 |
| } |
| |
| ### Results |
| |
| |
| #### Summary |
| |
| |
| |
| |
| ## Technical Specifications [optional] |
| { |
| "bf16": { |
| "desc": null, |
| "value": true |
| }, |
| "fp16": { |
| "desc": null, |
| "value": false |
| }, |
| "fsdp": { |
| "desc": null, |
| "value": [] |
| }, |
| "seed": { |
| "desc": null, |
| "value": 42 |
| }, |
| "tf32": { |
| "desc": null, |
| "value": false |
| }, |
| "debug": { |
| "desc": null, |
| "value": [] |
| }, |
| "optim": { |
| "desc": null, |
| "value": "adamw_bnb_8bit" |
| }, |
| "qlora": { |
| "desc": null, |
| "value": true |
| }, |
| "top_k": { |
| "desc": null, |
| "value": 50 |
| }, |
| "top_p": { |
| "desc": null, |
| "value": 1 |
| }, |
| "_wandb": { |
| "desc": null, |
| "value": { |
| "m": [ |
| { |
| "1": "train/global_step", |
| "6": [ |
| 3 |
| ] |
| }, |
| { |
| "1": "train/loss", |
| "5": 1, |
| "6": [ |
| 1 |
| ] |
| }, |
| { |
| "1": "train/grad_norm", |
| "5": 1, |
| "6": [ |
| 1 |
| ] |
| }, |
| { |
| "1": "train/learning_rate", |
| "5": 1, |
| "6": [ |
| 1 |
| ] |
| }, |
| { |
| "1": "train/epoch", |
| "5": 1, |
| "6": [ |
| 1 |
| ] |
| }, |
| { |
| "1": "eval/loss", |
| "5": 1, |
| "6": [ |
| 1 |
| ] |
| }, |
| { |
| "1": "eval/runtime", |
| "5": 1, |
| "6": [ |
| 1 |
| ] |
| }, |
| { |
| "1": "eval/samples_per_second", |
| "5": 1, |
| "6": [ |
| 1 |
| ] |
| }, |
| { |
| "1": "eval/steps_per_second", |
| "5": 1, |
| "6": [ |
| 1 |
| ] |
| } |
| ], |
| "t": { |
| "1": [ |
| 1, |
| 5, |
| 11, |
| 49, |
| 51, |
| 53, |
| 55, |
| 71, |
| 84, |
| 98, |
| 99, |
| 100, |
| 105 |
| ], |
| "2": [ |
| 1, |
| 5, |
| 11, |
| 49, |
| 51, |
| 53, |
| 55, |
| 71, |
| 84, |
| 98, |
| 99, |
| 100, |
| 105 |
| ], |
| "3": [ |
| 3, |
| 7, |
| 23 |
| ], |
| "4": "3.10.13", |
| "5": "0.16.4", |
| "6": "4.39.0.dev0", |
| "8": [ |
| 5 |
| ], |
| "9": { |
| "1": "transformers_trainer" |
| }, |
| "13": "linux-x86_64" |
| }, |
| "framework": "huggingface", |
| "start_time": 1711016068, |
| "cli_version": "0.16.4", |
| "is_jupyter_run": false, |
| "python_version": "3.10.13", |
| "is_kaggle_kernel": false, |
| "huggingface_version": "4.39.0.dev0" |
| } |
| }, |
| "prefix": { |
| "desc": null, |
| "value": null |
| }, |
| "do_eval": { |
| "desc": null, |
| "value": true |
| }, |
| "no_cuda": { |
| "desc": null, |
| "value": false |
| }, |
| "use_cpu": { |
| "desc": null, |
| "value": false |
| }, |
| "do_train": { |
| "desc": null, |
| "value": false |
| }, |
| "head_dim": { |
| "desc": null, |
| "value": 256 |
| }, |
| "id2label": { |
| "desc": null, |
| "value": { |
| "0": "LABEL_0", |
| "1": "LABEL_1" |
| } |
| }, |
| "label2id": { |
| "desc": null, |
| "value": { |
| "LABEL_0": 0, |
| "LABEL_1": 1 |
| } |
| }, |
| "run_name": { |
| "desc": null, |
| "value": "./out" |
| }, |
| "use_ipex": { |
| "desc": null, |
| "value": false |
| }, |
| "adafactor": { |
| "desc": null, |
| "value": false |
| }, |
| "data_seed": { |
| "desc": null, |
| "value": null |
| }, |
| "deepspeed": { |
| "desc": null, |
| "value": "deepspeed_configs/zero1.json" |
| }, |
| "do_sample": { |
| "desc": null, |
| "value": false |
| }, |
| "hub_token": { |
| "desc": null, |
| }, |
| "log_level": { |
| "desc": null, |
| "value": "passive" |
| }, |
| "max_steps": { |
| "desc": null, |
| "value": -1 |
| }, |
| "num_beams": { |
| "desc": null, |
| "value": 1 |
| }, |
| "ray_scope": { |
| "desc": null, |
| "value": "last" |
| }, |
| "report_to": { |
| "desc": null, |
| "value": [ |
| "wandb" |
| ] |
| }, |
| "typical_p": { |
| "desc": null, |
| "value": 1 |
| }, |
| "use_cache": { |
| "desc": null, |
| "value": false |
| }, |
| "adam_beta1": { |
| "desc": null, |
| "value": 0.9 |
| }, |
| "adam_beta2": { |
| "desc": null, |
| "value": 0.999 |
| }, |
| "do_predict": { |
| "desc": null, |
| "value": false |
| }, |
| "eval_delay": { |
| "desc": null, |
| "value": 0 |
| }, |
| "eval_steps": { |
| "desc": null, |
| "value": 0.03125 |
| }, |
| "hidden_act": { |
| "desc": null, |
| "value": "gelu" |
| }, |
| "is_decoder": { |
| "desc": null, |
| "value": false |
| }, |
| "local_rank": { |
| "desc": null, |
| "value": 0 |
| }, |
| "max_length": { |
| "desc": null, |
| "value": 20 |
| }, |
| "min_length": { |
| "desc": null, |
| "value": 0 |
| }, |
| "model_type": { |
| "desc": null, |
| "value": "gemma" |
| }, |
| "optim_args": { |
| "desc": null, |
| "value": null |
| }, |
| "orpo_alpha": { |
| "desc": null, |
| "value": null |
| }, |
| "output_dir": { |
| "desc": null, |
| "value": "./out" |
| }, |
| "past_index": { |
| "desc": null, |
| "value": -1 |
| }, |
| "rope_theta": { |
| "desc": null, |
| "value": 10000 |
| }, |
| "save_steps": { |
| "desc": null, |
| "value": 0.125 |
| }, |
| "vocab_size": { |
| "desc": null, |
| "value": 256000 |
| }, |
| "bench_split": { |
| "desc": null, |
| "value": "eval" |
| }, |
| "ddp_backend": { |
| "desc": null, |
| "value": null |
| }, |
| "ddp_timeout": { |
| "desc": null, |
| "value": 1800 |
| }, |
| "fsdp_config": { |
| "desc": null, |
| "value": { |
| "xla": false, |
| "xla_fsdp_v2": false, |
| "min_num_params": 0, |
| "xla_fsdp_grad_ckpt": false |
| } |
| }, |
| "hidden_size": { |
| "desc": null, |
| "value": 2048 |
| }, |
| "label_names": { |
| "desc": null, |
| "value": null |
| }, |
| "logging_dir": { |
| "desc": null, |
| "value": "./out/runs/Mar21_10-14-24_8205afe3ecd2" |
| }, |
| "pretraining": { |
| "desc": null, |
| "value": false |
| }, |
| "push_to_hub": { |
| "desc": null, |
| "value": false |
| }, |
| "return_dict": { |
| "desc": null, |
| "value": true |
| }, |
| "temperature": { |
| "desc": null, |
| "value": 1 |
| }, |
| "torch_dtype": { |
| "desc": null, |
| "value": "bfloat16" |
| }, |
| "torchdynamo": { |
| "desc": null, |
| "value": null |
| }, |
| "torchscript": { |
| "desc": null, |
| "value": false |
| }, |
| "adam_epsilon": { |
| "desc": null, |
| "value": 1e-8 |
| }, |
| "bos_token_id": { |
| "desc": null, |
| "value": 2 |
| }, |
| "disable_tqdm": { |
| "desc": null, |
| "value": false |
| }, |
| "eos_token_id": { |
| "desc": null, |
| "value": 1 |
| }, |
| "fp16_backend": { |
| "desc": null, |
| "value": "auto" |
| }, |
| "hub_model_id": { |
| "desc": null, |
| "value": null |
| }, |
| "hub_strategy": { |
| "desc": null, |
| "value": "every_save" |
| }, |
| "pad_token_id": { |
| "desc": null, |
| "value": 0 |
| }, |
| "problem_type": { |
| "desc": null, |
| "value": null |
| }, |
| "pruned_heads": { |
| "desc": null, |
| "value": {} |
| }, |
| "relora_steps": { |
| "desc": null, |
| "value": null |
| }, |
| "rms_norm_eps": { |
| "desc": null, |
| "value": 0.000001 |
| }, |
| "rope_scaling": { |
| "desc": null, |
| "value": null |
| }, |
| "sep_token_id": { |
| "desc": null, |
| "value": null |
| }, |
| "use_bfloat16": { |
| "desc": null, |
| "value": false |
| }, |
| "warmup_ratio": { |
| "desc": null, |
| "value": 0 |
| }, |
| "warmup_steps": { |
| "desc": null, |
| "value": 3135 |
| }, |
| "weight_decay": { |
| "desc": null, |
| "value": 0 |
| }, |
| "_name_or_path": { |
| "desc": null, |
| "value": "dvdmrs09/gemma2b-train" |
| }, |
| "architectures": { |
| "desc": null, |
| "value": [ |
| "GemmaForCausalLM" |
| ] |
| }, |
| "bad_words_ids": { |
| "desc": null, |
| "value": null |
| }, |
| "bench_dataset": { |
| "desc": null, |
| "value": "pharaouk/dharma-1/dharma_1_mini.json" |
| }, |
| "do_bench_eval": { |
| "desc": null, |
| "value": false |
| }, |
| "jit_mode_eval": { |
| "desc": null, |
| "value": false |
| }, |
| "learning_rate": { |
| "desc": null, |
| "value": 0.0002 |
| }, |
| "logging_steps": { |
| "desc": null, |
| "value": 1 |
| }, |
| "max_grad_norm": { |
| "desc": null, |
| "value": 1 |
| }, |
| "mp_parameters": { |
| "desc": null, |
| "value": "" |
| }, |
| "output_scores": { |
| "desc": null, |
| "value": false |
| }, |
| "save_strategy": { |
| "desc": null, |
| "value": "steps" |
| }, |
| "split_batches": { |
| "desc": null, |
| "value": null |
| }, |
| "torch_compile": { |
| "desc": null, |
| "value": false |
| }, |
| "tpu_num_cores": { |
| "desc": null, |
| "value": null |
| }, |
| "attention_bias": { |
| "desc": null, |
| "value": false |
| }, |
| "bf16_full_eval": { |
| "desc": null, |
| "value": false |
| }, |
| "early_stopping": { |
| "desc": null, |
| "value": false |
| }, |
| "fp16_full_eval": { |
| "desc": null, |
| "value": false |
| }, |
| "fp16_opt_level": { |
| "desc": null, |
| "value": "O1" |
| }, |
| "length_penalty": { |
| "desc": null, |
| "value": 1 |
| }, |
| "max_seq_length": { |
| "desc": null, |
| "value": 4096 |
| }, |
| "sample_packing": { |
| "desc": null, |
| "value": false |
| }, |
| "tf_legacy_loss": { |
| "desc": null, |
| "value": false |
| }, |
| "use_mps_device": { |
| "desc": null, |
| "value": false |
| }, |
| "finetuning_task": { |
| "desc": null, |
| "value": null |
| }, |
| "group_by_length": { |
| "desc": null, |
| "value": false |
| }, |
| "hub_always_push": { |
| "desc": null, |
| "value": false |
| }, |
| "num_beam_groups": { |
| "desc": null, |
| "value": 1 |
| }, |
| "save_only_model": { |
| "desc": null, |
| "value": false |
| }, |
| "suppress_tokens": { |
| "desc": null, |
| "value": null |
| }, |
| "tokenizer_class": { |
| "desc": null, |
| "value": null |
| }, |
| "dispatch_batches": { |
| "desc": null, |
| "value": null |
| }, |
| "full_determinism": { |
| "desc": null, |
| "value": false |
| }, |
| "hub_private_repo": { |
| "desc": null, |
| "value": false |
| }, |
| "ignore_data_skip": { |
| "desc": null, |
| "value": false |
| }, |
| "log_on_each_node": { |
| "desc": null, |
| "value": true |
| }, |
| "logging_strategy": { |
| "desc": null, |
| "value": "steps" |
| }, |
| "num_train_epochs": { |
| "desc": null, |
| "value": 8 |
| }, |
| "save_safetensors": { |
| "desc": null, |
| "value": true |
| }, |
| "save_total_limit": { |
| "desc": null, |
| "value": 4 |
| }, |
| "attention_dropout": { |
| "desc": null, |
| "value": 0 |
| }, |
| "ddp_bucket_cap_mb": { |
| "desc": null, |
| "value": null |
| }, |
| "diversity_penalty": { |
| "desc": null, |
| "value": 0 |
| }, |
| "do_causal_lm_eval": { |
| "desc": null, |
| "value": false |
| }, |
| "greater_is_better": { |
| "desc": null, |
| "value": false |
| }, |
| "initializer_range": { |
| "desc": null, |
| "value": 0.02 |
| }, |
| "intermediate_size": { |
| "desc": null, |
| "value": 16384 |
| }, |
| "log_level_replica": { |
| "desc": null, |
| "value": "warning" |
| }, |
| "loraplus_lr_ratio": { |
| "desc": null, |
| "value": null |
| }, |
| "lr_scheduler_type": { |
| "desc": null, |
| "value": "cosine" |
| }, |
| "max_bench_samples": { |
| "desc": null, |
| "value": null |
| }, |
| "num_hidden_layers": { |
| "desc": null, |
| "value": 18 |
| }, |
| "output_attentions": { |
| "desc": null, |
| "value": false |
| }, |
| "push_to_hub_token": { |
| "desc": null, |
| "value": "<PUSH_TO_HUB_TOKEN>" |
| }, |
| "save_on_each_node": { |
| "desc": null, |
| "value": false |
| }, |
| "tpu_metrics_debug": { |
| "desc": null, |
| "value": false |
| }, |
| "accelerator_config": { |
| "desc": null, |
| "value": { |
| "even_batches": true, |
| "split_batches": false, |
| "dispatch_batches": null, |
| "use_seedable_sampler": true |
| } |
| }, |
| "is_encoder_decoder": { |
| "desc": null, |
| "value": false |
| }, |
| "length_column_name": { |
| "desc": null, |
| "value": "length" |
| }, |
| "logging_first_step": { |
| "desc": null, |
| "value": false |
| }, |
| "relora_prune_ratio": { |
| "desc": null, |
| "value": 0.9 |
| }, |
| "repetition_penalty": { |
| "desc": null, |
| "value": 1 |
| }, |
| "torch_compile_mode": { |
| "desc": null, |
| "value": null |
| }, |
| "add_cross_attention": { |
| "desc": null, |
| "value": false |
| }, |
| "cosine_min_lr_ratio": { |
| "desc": null, |
| "value": null |
| }, |
| "eval_sample_packing": { |
| "desc": null, |
| "value": false |
| }, |
| "evaluation_strategy": { |
| "desc": null, |
| "value": "steps" |
| }, |
| "forced_bos_token_id": { |
| "desc": null, |
| "value": null |
| }, |
| "forced_eos_token_id": { |
| "desc": null, |
| "value": null |
| }, |
| "fsdp_min_num_params": { |
| "desc": null, |
| "value": 0 |
| }, |
| "lr_quadratic_warmup": { |
| "desc": null, |
| "value": false |
| }, |
| "lr_scheduler_kwargs": { |
| "desc": null, |
| "value": {} |
| }, |
| "neftune_noise_alpha": { |
| "desc": null, |
| "value": null |
| }, |
| "num_attention_heads": { |
| "desc": null, |
| "value": 8 |
| }, |
| "num_key_value_heads": { |
| "desc": null, |
| "value": 1 |
| }, |
| "quantization_config": { |
| "desc": null, |
| "value": { |
| "load_in_4bit": true, |
| "load_in_8bit": false, |
| "quant_method": "QuantizationMethod.BITS_AND_BYTES", |
| "_load_in_4bit": true, |
| "_load_in_8bit": false, |
| "llm_int8_threshold": 6, |
| "bnb_4bit_quant_type": "nf4", |
| "llm_int8_skip_modules": null, |
| "bnb_4bit_compute_dtype": "bfloat16", |
| "bnb_4bit_quant_storage": "uint8", |
| "llm_int8_has_fp16_weight": false, |
| "bnb_4bit_use_double_quant": true, |
| "llm_int8_enable_fp32_cpu_offload": false |
| } |
| }, |
| "relora_anneal_steps": { |
| "desc": null, |
| "value": null |
| }, |
| "relora_warmup_steps": { |
| "desc": null, |
| "value": null |
| }, |
| "skip_memory_metrics": { |
| "desc": null, |
| "value": true |
| }, |
| "tie_encoder_decoder": { |
| "desc": null, |
| "value": false |
| }, |
| "tie_word_embeddings": { |
| "desc": null, |
| "value": true |
| }, |
| "auto_find_batch_size": { |
| "desc": null, |
| "value": false |
| }, |
| "bench_source_max_len": { |
| "desc": null, |
| "value": 2048 |
| }, |
| "dataloader_drop_last": { |
| "desc": null, |
| "value": false |
| }, |
| "no_repeat_ngram_size": { |
| "desc": null, |
| "value": 0 |
| }, |
| "num_return_sequences": { |
| "desc": null, |
| "value": 1 |
| }, |
| "optim_target_modules": { |
| "desc": null, |
| "value": null |
| }, |
| "output_hidden_states": { |
| "desc": null, |
| "value": false |
| }, |
| "overwrite_output_dir": { |
| "desc": null, |
| "value": false |
| }, |
| "prediction_loss_only": { |
| "desc": null, |
| "value": false |
| }, |
| "push_to_hub_model_id": { |
| "desc": null, |
| "value": null |
| }, |
| "task_specific_params": { |
| "desc": null, |
| "value": null |
| }, |
| "transformers_version": { |
| "desc": null, |
| "value": "4.39.0.dev0" |
| }, |
| "begin_suppress_tokens": { |
| "desc": null, |
| "value": null |
| }, |
| "dataloader_pin_memory": { |
| "desc": null, |
| "value": true |
| }, |
| "ddp_broadcast_buffers": { |
| "desc": null, |
| "value": null |
| }, |
| "loraplus_lr_embedding": { |
| "desc": null, |
| "value": null |
| }, |
| "metric_for_best_model": { |
| "desc": null, |
| "value": "loss" |
| }, |
| "remove_invalid_values": { |
| "desc": null, |
| "value": false |
| }, |
| "remove_unused_columns": { |
| "desc": null, |
| "value": true |
| }, |
| "torch_compile_backend": { |
| "desc": null, |
| "value": null |
| }, |
| "dataloader_num_workers": { |
| "desc": null, |
| "value": 0 |
| }, |
| "decoder_start_token_id": { |
| "desc": null, |
| "value": null |
| }, |
| "gradient_checkpointing": { |
| "desc": null, |
| "value": true |
| }, |
| "half_precision_backend": { |
| "desc": null, |
| "value": "auto" |
| }, |
| "label_smoothing_factor": { |
| "desc": null, |
| "value": 0 |
| }, |
| "load_best_model_at_end": { |
| "desc": null, |
| "value": true |
| }, |
| "logging_nan_inf_filter": { |
| "desc": null, |
| "value": true |
| }, |
| "multipack_real_batches": { |
| "desc": null, |
| "value": false |
| }, |
| "resume_from_checkpoint": { |
| "desc": null, |
| "value": null |
| }, |
| "chunk_size_feed_forward": { |
| "desc": null, |
| "value": 0 |
| }, |
| "eval_accumulation_steps": { |
| "desc": null, |
| "value": 3 |
| }, |
| "max_position_embeddings": { |
| "desc": null, |
| "value": 8192 |
| }, |
| "per_gpu_eval_batch_size": { |
| "desc": null, |
| "value": null |
| }, |
| "return_dict_in_generate": { |
| "desc": null, |
| "value": false |
| }, |
| "cosine_constant_lr_ratio": { |
| "desc": null, |
| "value": null |
| }, |
| "per_gpu_train_batch_size": { |
| "desc": null, |
| "value": null |
| }, |
| "push_to_hub_organization": { |
| "desc": null, |
| "value": null |
| }, |
| "include_tokens_per_second": { |
| "desc": null, |
| "value": false |
| }, |
| "sample_packing_efficiency": { |
| "desc": null, |
| "value": 1 |
| }, |
| "dataloader_prefetch_factor": { |
| "desc": null, |
| "value": null |
| }, |
| "ddp_find_unused_parameters": { |
| "desc": null, |
| "value": false |
| }, |
| "include_inputs_for_metrics": { |
| "desc": null, |
| "value": false |
| }, |
| "per_device_eval_batch_size": { |
| "desc": null, |
| "value": 2 |
| }, |
| "use_legacy_prediction_loop": { |
| "desc": null, |
| "value": false |
| }, |
| "cross_attention_hidden_size": { |
| "desc": null, |
| "value": null |
| }, |
| "gradient_accumulation_steps": { |
| "desc": null, |
| "value": 3 |
| }, |
| "per_device_train_batch_size": { |
| "desc": null, |
| "value": 2 |
| }, |
| "encoder_no_repeat_ngram_size": { |
| "desc": null, |
| "value": 0 |
| }, |
| "dataloader_persistent_workers": { |
| "desc": null, |
| "value": false |
| }, |
| "gradient_checkpointing_kwargs": { |
| "desc": null, |
| "value": { |
| "use_reentrant": true |
| } |
| }, |
| "include_num_input_tokens_seen": { |
| "desc": null, |
| "value": false |
| }, |
| "exponential_decay_length_penalty": { |
| "desc": null, |
| "value": null |
| }, |
| "sample_packing_seq_len_multiplier": { |
| "desc": null, |
| "value": 2 |
| }, |
| "fsdp_transformer_layer_cls_to_wrap": { |
| "desc": null, |
| "value": null |
| } |
| } |
| ### Model Architecture and Objective |
| |
| ## Citation [optional] |
|
|
|
|
| ## Glossary [optional] |
| ### Framework versions |
|
|
| - PEFT 0.9.0 |
