Add files using upload-large-folder tool

runs/single_llava/context/L01_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/context/L01_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.1.mlp.down_proj",
+    "model.language_model.layers.1.self_attn.v_proj",
+    "model.language_model.layers.1.self_attn.k_proj",
+    "model.language_model.layers.1.self_attn.o_proj",
+    "model.language_model.layers.1.mlp.gate_proj",
+    "model.language_model.layers.1.mlp.up_proj",
+    "model.language_model.layers.1.self_attn.q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/context/L01_s13/config.json

@@ -0,0 +1,24 @@
+{
+  "behavior": "context",
+  "seed": 13,
+  "model": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "backbone": "llava",
+  "train": "/data/OPSD/assets/v3/context.jsonl",
+  "items": 1000,
+  "steps": 500,
+  "epochs": 2,
+  "steps_per_epoch": 250,
+  "groups_per_step": 4,
+  "lr": 2e-05,
+  "scheduler": "constant",
+  "warmup_steps": 0,
+  "final_fraction": 1.0,
+  "weight_decay": 0.0,
+  "clip": 1.0,
+  "initial_loss_scale": 4.0,
+  "adapter": {
+    "layer": 1,
+    "rank": 16,
+    "trainable_parameters": 1249280
+  }
+}

runs/single_llava/context/L02_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/context/L02_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.2.self_attn.o_proj",
+    "model.language_model.layers.2.self_attn.v_proj",
+    "model.language_model.layers.2.mlp.gate_proj",
+    "model.language_model.layers.2.mlp.up_proj",
+    "model.language_model.layers.2.mlp.down_proj",
+    "model.language_model.layers.2.self_attn.q_proj",
+    "model.language_model.layers.2.self_attn.k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/context/L08_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/context/L08_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.8.mlp.up_proj",
+    "model.language_model.layers.8.self_attn.o_proj",
+    "model.language_model.layers.8.mlp.down_proj",
+    "model.language_model.layers.8.mlp.gate_proj",
+    "model.language_model.layers.8.self_attn.q_proj",
+    "model.language_model.layers.8.self_attn.v_proj",
+    "model.language_model.layers.8.self_attn.k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/context/L08_s13/config.json

@@ -0,0 +1,24 @@
+{
+  "behavior": "context",
+  "seed": 13,
+  "model": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "backbone": "llava",
+  "train": "/data/OPSD/assets/v3/context.jsonl",
+  "items": 1000,
+  "steps": 500,
+  "epochs": 2,
+  "steps_per_epoch": 250,
+  "groups_per_step": 4,
+  "lr": 2e-05,
+  "scheduler": "constant",
+  "warmup_steps": 0,
+  "final_fraction": 1.0,
+  "weight_decay": 0.0,
+  "clip": 1.0,
+  "initial_loss_scale": 4.0,
+  "adapter": {
+    "layer": 8,
+    "rank": 16,
+    "trainable_parameters": 1249280
+  }
+}

runs/single_llava/context/L14_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/context/L14_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.14.self_attn.v_proj",
+    "model.language_model.layers.14.mlp.down_proj",
+    "model.language_model.layers.14.mlp.gate_proj",
+    "model.language_model.layers.14.self_attn.k_proj",
+    "model.language_model.layers.14.self_attn.o_proj",
+    "model.language_model.layers.14.mlp.up_proj",
+    "model.language_model.layers.14.self_attn.q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/context/L14_s13/config.json

@@ -0,0 +1,24 @@
+{
+  "behavior": "context",
+  "seed": 13,
+  "model": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "backbone": "llava",
+  "train": "/data/OPSD/assets/v3/context.jsonl",
+  "items": 1000,
+  "steps": 500,
+  "epochs": 2,
+  "steps_per_epoch": 250,
+  "groups_per_step": 4,
+  "lr": 2e-05,
+  "scheduler": "constant",
+  "warmup_steps": 0,
+  "final_fraction": 1.0,
+  "weight_decay": 0.0,
+  "clip": 1.0,
+  "initial_loss_scale": 4.0,
+  "adapter": {
+    "layer": 14,
+    "rank": 16,
+    "trainable_parameters": 1249280
+  }
+}

runs/single_llava/context/L18_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/context/L18_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.18.mlp.up_proj",
+    "model.language_model.layers.18.mlp.gate_proj",
+    "model.language_model.layers.18.self_attn.o_proj",
+    "model.language_model.layers.18.self_attn.k_proj",
+    "model.language_model.layers.18.self_attn.v_proj",
+    "model.language_model.layers.18.self_attn.q_proj",
+    "model.language_model.layers.18.mlp.down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/context/L18_s13/config.json

@@ -0,0 +1,24 @@
+{
+  "behavior": "context",
+  "seed": 13,
+  "model": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "backbone": "llava",
+  "train": "/data/OPSD/assets/v3/context.jsonl",
+  "items": 1000,
+  "steps": 500,
+  "epochs": 2,
+  "steps_per_epoch": 250,
+  "groups_per_step": 4,
+  "lr": 2e-05,
+  "scheduler": "constant",
+  "warmup_steps": 0,
+  "final_fraction": 1.0,
+  "weight_decay": 0.0,
+  "clip": 1.0,
+  "initial_loss_scale": 4.0,
+  "adapter": {
+    "layer": 18,
+    "rank": 16,
+    "trainable_parameters": 1249280
+  }
+}

runs/single_llava/context/L22_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/context/L22_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.22.self_attn.v_proj",
+    "model.language_model.layers.22.mlp.down_proj",
+    "model.language_model.layers.22.self_attn.k_proj",
+    "model.language_model.layers.22.mlp.gate_proj",
+    "model.language_model.layers.22.self_attn.q_proj",
+    "model.language_model.layers.22.mlp.up_proj",
+    "model.language_model.layers.22.self_attn.o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/context/L22_s13/config.json

@@ -0,0 +1,24 @@
+{
+  "behavior": "context",
+  "seed": 13,
+  "model": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "backbone": "llava",
+  "train": "/data/OPSD/assets/v3/context.jsonl",
+  "items": 1000,
+  "steps": 500,
+  "epochs": 2,
+  "steps_per_epoch": 250,
+  "groups_per_step": 4,
+  "lr": 2e-05,
+  "scheduler": "constant",
+  "warmup_steps": 0,
+  "final_fraction": 1.0,
+  "weight_decay": 0.0,
+  "clip": 1.0,
+  "initial_loss_scale": 4.0,
+  "adapter": {
+    "layer": 22,
+    "rank": 16,
+    "trainable_parameters": 1249280
+  }
+}

runs/single_llava/context/L26_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/context/L26_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.26.self_attn.k_proj",
+    "model.language_model.layers.26.mlp.up_proj",
+    "model.language_model.layers.26.self_attn.v_proj",
+    "model.language_model.layers.26.mlp.down_proj",
+    "model.language_model.layers.26.self_attn.o_proj",
+    "model.language_model.layers.26.self_attn.q_proj",
+    "model.language_model.layers.26.mlp.gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/context/L26_s13/config.json

@@ -0,0 +1,24 @@
+{
+  "behavior": "context",
+  "seed": 13,
+  "model": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "backbone": "llava",
+  "train": "/data/OPSD/assets/v3/context.jsonl",
+  "items": 1000,
+  "steps": 500,
+  "epochs": 2,
+  "steps_per_epoch": 250,
+  "groups_per_step": 4,
+  "lr": 2e-05,
+  "scheduler": "constant",
+  "warmup_steps": 0,
+  "final_fraction": 1.0,
+  "weight_decay": 0.0,
+  "clip": 1.0,
+  "initial_loss_scale": 4.0,
+  "adapter": {
+    "layer": 26,
+    "rank": 16,
+    "trainable_parameters": 1249280
+  }
+}

runs/single_llava/context/L30_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/context/L30_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.30.self_attn.v_proj",
+    "model.language_model.layers.30.self_attn.k_proj",
+    "model.language_model.layers.30.mlp.up_proj",
+    "model.language_model.layers.30.self_attn.o_proj",
+    "model.language_model.layers.30.self_attn.q_proj",
+    "model.language_model.layers.30.mlp.down_proj",
+    "model.language_model.layers.30.mlp.gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/context/L30_s13/config.json

@@ -0,0 +1,24 @@
+{
+  "behavior": "context",
+  "seed": 13,
+  "model": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "backbone": "llava",
+  "train": "/data/OPSD/assets/v3/context.jsonl",
+  "items": 1000,
+  "steps": 500,
+  "epochs": 2,
+  "steps_per_epoch": 250,
+  "groups_per_step": 4,
+  "lr": 2e-05,
+  "scheduler": "constant",
+  "warmup_steps": 0,
+  "final_fraction": 1.0,
+  "weight_decay": 0.0,
+  "clip": 1.0,
+  "initial_loss_scale": 4.0,
+  "adapter": {
+    "layer": 30,
+    "rank": 16,
+    "trainable_parameters": 1249280
+  }
+}

runs/single_llava/removal/L10_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/removal/L10_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.10.mlp.up_proj",
+    "model.language_model.layers.10.mlp.down_proj",
+    "model.language_model.layers.10.self_attn.v_proj",
+    "model.language_model.layers.10.self_attn.q_proj",
+    "model.language_model.layers.10.self_attn.k_proj",
+    "model.language_model.layers.10.mlp.gate_proj",
+    "model.language_model.layers.10.self_attn.o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/removal/L10_s13/config.json

@@ -0,0 +1,24 @@
+{
+  "behavior": "removal",
+  "seed": 13,
+  "model": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "backbone": "llava",
+  "train": "/data/OPSD/assets/v3/removal.jsonl",
+  "items": 1000,
+  "steps": 500,
+  "epochs": 2,
+  "steps_per_epoch": 250,
+  "groups_per_step": 4,
+  "lr": 2e-05,
+  "scheduler": "constant",
+  "warmup_steps": 0,
+  "final_fraction": 1.0,
+  "weight_decay": 0.0,
+  "clip": 1.0,
+  "initial_loss_scale": 4.0,
+  "adapter": {
+    "layer": 10,
+    "rank": 16,
+    "trainable_parameters": 1249280
+  }
+}

runs/single_llava/removal/L10_s13/steps.jsonl

@@ -0,0 +1,500 @@
+{"step": 1, "steps": 500, "lr": 2e-05, "grad_norm": 0.2653537094593048, "loss_scale": 4.0, "retries": 0, "seconds": 2.7, "max_memory_gb": 17.03, "ce": 1.6287612388841808, "rank": 0.13243506848812103}
+{"step": 2, "steps": 500, "lr": 2e-05, "grad_norm": 0.3255559504032135, "loss_scale": 4.0, "retries": 0, "seconds": 4.4, "max_memory_gb": 17.05, "ce": 2.6751189748756588, "rank": 0.0325009822845459}
+{"step": 3, "steps": 500, "lr": 2e-05, "grad_norm": 0.7395737171173096, "loss_scale": 4.0, "retries": 0, "seconds": 6.3, "max_memory_gb": 17.05, "ce": 5.27434634976089, "rank": 0.15134704113006592}
+{"step": 4, "steps": 500, "lr": 2e-05, "grad_norm": 0.4953780770301819, "loss_scale": 4.0, "retries": 0, "seconds": 8.1, "max_memory_gb": 17.05, "ce": 3.7998799041379243, "rank": 0.12265133857727051}
+{"step": 5, "steps": 500, "lr": 2e-05, "grad_norm": 0.09839017689228058, "loss_scale": 4.0, "retries": 0, "seconds": 9.9, "max_memory_gb": 17.05, "ce": 0.13944480172358453, "rank": 0.0}
+{"step": 6, "steps": 500, "lr": 2e-05, "grad_norm": 0.26154622435569763, "loss_scale": 4.0, "retries": 0, "seconds": 11.7, "max_memory_gb": 17.05, "ce": 1.5112212565727532, "rank": 0.0}
+{"step": 7, "steps": 500, "lr": 2e-05, "grad_norm": 0.537808895111084, "loss_scale": 4.0, "retries": 0, "seconds": 13.5, "max_memory_gb": 17.05, "ce": 3.453142264857888, "rank": 0.019974112510681152}
+{"step": 8, "steps": 500, "lr": 2e-05, "grad_norm": 0.6581587195396423, "loss_scale": 4.0, "retries": 0, "seconds": 15.3, "max_memory_gb": 17.05, "ce": 3.693382900208235, "rank": 0.20045030117034912}
+{"step": 9, "steps": 500, "lr": 2e-05, "grad_norm": 0.5057637095451355, "loss_scale": 4.0, "retries": 0, "seconds": 17.1, "max_memory_gb": 17.05, "ce": 2.8252401500940323, "rank": 0.040686845779418945}
+{"step": 10, "steps": 500, "lr": 2e-05, "grad_norm": 0.32875996828079224, "loss_scale": 4.0, "retries": 0, "seconds": 18.9, "max_memory_gb": 17.05, "ce": 1.612287218682468, "rank": 0.0}
+{"step": 11, "steps": 500, "lr": 2e-05, "grad_norm": 0.6271861791610718, "loss_scale": 4.0, "retries": 0, "seconds": 20.8, "max_memory_gb": 17.05, "ce": 2.607816875912249, "rank": 0.2079923003911972}
+{"step": 12, "steps": 500, "lr": 2e-05, "grad_norm": 0.43488457798957825, "loss_scale": 4.0, "retries": 0, "seconds": 22.6, "max_memory_gb": 17.05, "ce": 1.4975509156938642, "rank": 0.048140883445739746}
+{"step": 13, "steps": 500, "lr": 2e-05, "grad_norm": 0.5896051526069641, "loss_scale": 4.0, "retries": 0, "seconds": 24.4, "max_memory_gb": 17.05, "ce": 2.648462751531042, "rank": 0.0}
+{"step": 14, "steps": 500, "lr": 2e-05, "grad_norm": 0.9066510200500488, "loss_scale": 4.0, "retries": 0, "seconds": 26.2, "max_memory_gb": 17.05, "ce": 3.834084752947092, "rank": 0.142614483833313}
+{"step": 15, "steps": 500, "lr": 2e-05, "grad_norm": 0.6774967312812805, "loss_scale": 4.0, "retries": 0, "seconds": 28.0, "max_memory_gb": 17.05, "ce": 2.567360366578214, "rank": 0.17147791385650635}
+{"step": 16, "steps": 500, "lr": 2e-05, "grad_norm": 0.4948481619358063, "loss_scale": 4.0, "retries": 0, "seconds": 29.7, "max_memory_gb": 17.05, "ce": 1.3962891747942194, "rank": 0.0}
+{"step": 17, "steps": 500, "lr": 2e-05, "grad_norm": 0.47296541929244995, "loss_scale": 4.0, "retries": 0, "seconds": 31.5, "max_memory_gb": 17.05, "ce": 1.5712507923599333, "rank": 0.03492283821105957}
+{"step": 18, "steps": 500, "lr": 2e-05, "grad_norm": 1.0379005670547485, "loss_scale": 4.0, "retries": 0, "seconds": 33.3, "max_memory_gb": 17.06, "ce": 3.5272043216973543, "rank": 0.18897509574890137}
+{"step": 19, "steps": 500, "lr": 2e-05, "grad_norm": 1.5730359554290771, "loss_scale": 4.0, "retries": 0, "seconds": 35.1, "max_memory_gb": 17.06, "ce": 5.179475121200085, "rank": 0.3090568780899048}
+{"step": 20, "steps": 500, "lr": 2e-05, "grad_norm": 0.0360497310757637, "loss_scale": 4.0, "retries": 0, "seconds": 36.9, "max_memory_gb": 17.06, "ce": 0.06557388452347368, "rank": 0.10982850939035416}
+{"step": 21, "steps": 500, "lr": 2e-05, "grad_norm": 0.9881109595298767, "loss_scale": 4.0, "retries": 0, "seconds": 38.7, "max_memory_gb": 17.06, "ce": 2.6744827412767336, "rank": 0.14484751224517822}
+{"step": 22, "steps": 500, "lr": 2e-05, "grad_norm": 1.0270748138427734, "loss_scale": 4.0, "retries": 0, "seconds": 40.6, "max_memory_gb": 17.06, "ce": 2.867347010411322, "rank": 0.18620800971984863}
+{"step": 23, "steps": 500, "lr": 2e-05, "grad_norm": 0.9474174380302429, "loss_scale": 4.0, "retries": 0, "seconds": 42.4, "max_memory_gb": 17.06, "ce": 2.675312611158006, "rank": 0.12389874458312988}
+{"step": 24, "steps": 500, "lr": 2e-05, "grad_norm": 1.7487622499465942, "loss_scale": 4.0, "retries": 0, "seconds": 44.2, "max_memory_gb": 17.06, "ce": 4.851570934988558, "rank": 0.26078903675079346}
+{"step": 25, "steps": 500, "lr": 2e-05, "grad_norm": 1.7818938493728638, "loss_scale": 4.0, "retries": 0, "seconds": 46.1, "max_memory_gb": 17.06, "ce": 3.9649509941227734, "rank": 0.03110659122467041}
+{"step": 26, "steps": 500, "lr": 2e-05, "grad_norm": 2.4068167209625244, "loss_scale": 4.0, "retries": 0, "seconds": 47.9, "max_memory_gb": 17.06, "ce": 4.932674236595631, "rank": 0.03971457481384277}
+{"step": 27, "steps": 500, "lr": 2e-05, "grad_norm": 1.9962598085403442, "loss_scale": 4.0, "retries": 0, "seconds": 49.8, "max_memory_gb": 17.06, "ce": 4.005262214690447, "rank": 0.46315670013427734}
+{"step": 28, "steps": 500, "lr": 2e-05, "grad_norm": 1.2854247093200684, "loss_scale": 4.0, "retries": 0, "seconds": 51.6, "max_memory_gb": 17.06, "ce": 2.6268937969580293, "rank": 0.0}
+{"step": 29, "steps": 500, "lr": 2e-05, "grad_norm": 1.1707673072814941, "loss_scale": 4.0, "retries": 0, "seconds": 53.4, "max_memory_gb": 17.06, "ce": 2.5489916619844735, "rank": 0.0}
+{"step": 30, "steps": 500, "lr": 2e-05, "grad_norm": 1.6258230209350586, "loss_scale": 4.0, "retries": 0, "seconds": 55.2, "max_memory_gb": 17.06, "ce": 2.567375147365965, "rank": 0.0}
+{"step": 31, "steps": 500, "lr": 2e-05, "grad_norm": 1.8605738878250122, "loss_scale": 4.0, "retries": 0, "seconds": 57.1, "max_memory_gb": 17.06, "ce": 2.606330675072968, "rank": 0.1318296194076538}
+{"step": 32, "steps": 500, "lr": 2e-05, "grad_norm": 3.284130334854126, "loss_scale": 4.0, "retries": 0, "seconds": 59.2, "max_memory_gb": 17.06, "ce": 4.901888711377978, "rank": 0.20302224159240723}
+{"step": 33, "steps": 500, "lr": 2e-05, "grad_norm": 1.4956361055374146, "loss_scale": 4.0, "retries": 0, "seconds": 61.1, "max_memory_gb": 17.06, "ce": 2.3389590131118894, "rank": 0.00721915066242218}
+{"step": 34, "steps": 500, "lr": 2e-05, "grad_norm": 1.7855329513549805, "loss_scale": 4.0, "retries": 0, "seconds": 63.1, "max_memory_gb": 17.06, "ce": 2.5038613585056737, "rank": 0.0}
+{"step": 35, "steps": 500, "lr": 2e-05, "grad_norm": 2.2163290977478027, "loss_scale": 4.0, "retries": 0, "seconds": 65.0, "max_memory_gb": 17.06, "ce": 2.4256789283826947, "rank": 0.12752044200897217}
+{"step": 36, "steps": 500, "lr": 2e-05, "grad_norm": 1.6522941589355469, "loss_scale": 4.0, "retries": 0, "seconds": 66.9, "max_memory_gb": 17.06, "ce": 2.365787868387997, "rank": 0.0}
+{"step": 37, "steps": 500, "lr": 2e-05, "grad_norm": 2.055875062942505, "loss_scale": 4.0, "retries": 0, "seconds": 68.6, "max_memory_gb": 17.06, "ce": 2.3350269524380565, "rank": 0.0}
+{"step": 38, "steps": 500, "lr": 2e-05, "grad_norm": 3.657377243041992, "loss_scale": 4.0, "retries": 0, "seconds": 70.4, "max_memory_gb": 17.06, "ce": 4.319590548984706, "rank": 0.1314762830734253}
+{"step": 39, "steps": 500, "lr": 2e-05, "grad_norm": 1.8457865715026855, "loss_scale": 4.0, "retries": 0, "seconds": 72.2, "max_memory_gb": 17.06, "ce": 1.2229944390710443, "rank": 0.10780692100524902}
+{"step": 40, "steps": 500, "lr": 2e-05, "grad_norm": 2.3702375888824463, "loss_scale": 4.0, "retries": 0, "seconds": 74.0, "max_memory_gb": 17.06, "ce": 2.3462868672795594, "rank": 0.0}
+{"step": 41, "steps": 500, "lr": 2e-05, "grad_norm": 1.1078883409500122, "loss_scale": 4.0, "retries": 0, "seconds": 75.8, "max_memory_gb": 17.06, "ce": 1.3726576752960682, "rank": 0.0}
+{"step": 42, "steps": 500, "lr": 2e-05, "grad_norm": 2.8637397289276123, "loss_scale": 4.0, "retries": 0, "seconds": 77.6, "max_memory_gb": 17.06, "ce": 1.9637418282218277, "rank": 0.0}
+{"step": 43, "steps": 500, "lr": 2e-05, "grad_norm": 2.805659055709839, "loss_scale": 4.0, "retries": 0, "seconds": 79.3, "max_memory_gb": 17.06, "ce": 2.0408092385623604, "rank": 0.0}
+{"step": 44, "steps": 500, "lr": 2e-05, "grad_norm": 1.4033801555633545, "loss_scale": 4.0, "retries": 0, "seconds": 81.2, "max_memory_gb": 17.06, "ce": 1.2386307143606246, "rank": 0.0}
+{"step": 45, "steps": 500, "lr": 2e-05, "grad_norm": 1.5849415063858032, "loss_scale": 4.0, "retries": 0, "seconds": 82.9, "max_memory_gb": 17.06, "ce": 1.1921602706424892, "rank": 0.0}
+{"step": 46, "steps": 500, "lr": 2e-05, "grad_norm": 2.730700969696045, "loss_scale": 4.0, "retries": 0, "seconds": 84.7, "max_memory_gb": 17.06, "ce": 1.949591614305973, "rank": 0.0}
+{"step": 47, "steps": 500, "lr": 2e-05, "grad_norm": 3.1206295490264893, "loss_scale": 4.0, "retries": 0, "seconds": 86.5, "max_memory_gb": 17.06, "ce": 1.5866909674368799, "rank": 0.0}
+{"step": 48, "steps": 500, "lr": 2e-05, "grad_norm": 6.118156433105469, "loss_scale": 4.0, "retries": 0, "seconds": 88.3, "max_memory_gb": 17.06, "ce": 2.861174626275897, "rank": 0.022043466567993164}
+{"step": 49, "steps": 500, "lr": 2e-05, "grad_norm": 5.640113353729248, "loss_scale": 4.0, "retries": 0, "seconds": 90.1, "max_memory_gb": 17.06, "ce": 2.6931647709570825, "rank": 0.0}
+{"step": 50, "steps": 500, "lr": 2e-05, "grad_norm": 5.561234951019287, "loss_scale": 4.0, "retries": 0, "seconds": 91.9, "max_memory_gb": 17.06, "ce": 2.7027317910688, "rank": 0.0}
+{"step": 51, "steps": 500, "lr": 2e-05, "grad_norm": 2.3553152084350586, "loss_scale": 4.0, "retries": 0, "seconds": 93.8, "max_memory_gb": 17.06, "ce": 1.0628476405981928, "rank": 0.0}
+{"step": 52, "steps": 500, "lr": 2e-05, "grad_norm": 4.116091251373291, "loss_scale": 4.0, "retries": 0, "seconds": 95.6, "max_memory_gb": 17.06, "ce": 1.6738802595064044, "rank": 0.0}
+{"step": 53, "steps": 500, "lr": 2e-05, "grad_norm": 6.564763069152832, "loss_scale": 4.0, "retries": 0, "seconds": 97.4, "max_memory_gb": 17.06, "ce": 2.700189939700067, "rank": 0.0}
+{"step": 54, "steps": 500, "lr": 2e-05, "grad_norm": 7.344996929168701, "loss_scale": 4.0, "retries": 0, "seconds": 99.4, "max_memory_gb": 17.06, "ce": 2.553074872121215, "rank": 0.0}
+{"step": 55, "steps": 500, "lr": 2e-05, "grad_norm": 4.873615741729736, "loss_scale": 4.0, "retries": 0, "seconds": 101.3, "max_memory_gb": 17.06, "ce": 1.5389347891323268, "rank": 0.0}
+{"step": 56, "steps": 500, "lr": 2e-05, "grad_norm": 10.106751441955566, "loss_scale": 4.0, "retries": 0, "seconds": 103.3, "max_memory_gb": 17.06, "ce": 2.9161088475957513, "rank": 0.0}
+{"step": 57, "steps": 500, "lr": 2e-05, "grad_norm": 2.4290692806243896, "loss_scale": 4.0, "retries": 0, "seconds": 105.2, "max_memory_gb": 17.06, "ce": 0.828968808054924, "rank": 0.010039225220680237}
+{"step": 58, "steps": 500, "lr": 2e-05, "grad_norm": 5.294338226318359, "loss_scale": 4.0, "retries": 0, "seconds": 107.1, "max_memory_gb": 17.06, "ce": 1.2473032695706934, "rank": 0.0}
+{"step": 59, "steps": 500, "lr": 2e-05, "grad_norm": 2.4851014614105225, "loss_scale": 4.0, "retries": 0, "seconds": 108.9, "max_memory_gb": 17.06, "ce": 1.0590149187482893, "rank": 0.0}
+{"step": 60, "steps": 500, "lr": 2e-05, "grad_norm": 5.569611549377441, "loss_scale": 4.0, "retries": 0, "seconds": 110.7, "max_memory_gb": 17.06, "ce": 1.3420988398138434, "rank": 0.0}
+{"step": 61, "steps": 500, "lr": 2e-05, "grad_norm": 7.717735290527344, "loss_scale": 4.0, "retries": 0, "seconds": 112.5, "max_memory_gb": 17.06, "ce": 1.612314193509519, "rank": 0.0}
+{"step": 62, "steps": 500, "lr": 2e-05, "grad_norm": 3.23911714553833, "loss_scale": 4.0, "retries": 0, "seconds": 114.3, "max_memory_gb": 17.06, "ce": 0.7734976173378527, "rank": 0.0}
+{"step": 63, "steps": 500, "lr": 2e-05, "grad_norm": 7.831045627593994, "loss_scale": 4.0, "retries": 0, "seconds": 116.1, "max_memory_gb": 17.06, "ce": 1.8006023336201906, "rank": 0.03168439865112305}
+{"step": 64, "steps": 500, "lr": 2e-05, "grad_norm": 7.653633117675781, "loss_scale": 4.0, "retries": 0, "seconds": 118.0, "max_memory_gb": 17.06, "ce": 1.4808640282135457, "rank": 0.0}
+{"step": 65, "steps": 500, "lr": 2e-05, "grad_norm": 6.547536849975586, "loss_scale": 4.0, "retries": 0, "seconds": 119.8, "max_memory_gb": 17.06, "ce": 1.713633969426155, "rank": 0.0}
+{"step": 66, "steps": 500, "lr": 2e-05, "grad_norm": 1.744053602218628, "loss_scale": 4.0, "retries": 0, "seconds": 121.6, "max_memory_gb": 17.06, "ce": 0.6007788652786985, "rank": 0.20530902594327927}
+{"step": 67, "steps": 500, "lr": 2e-05, "grad_norm": 5.602546691894531, "loss_scale": 4.0, "retries": 0, "seconds": 123.4, "max_memory_gb": 17.06, "ce": 1.1417332235723734, "rank": 0.0}
+{"step": 68, "steps": 500, "lr": 2e-05, "grad_norm": 2.6935009956359863, "loss_scale": 4.0, "retries": 0, "seconds": 125.3, "max_memory_gb": 17.06, "ce": 0.5144030337687582, "rank": 0.0}
+{"step": 69, "steps": 500, "lr": 2e-05, "grad_norm": 4.137938976287842, "loss_scale": 4.0, "retries": 0, "seconds": 127.1, "max_memory_gb": 17.06, "ce": 0.7994859828613698, "rank": 0.0}
+{"step": 70, "steps": 500, "lr": 2e-05, "grad_norm": 3.541605234146118, "loss_scale": 4.0, "retries": 0, "seconds": 128.9, "max_memory_gb": 17.06, "ce": 0.6496156393550336, "rank": 0.0}
+{"step": 71, "steps": 500, "lr": 2e-05, "grad_norm": 5.711341857910156, "loss_scale": 4.0, "retries": 0, "seconds": 130.7, "max_memory_gb": 17.06, "ce": 0.8422634452581406, "rank": 0.0}
+{"step": 72, "steps": 500, "lr": 2e-05, "grad_norm": 5.83536434173584, "loss_scale": 4.0, "retries": 0, "seconds": 132.6, "max_memory_gb": 17.06, "ce": 1.0015619779005647, "rank": 0.0}
+{"step": 73, "steps": 500, "lr": 2e-05, "grad_norm": 0.3484004735946655, "loss_scale": 4.0, "retries": 0, "seconds": 134.3, "max_memory_gb": 17.06, "ce": 0.24749177345074713, "rank": 0.0}
+{"step": 74, "steps": 500, "lr": 2e-05, "grad_norm": 1.9247084856033325, "loss_scale": 4.0, "retries": 0, "seconds": 136.1, "max_memory_gb": 17.06, "ce": 0.5031799962744117, "rank": 0.0}
+{"step": 75, "steps": 500, "lr": 2e-05, "grad_norm": 1.6952288150787354, "loss_scale": 4.0, "retries": 0, "seconds": 137.9, "max_memory_gb": 17.06, "ce": 0.830964651890099, "rank": 0.27655136585235596}
+{"step": 76, "steps": 500, "lr": 2e-05, "grad_norm": 1.8844772577285767, "loss_scale": 4.0, "retries": 0, "seconds": 139.8, "max_memory_gb": 17.06, "ce": 0.5383810237981379, "rank": 0.0}
+{"step": 77, "steps": 500, "lr": 2e-05, "grad_norm": 2.9208016395568848, "loss_scale": 4.0, "retries": 0, "seconds": 141.6, "max_memory_gb": 17.06, "ce": 0.759094045497477, "rank": 0.0}
+{"step": 78, "steps": 500, "lr": 2e-05, "grad_norm": 1.6616888046264648, "loss_scale": 4.0, "retries": 0, "seconds": 143.3, "max_memory_gb": 17.06, "ce": 0.45800743578001857, "rank": 0.0}
+{"step": 79, "steps": 500, "lr": 2e-05, "grad_norm": 0.9012949466705322, "loss_scale": 4.0, "retries": 0, "seconds": 145.1, "max_memory_gb": 17.06, "ce": 0.30008298996835947, "rank": 0.0036847442388534546}
+{"step": 80, "steps": 500, "lr": 2e-05, "grad_norm": 1.277734637260437, "loss_scale": 4.0, "retries": 0, "seconds": 146.9, "max_memory_gb": 17.06, "ce": 0.37472984846681356, "rank": 0.0}
+{"step": 81, "steps": 500, "lr": 2e-05, "grad_norm": 1.4055503606796265, "loss_scale": 4.0, "retries": 0, "seconds": 148.7, "max_memory_gb": 17.06, "ce": 0.6223121527582407, "rank": 0.0}
+{"step": 82, "steps": 500, "lr": 2e-05, "grad_norm": 0.8037723302841187, "loss_scale": 4.0, "retries": 0, "seconds": 150.5, "max_memory_gb": 17.06, "ce": 0.50034453952685, "rank": 0.0}
+{"step": 83, "steps": 500, "lr": 2e-05, "grad_norm": 0.8398341536521912, "loss_scale": 4.0, "retries": 0, "seconds": 152.3, "max_memory_gb": 17.06, "ce": 0.5743882453534752, "rank": 0.0}
+{"step": 84, "steps": 500, "lr": 2e-05, "grad_norm": 0.6813604235649109, "loss_scale": 4.0, "retries": 0, "seconds": 154.0, "max_memory_gb": 17.06, "ce": 0.3013541620457545, "rank": 0.0}
+{"step": 85, "steps": 500, "lr": 2e-05, "grad_norm": 0.41978543996810913, "loss_scale": 4.0, "retries": 0, "seconds": 155.9, "max_memory_gb": 17.06, "ce": 0.12983459560200572, "rank": 0.0}
+{"step": 86, "steps": 500, "lr": 2e-05, "grad_norm": 1.0865713357925415, "loss_scale": 4.0, "retries": 0, "seconds": 157.7, "max_memory_gb": 17.06, "ce": 0.5939884511753917, "rank": 0.0}
+{"step": 87, "steps": 500, "lr": 2e-05, "grad_norm": 0.7596805095672607, "loss_scale": 4.0, "retries": 0, "seconds": 159.5, "max_memory_gb": 17.06, "ce": 0.4927105763927102, "rank": 0.0}
+{"step": 88, "steps": 500, "lr": 2e-05, "grad_norm": 0.5810946226119995, "loss_scale": 4.0, "retries": 0, "seconds": 161.3, "max_memory_gb": 17.06, "ce": 0.45623165159486234, "rank": 0.0}
+{"step": 89, "steps": 500, "lr": 2e-05, "grad_norm": 0.3421095609664917, "loss_scale": 4.0, "retries": 0, "seconds": 163.1, "max_memory_gb": 17.06, "ce": 0.17280211357865483, "rank": 0.1241958737373352}
+{"step": 90, "steps": 500, "lr": 2e-05, "grad_norm": 0.4941292405128479, "loss_scale": 4.0, "retries": 0, "seconds": 164.9, "max_memory_gb": 17.06, "ce": 0.39960496174171567, "rank": 0.0}
+{"step": 91, "steps": 500, "lr": 2e-05, "grad_norm": 0.4931747317314148, "loss_scale": 4.0, "retries": 0, "seconds": 166.7, "max_memory_gb": 17.06, "ce": 0.360349303111434, "rank": 0.0}
+{"step": 92, "steps": 500, "lr": 2e-05, "grad_norm": 0.36913952231407166, "loss_scale": 4.0, "retries": 0, "seconds": 168.5, "max_memory_gb": 17.06, "ce": 0.29016237088944763, "rank": 0.0}
+{"step": 93, "steps": 500, "lr": 2e-05, "grad_norm": 0.4943414628505707, "loss_scale": 4.0, "retries": 0, "seconds": 170.3, "max_memory_gb": 17.06, "ce": 0.343200626084581, "rank": 0.0}
+{"step": 94, "steps": 500, "lr": 2e-05, "grad_norm": 0.540108859539032, "loss_scale": 4.0, "retries": 0, "seconds": 172.2, "max_memory_gb": 17.06, "ce": 0.5166195062920451, "rank": 0.002531766891479492}
+{"step": 95, "steps": 500, "lr": 2e-05, "grad_norm": 0.6709714531898499, "loss_scale": 4.0, "retries": 0, "seconds": 174.0, "max_memory_gb": 17.06, "ce": 0.48902175994589925, "rank": 0.0}
+{"step": 96, "steps": 500, "lr": 2e-05, "grad_norm": 0.7049700617790222, "loss_scale": 4.0, "retries": 0, "seconds": 175.8, "max_memory_gb": 17.06, "ce": 0.5343865472823381, "rank": 0.062232330441474915}
+{"step": 97, "steps": 500, "lr": 2e-05, "grad_norm": 0.6360589265823364, "loss_scale": 4.0, "retries": 0, "seconds": 177.6, "max_memory_gb": 17.06, "ce": 0.31412726896815, "rank": 0.016329094767570496}
+{"step": 98, "steps": 500, "lr": 2e-05, "grad_norm": 0.35962173342704773, "loss_scale": 4.0, "retries": 0, "seconds": 179.4, "max_memory_gb": 17.06, "ce": 0.309364530723542, "rank": 0.0}
+{"step": 99, "steps": 500, "lr": 2e-05, "grad_norm": 0.7133542895317078, "loss_scale": 4.0, "retries": 0, "seconds": 181.2, "max_memory_gb": 17.06, "ce": 0.22086609352845699, "rank": 0.024518698453903198}
+{"step": 100, "steps": 500, "lr": 2e-05, "grad_norm": 0.5450439453125, "loss_scale": 4.0, "retries": 0, "seconds": 183.1, "max_memory_gb": 17.06, "ce": 0.5255201244726777, "rank": 0.0}
+{"step": 101, "steps": 500, "lr": 2e-05, "grad_norm": 0.5074237585067749, "loss_scale": 4.0, "retries": 0, "seconds": 185.0, "max_memory_gb": 17.06, "ce": 0.4749835589900613, "rank": 0.0}
+{"step": 102, "steps": 500, "lr": 2e-05, "grad_norm": 0.9613125324249268, "loss_scale": 4.0, "retries": 0, "seconds": 186.9, "max_memory_gb": 17.06, "ce": 0.6470595630817115, "rank": 0.0}
+{"step": 103, "steps": 500, "lr": 2e-05, "grad_norm": 0.35918304324150085, "loss_scale": 4.0, "retries": 0, "seconds": 188.7, "max_memory_gb": 17.06, "ce": 0.3163674407405779, "rank": 0.1175977811217308}
+{"step": 104, "steps": 500, "lr": 2e-05, "grad_norm": 0.2453080713748932, "loss_scale": 4.0, "retries": 0, "seconds": 190.4, "max_memory_gb": 17.06, "ce": 0.21573205257300287, "rank": 0.08932866156101227}
+{"step": 105, "steps": 500, "lr": 2e-05, "grad_norm": 0.7522282600402832, "loss_scale": 4.0, "retries": 0, "seconds": 192.2, "max_memory_gb": 17.06, "ce": 0.4574836092069745, "rank": 0.05411801487207413}
+{"step": 106, "steps": 500, "lr": 2e-05, "grad_norm": 0.4991978406906128, "loss_scale": 4.0, "retries": 0, "seconds": 194.0, "max_memory_gb": 17.06, "ce": 0.479890544898808, "rank": 0.09016163647174835}
+{"step": 107, "steps": 500, "lr": 2e-05, "grad_norm": 0.3230806887149811, "loss_scale": 4.0, "retries": 0, "seconds": 195.8, "max_memory_gb": 17.06, "ce": 0.27099999971687794, "rank": 0.0}
+{"step": 108, "steps": 500, "lr": 2e-05, "grad_norm": 0.37596431374549866, "loss_scale": 4.0, "retries": 0, "seconds": 197.6, "max_memory_gb": 17.06, "ce": 0.27759790723212063, "rank": 0.0}
+{"step": 109, "steps": 500, "lr": 2e-05, "grad_norm": 0.6493387222290039, "loss_scale": 4.0, "retries": 0, "seconds": 199.4, "max_memory_gb": 17.06, "ce": 0.6013336610049009, "rank": 0.0}
+{"step": 110, "steps": 500, "lr": 2e-05, "grad_norm": 0.5871270895004272, "loss_scale": 4.0, "retries": 0, "seconds": 201.2, "max_memory_gb": 17.06, "ce": 0.39380110474303365, "rank": 0.05510471761226654}
+{"step": 111, "steps": 500, "lr": 2e-05, "grad_norm": 0.4552331566810608, "loss_scale": 4.0, "retries": 0, "seconds": 203.0, "max_memory_gb": 17.06, "ce": 0.3310268234927207, "rank": 0.00012758374214172363}
+{"step": 112, "steps": 500, "lr": 2e-05, "grad_norm": 0.4337981045246124, "loss_scale": 4.0, "retries": 0, "seconds": 204.8, "max_memory_gb": 17.06, "ce": 0.3309075792785734, "rank": 0.0}
+{"step": 113, "steps": 500, "lr": 2e-05, "grad_norm": 0.36006665229797363, "loss_scale": 4.0, "retries": 0, "seconds": 206.6, "max_memory_gb": 17.06, "ce": 0.3795342445373535, "rank": 0.0}
+{"step": 114, "steps": 500, "lr": 2e-05, "grad_norm": 0.7231184244155884, "loss_scale": 4.0, "retries": 0, "seconds": 208.5, "max_memory_gb": 17.06, "ce": 0.6707635335624218, "rank": 0.1327698975801468}
+{"step": 115, "steps": 500, "lr": 2e-05, "grad_norm": 0.4730296730995178, "loss_scale": 4.0, "retries": 0, "seconds": 210.3, "max_memory_gb": 17.06, "ce": 0.4467210005968809, "rank": 0.1201343908905983}
+{"step": 116, "steps": 500, "lr": 2e-05, "grad_norm": 0.23845316469669342, "loss_scale": 4.0, "retries": 0, "seconds": 212.0, "max_memory_gb": 17.06, "ce": 0.14859285682905465, "rank": 0.0}
+{"step": 117, "steps": 500, "lr": 2e-05, "grad_norm": 0.8263993263244629, "loss_scale": 4.0, "retries": 0, "seconds": 213.8, "max_memory_gb": 17.06, "ce": 0.4337487407028675, "rank": 0.0}
+{"step": 118, "steps": 500, "lr": 2e-05, "grad_norm": 0.277067095041275, "loss_scale": 4.0, "retries": 0, "seconds": 215.6, "max_memory_gb": 17.06, "ce": 0.2947067031636834, "rank": 0.0}
+{"step": 119, "steps": 500, "lr": 2e-05, "grad_norm": 0.3403720557689667, "loss_scale": 4.0, "retries": 0, "seconds": 217.4, "max_memory_gb": 17.06, "ce": 0.26700756838545203, "rank": 0.0}
+{"step": 120, "steps": 500, "lr": 2e-05, "grad_norm": 0.7506887316703796, "loss_scale": 4.0, "retries": 0, "seconds": 219.3, "max_memory_gb": 17.06, "ce": 0.4234173677396029, "rank": 0.0}
+{"step": 121, "steps": 500, "lr": 2e-05, "grad_norm": 0.5798329710960388, "loss_scale": 4.0, "retries": 0, "seconds": 221.1, "max_memory_gb": 17.06, "ce": 0.33532921213191, "rank": 0.0}
+{"step": 122, "steps": 500, "lr": 2e-05, "grad_norm": 0.3123505711555481, "loss_scale": 4.0, "retries": 0, "seconds": 222.9, "max_memory_gb": 17.06, "ce": 0.3668722028378397, "rank": 0.0}
+{"step": 123, "steps": 500, "lr": 2e-05, "grad_norm": 0.810811460018158, "loss_scale": 4.0, "retries": 0, "seconds": 224.6, "max_memory_gb": 17.06, "ce": 0.5688952417112887, "rank": 0.0}
+{"step": 124, "steps": 500, "lr": 2e-05, "grad_norm": 0.7536030411720276, "loss_scale": 4.0, "retries": 0, "seconds": 226.3, "max_memory_gb": 17.06, "ce": 0.39749180153012276, "rank": 0.0452866405248642}
+{"step": 125, "steps": 500, "lr": 2e-05, "grad_norm": 0.3285997807979584, "loss_scale": 4.0, "retries": 0, "seconds": 228.0, "max_memory_gb": 17.06, "ce": 0.29196617053821683, "rank": 0.0}
+{"step": 126, "steps": 500, "lr": 2e-05, "grad_norm": 0.5237862467765808, "loss_scale": 4.0, "retries": 0, "seconds": 229.8, "max_memory_gb": 17.06, "ce": 0.5323069337755442, "rank": 0.0}
+{"step": 127, "steps": 500, "lr": 2e-05, "grad_norm": 0.2156722992658615, "loss_scale": 4.0, "retries": 0, "seconds": 231.6, "max_memory_gb": 17.06, "ce": 0.17327834200114012, "rank": 0.0}
+{"step": 128, "steps": 500, "lr": 2e-05, "grad_norm": 0.48926183581352234, "loss_scale": 4.0, "retries": 0, "seconds": 233.5, "max_memory_gb": 17.06, "ce": 0.37722240679431707, "rank": 0.0}
+{"step": 129, "steps": 500, "lr": 2e-05, "grad_norm": 0.44464948773384094, "loss_scale": 4.0, "retries": 0, "seconds": 235.2, "max_memory_gb": 17.06, "ce": 0.5191492298617959, "rank": 0.0}
+{"step": 130, "steps": 500, "lr": 2e-05, "grad_norm": 0.8335224986076355, "loss_scale": 4.0, "retries": 0, "seconds": 237.0, "max_memory_gb": 17.06, "ce": 0.5335880378261209, "rank": 0.0}
+{"step": 131, "steps": 500, "lr": 2e-05, "grad_norm": 0.9830713868141174, "loss_scale": 4.0, "retries": 0, "seconds": 238.8, "max_memory_gb": 17.06, "ce": 0.36861937306821346, "rank": 0.0}
+{"step": 132, "steps": 500, "lr": 2e-05, "grad_norm": 0.968222439289093, "loss_scale": 4.0, "retries": 0, "seconds": 240.6, "max_memory_gb": 17.06, "ce": 0.6053362132515758, "rank": 0.0719042718410492}
+{"step": 133, "steps": 500, "lr": 2e-05, "grad_norm": 1.0353920459747314, "loss_scale": 4.0, "retries": 0, "seconds": 242.4, "max_memory_gb": 17.06, "ce": 0.5148463854566216, "rank": 0.041343286633491516}
+{"step": 134, "steps": 500, "lr": 2e-05, "grad_norm": 0.5655443072319031, "loss_scale": 4.0, "retries": 0, "seconds": 244.2, "max_memory_gb": 17.06, "ce": 0.19463899871334434, "rank": 0.0}
+{"step": 135, "steps": 500, "lr": 2e-05, "grad_norm": 0.5050235390663147, "loss_scale": 4.0, "retries": 0, "seconds": 246.0, "max_memory_gb": 17.06, "ce": 0.4411933533847332, "rank": 0.031171157956123352}
+{"step": 136, "steps": 500, "lr": 2e-05, "grad_norm": 0.6178869605064392, "loss_scale": 4.0, "retries": 0, "seconds": 247.8, "max_memory_gb": 17.06, "ce": 0.4342943197116256, "rank": 0.0}
+{"step": 137, "steps": 500, "lr": 2e-05, "grad_norm": 0.5767912864685059, "loss_scale": 4.0, "retries": 0, "seconds": 249.6, "max_memory_gb": 17.06, "ce": 0.3575440489221364, "rank": 0.0}
+{"step": 138, "steps": 500, "lr": 2e-05, "grad_norm": 0.3989766836166382, "loss_scale": 4.0, "retries": 0, "seconds": 251.5, "max_memory_gb": 17.06, "ce": 0.36565400566905737, "rank": 0.0}
+{"step": 139, "steps": 500, "lr": 2e-05, "grad_norm": 1.069201946258545, "loss_scale": 4.0, "retries": 0, "seconds": 253.5, "max_memory_gb": 17.06, "ce": 0.5772446077316999, "rank": 0.0}
+{"step": 140, "steps": 500, "lr": 2e-05, "grad_norm": 0.4011526107788086, "loss_scale": 4.0, "retries": 0, "seconds": 255.4, "max_memory_gb": 17.06, "ce": 0.2657060065539554, "rank": 0.0}
+{"step": 141, "steps": 500, "lr": 2e-05, "grad_norm": 0.5480960607528687, "loss_scale": 4.0, "retries": 0, "seconds": 257.3, "max_memory_gb": 17.06, "ce": 0.38858985202386975, "rank": 0.0}
+{"step": 142, "steps": 500, "lr": 2e-05, "grad_norm": 0.3574036955833435, "loss_scale": 4.0, "retries": 0, "seconds": 259.1, "max_memory_gb": 17.06, "ce": 0.3530458190944046, "rank": 0.0}
+{"step": 143, "steps": 500, "lr": 2e-05, "grad_norm": 0.314845472574234, "loss_scale": 4.0, "retries": 0, "seconds": 261.0, "max_memory_gb": 17.06, "ce": 0.17333721253089607, "rank": 0.0}
+{"step": 144, "steps": 500, "lr": 2e-05, "grad_norm": 0.7380245327949524, "loss_scale": 4.0, "retries": 0, "seconds": 262.8, "max_memory_gb": 17.06, "ce": 0.3677304429002106, "rank": 0.0}
+{"step": 145, "steps": 500, "lr": 2e-05, "grad_norm": 0.2884214222431183, "loss_scale": 4.0, "retries": 0, "seconds": 264.6, "max_memory_gb": 17.06, "ce": 0.23225845757406205, "rank": 0.0}
+{"step": 146, "steps": 500, "lr": 2e-05, "grad_norm": 0.4080408811569214, "loss_scale": 4.0, "retries": 0, "seconds": 266.4, "max_memory_gb": 17.06, "ce": 0.3190428973175585, "rank": 0.0}
+{"step": 147, "steps": 500, "lr": 2e-05, "grad_norm": 0.6203790307044983, "loss_scale": 4.0, "retries": 0, "seconds": 268.2, "max_memory_gb": 17.06, "ce": 0.5144716261420399, "rank": 0.0}
+{"step": 148, "steps": 500, "lr": 2e-05, "grad_norm": 0.22727632522583008, "loss_scale": 4.0, "retries": 0, "seconds": 270.2, "max_memory_gb": 17.06, "ce": 0.157042468781583, "rank": 0.0}
+{"step": 149, "steps": 500, "lr": 2e-05, "grad_norm": 0.4264734089374542, "loss_scale": 4.0, "retries": 0, "seconds": 271.9, "max_memory_gb": 17.06, "ce": 0.22714193316642195, "rank": 0.13004708290100098}
+{"step": 150, "steps": 500, "lr": 2e-05, "grad_norm": 0.7655853033065796, "loss_scale": 4.0, "retries": 0, "seconds": 273.8, "max_memory_gb": 17.06, "ce": 0.47510278690606356, "rank": 0.0029802322387695312}
+{"step": 151, "steps": 500, "lr": 2e-05, "grad_norm": 0.5654059648513794, "loss_scale": 4.0, "retries": 0, "seconds": 275.6, "max_memory_gb": 17.06, "ce": 0.24621729145292193, "rank": 0.0}
+{"step": 152, "steps": 500, "lr": 2e-05, "grad_norm": 0.6040381193161011, "loss_scale": 4.0, "retries": 0, "seconds": 277.4, "max_memory_gb": 17.06, "ce": 0.4377418155781925, "rank": 0.0}
+{"step": 153, "steps": 500, "lr": 2e-05, "grad_norm": 0.49127042293548584, "loss_scale": 4.0, "retries": 0, "seconds": 279.2, "max_memory_gb": 17.06, "ce": 0.33668928185943514, "rank": 0.0}
+{"step": 154, "steps": 500, "lr": 2e-05, "grad_norm": 1.111899733543396, "loss_scale": 4.0, "retries": 0, "seconds": 281.0, "max_memory_gb": 17.06, "ce": 0.633968616835773, "rank": 0.11069561541080475}
+{"step": 155, "steps": 500, "lr": 2e-05, "grad_norm": 1.1119927167892456, "loss_scale": 4.0, "retries": 0, "seconds": 283.0, "max_memory_gb": 17.06, "ce": 0.5830888766795397, "rank": 0.0}
+{"step": 156, "steps": 500, "lr": 2e-05, "grad_norm": 0.4022523760795593, "loss_scale": 4.0, "retries": 0, "seconds": 284.9, "max_memory_gb": 17.06, "ce": 0.2567069035721943, "rank": 0.0}
+{"step": 157, "steps": 500, "lr": 2e-05, "grad_norm": 0.4134531021118164, "loss_scale": 4.0, "retries": 0, "seconds": 286.8, "max_memory_gb": 17.06, "ce": 0.2737285391194746, "rank": 0.0}
+{"step": 158, "steps": 500, "lr": 2e-05, "grad_norm": 0.6503630876541138, "loss_scale": 4.0, "retries": 0, "seconds": 288.7, "max_memory_gb": 17.06, "ce": 0.4192064991220832, "rank": 0.0}
+{"step": 159, "steps": 500, "lr": 2e-05, "grad_norm": 0.49696820974349976, "loss_scale": 4.0, "retries": 0, "seconds": 290.6, "max_memory_gb": 17.07, "ce": 0.27906945592258126, "rank": 0.0}
+{"step": 160, "steps": 500, "lr": 2e-05, "grad_norm": 0.591069757938385, "loss_scale": 4.0, "retries": 0, "seconds": 292.5, "max_memory_gb": 17.07, "ce": 0.35883168689906597, "rank": 0.0}
+{"step": 161, "steps": 500, "lr": 2e-05, "grad_norm": 0.702985942363739, "loss_scale": 4.0, "retries": 0, "seconds": 294.3, "max_memory_gb": 17.07, "ce": 0.34690025926101953, "rank": 0.0}
+{"step": 162, "steps": 500, "lr": 2e-05, "grad_norm": 0.36513763666152954, "loss_scale": 4.0, "retries": 0, "seconds": 296.0, "max_memory_gb": 17.07, "ce": 0.2183677013963461, "rank": 0.0}
+{"step": 163, "steps": 500, "lr": 2e-05, "grad_norm": 0.6617574691772461, "loss_scale": 4.0, "retries": 0, "seconds": 297.8, "max_memory_gb": 17.07, "ce": 0.3491540604736656, "rank": 0.0}
+{"step": 164, "steps": 500, "lr": 2e-05, "grad_norm": 0.728105366230011, "loss_scale": 4.0, "retries": 0, "seconds": 299.6, "max_memory_gb": 17.07, "ce": 0.3988931989297271, "rank": 0.030606761574745178}
+{"step": 165, "steps": 500, "lr": 2e-05, "grad_norm": 0.43257254362106323, "loss_scale": 4.0, "retries": 0, "seconds": 301.4, "max_memory_gb": 17.07, "ce": 0.21884999191388488, "rank": 0.0}
+{"step": 166, "steps": 500, "lr": 2e-05, "grad_norm": 0.694508969783783, "loss_scale": 4.0, "retries": 0, "seconds": 303.3, "max_memory_gb": 17.07, "ce": 0.3328569671139121, "rank": 0.0}
+{"step": 167, "steps": 500, "lr": 2e-05, "grad_norm": 0.8062127232551575, "loss_scale": 4.0, "retries": 0, "seconds": 305.1, "max_memory_gb": 17.07, "ce": 0.413093619979918, "rank": 0.0}
+{"step": 168, "steps": 500, "lr": 2e-05, "grad_norm": 0.419758141040802, "loss_scale": 4.0, "retries": 0, "seconds": 306.9, "max_memory_gb": 17.07, "ce": 0.2459282297641039, "rank": 0.0}
+{"step": 169, "steps": 500, "lr": 2e-05, "grad_norm": 1.2298731803894043, "loss_scale": 4.0, "retries": 0, "seconds": 308.7, "max_memory_gb": 17.07, "ce": 0.6875802236609161, "rank": 0.0}
+{"step": 170, "steps": 500, "lr": 2e-05, "grad_norm": 0.6318218111991882, "loss_scale": 4.0, "retries": 0, "seconds": 310.5, "max_memory_gb": 17.07, "ce": 0.26673670625314116, "rank": 0.0}
+{"step": 171, "steps": 500, "lr": 2e-05, "grad_norm": 0.7310258746147156, "loss_scale": 4.0, "retries": 0, "seconds": 312.3, "max_memory_gb": 17.07, "ce": 0.2753348869155161, "rank": 0.0}
+{"step": 172, "steps": 500, "lr": 2e-05, "grad_norm": 0.8699655532836914, "loss_scale": 4.0, "retries": 0, "seconds": 314.1, "max_memory_gb": 17.07, "ce": 0.40184302371926606, "rank": 0.0}
+{"step": 173, "steps": 500, "lr": 2e-05, "grad_norm": 0.8709095120429993, "loss_scale": 4.0, "retries": 0, "seconds": 315.9, "max_memory_gb": 17.07, "ce": 0.3568288432434201, "rank": 0.0}
+{"step": 174, "steps": 500, "lr": 2e-05, "grad_norm": 0.7714877128601074, "loss_scale": 4.0, "retries": 0, "seconds": 317.7, "max_memory_gb": 17.07, "ce": 0.2984608580591157, "rank": 0.0}
+{"step": 175, "steps": 500, "lr": 2e-05, "grad_norm": 0.6785001754760742, "loss_scale": 4.0, "retries": 0, "seconds": 319.4, "max_memory_gb": 17.07, "ce": 0.3577356254681945, "rank": 0.0}
+{"step": 176, "steps": 500, "lr": 2e-05, "grad_norm": 0.9343921542167664, "loss_scale": 4.0, "retries": 0, "seconds": 321.3, "max_memory_gb": 17.07, "ce": 0.3081168755888939, "rank": 0.0}
+{"step": 177, "steps": 500, "lr": 2e-05, "grad_norm": 0.9734153151512146, "loss_scale": 4.0, "retries": 0, "seconds": 323.1, "max_memory_gb": 17.07, "ce": 0.3668884492944926, "rank": 0.12646567821502686}
+{"step": 178, "steps": 500, "lr": 2e-05, "grad_norm": 0.5292765498161316, "loss_scale": 4.0, "retries": 0, "seconds": 324.9, "max_memory_gb": 17.07, "ce": 0.23341851984150708, "rank": 0.0}
+{"step": 179, "steps": 500, "lr": 2e-05, "grad_norm": 0.5627446174621582, "loss_scale": 4.0, "retries": 0, "seconds": 326.6, "max_memory_gb": 17.07, "ce": 0.4274149453267455, "rank": 0.0}
+{"step": 180, "steps": 500, "lr": 2e-05, "grad_norm": 1.3119733333587646, "loss_scale": 4.0, "retries": 0, "seconds": 328.4, "max_memory_gb": 17.07, "ce": 0.5287709203548729, "rank": 0.0}
+{"step": 181, "steps": 500, "lr": 2e-05, "grad_norm": 1.202962040901184, "loss_scale": 4.0, "retries": 0, "seconds": 330.3, "max_memory_gb": 17.07, "ce": 0.32800540467724204, "rank": 0.0}
+{"step": 182, "steps": 500, "lr": 2e-05, "grad_norm": 1.5466355085372925, "loss_scale": 4.0, "retries": 0, "seconds": 332.1, "max_memory_gb": 17.07, "ce": 0.3039122853660956, "rank": 0.0}
+{"step": 183, "steps": 500, "lr": 2e-05, "grad_norm": 0.5418186187744141, "loss_scale": 4.0, "retries": 0, "seconds": 333.9, "max_memory_gb": 17.07, "ce": 0.14617282332619652, "rank": 0.0}
+{"step": 184, "steps": 500, "lr": 2e-05, "grad_norm": 0.7454182505607605, "loss_scale": 4.0, "retries": 0, "seconds": 335.7, "max_memory_gb": 17.07, "ce": 0.21988520602462813, "rank": 0.09672726690769196}
+{"step": 185, "steps": 500, "lr": 2e-05, "grad_norm": 0.6906060576438904, "loss_scale": 4.0, "retries": 0, "seconds": 337.5, "max_memory_gb": 17.07, "ce": 0.22249354468658566, "rank": 0.0}
+{"step": 186, "steps": 500, "lr": 2e-05, "grad_norm": 0.6232712268829346, "loss_scale": 4.0, "retries": 0, "seconds": 339.4, "max_memory_gb": 17.07, "ce": 0.15910027222707868, "rank": 0.0}
+{"step": 187, "steps": 500, "lr": 2e-05, "grad_norm": 0.9907282590866089, "loss_scale": 4.0, "retries": 0, "seconds": 341.2, "max_memory_gb": 17.07, "ce": 0.3109053149819374, "rank": 0.13291069865226746}
+{"step": 188, "steps": 500, "lr": 2e-05, "grad_norm": 0.28445953130722046, "loss_scale": 4.0, "retries": 0, "seconds": 343.0, "max_memory_gb": 17.07, "ce": 0.1431518459576182, "rank": 0.0}
+{"step": 189, "steps": 500, "lr": 2e-05, "grad_norm": 1.0227481126785278, "loss_scale": 4.0, "retries": 0, "seconds": 344.8, "max_memory_gb": 17.07, "ce": 0.17614029441028833, "rank": 0.0}
+{"step": 190, "steps": 500, "lr": 2e-05, "grad_norm": 1.1541433334350586, "loss_scale": 4.0, "retries": 0, "seconds": 346.6, "max_memory_gb": 17.07, "ce": 0.25326236709952354, "rank": 0.0}
+{"step": 191, "steps": 500, "lr": 2e-05, "grad_norm": 1.0163580179214478, "loss_scale": 4.0, "retries": 0, "seconds": 348.3, "max_memory_gb": 17.07, "ce": 0.2965575170237571, "rank": 0.0}
+{"step": 192, "steps": 500, "lr": 2e-05, "grad_norm": 0.5788617134094238, "loss_scale": 4.0, "retries": 0, "seconds": 350.1, "max_memory_gb": 17.07, "ce": 0.28357991483062506, "rank": 0.0}
+{"step": 193, "steps": 500, "lr": 2e-05, "grad_norm": 0.7300437688827515, "loss_scale": 4.0, "retries": 0, "seconds": 352.0, "max_memory_gb": 17.07, "ce": 0.20343787432648242, "rank": 0.12524455785751343}
+{"step": 194, "steps": 500, "lr": 2e-05, "grad_norm": 1.901782512664795, "loss_scale": 4.0, "retries": 0, "seconds": 353.8, "max_memory_gb": 17.07, "ce": 0.35861006332561374, "rank": 0.0}
+{"step": 195, "steps": 500, "lr": 2e-05, "grad_norm": 0.8357698917388916, "loss_scale": 4.0, "retries": 0, "seconds": 355.6, "max_memory_gb": 17.07, "ce": 0.2675772560760379, "rank": 0.0}
+{"step": 196, "steps": 500, "lr": 2e-05, "grad_norm": 0.6202563047409058, "loss_scale": 4.0, "retries": 0, "seconds": 357.5, "max_memory_gb": 17.07, "ce": 0.16595241497270763, "rank": 0.0}
+{"step": 197, "steps": 500, "lr": 2e-05, "grad_norm": 0.6197959780693054, "loss_scale": 4.0, "retries": 0, "seconds": 359.3, "max_memory_gb": 17.07, "ce": 0.17292138887569308, "rank": 0.09890955686569214}
+{"step": 198, "steps": 500, "lr": 2e-05, "grad_norm": 0.6396095752716064, "loss_scale": 4.0, "retries": 0, "seconds": 361.1, "max_memory_gb": 17.07, "ce": 0.21246481966227293, "rank": 0.0}
+{"step": 199, "steps": 500, "lr": 2e-05, "grad_norm": 1.218388557434082, "loss_scale": 4.0, "retries": 0, "seconds": 362.9, "max_memory_gb": 17.07, "ce": 0.23103967809583992, "rank": 0.0}
+{"step": 200, "steps": 500, "lr": 2e-05, "grad_norm": 0.9482417106628418, "loss_scale": 4.0, "retries": 0, "seconds": 364.7, "max_memory_gb": 17.07, "ce": 0.27124008047394454, "rank": 0.0}
+{"step": 201, "steps": 500, "lr": 2e-05, "grad_norm": 0.6094931960105896, "loss_scale": 4.0, "retries": 0, "seconds": 366.5, "max_memory_gb": 17.07, "ce": 0.1408142144791782, "rank": 0.0}
+{"step": 202, "steps": 500, "lr": 2e-05, "grad_norm": 0.2668677568435669, "loss_scale": 4.0, "retries": 0, "seconds": 368.2, "max_memory_gb": 17.07, "ce": 0.07008761027827859, "rank": 0.0}
+{"step": 203, "steps": 500, "lr": 2e-05, "grad_norm": 1.1558542251586914, "loss_scale": 4.0, "retries": 0, "seconds": 370.0, "max_memory_gb": 17.07, "ce": 0.2276349274907261, "rank": 0.0}
+{"step": 204, "steps": 500, "lr": 2e-05, "grad_norm": 1.197653889656067, "loss_scale": 4.0, "retries": 0, "seconds": 371.8, "max_memory_gb": 17.07, "ce": 0.256359769962728, "rank": 0.0}
+{"step": 205, "steps": 500, "lr": 2e-05, "grad_norm": 0.8773912787437439, "loss_scale": 4.0, "retries": 0, "seconds": 373.6, "max_memory_gb": 17.07, "ce": 0.21622078301152214, "rank": 0.0}
+{"step": 206, "steps": 500, "lr": 2e-05, "grad_norm": 0.837119996547699, "loss_scale": 4.0, "retries": 0, "seconds": 375.4, "max_memory_gb": 17.07, "ce": 0.1657766808057204, "rank": 0.0}
+{"step": 207, "steps": 500, "lr": 2e-05, "grad_norm": 1.8075042963027954, "loss_scale": 4.0, "retries": 0, "seconds": 377.2, "max_memory_gb": 17.07, "ce": 0.500276559847407, "rank": 0.23895415663719177}
+{"step": 208, "steps": 500, "lr": 2e-05, "grad_norm": 0.6146867275238037, "loss_scale": 4.0, "retries": 0, "seconds": 379.0, "max_memory_gb": 17.07, "ce": 0.10501860844669864, "rank": 0.0}
+{"step": 209, "steps": 500, "lr": 2e-05, "grad_norm": 0.7108584642410278, "loss_scale": 4.0, "retries": 0, "seconds": 380.9, "max_memory_gb": 17.07, "ce": 0.21449734084308147, "rank": 0.12464970350265503}
+{"step": 210, "steps": 500, "lr": 2e-05, "grad_norm": 1.2613439559936523, "loss_scale": 4.0, "retries": 0, "seconds": 382.8, "max_memory_gb": 17.07, "ce": 0.29874704021494836, "rank": 0.0}
+{"step": 211, "steps": 500, "lr": 2e-05, "grad_norm": 0.7386890053749084, "loss_scale": 4.0, "retries": 0, "seconds": 384.6, "max_memory_gb": 17.07, "ce": 0.24830687209032476, "rank": 0.0}
+{"step": 212, "steps": 500, "lr": 2e-05, "grad_norm": 1.6110005378723145, "loss_scale": 4.0, "retries": 0, "seconds": 386.4, "max_memory_gb": 17.07, "ce": 0.3558224947191775, "rank": 0.0}
+{"step": 213, "steps": 500, "lr": 2e-05, "grad_norm": 0.904782235622406, "loss_scale": 4.0, "retries": 0, "seconds": 388.2, "max_memory_gb": 17.07, "ce": 0.32647723238915205, "rank": 0.0}
+{"step": 214, "steps": 500, "lr": 2e-05, "grad_norm": 0.9413356184959412, "loss_scale": 4.0, "retries": 0, "seconds": 390.0, "max_memory_gb": 17.07, "ce": 0.3098914442816749, "rank": 0.0}
+{"step": 215, "steps": 500, "lr": 2e-05, "grad_norm": 2.467745065689087, "loss_scale": 4.0, "retries": 0, "seconds": 391.9, "max_memory_gb": 17.07, "ce": 0.46963578544091433, "rank": 0.0}
+{"step": 216, "steps": 500, "lr": 2e-05, "grad_norm": 1.1583045721054077, "loss_scale": 4.0, "retries": 0, "seconds": 393.7, "max_memory_gb": 17.07, "ce": 0.1941739316098392, "rank": 0.0}
+{"step": 217, "steps": 500, "lr": 2e-05, "grad_norm": 0.553604006767273, "loss_scale": 4.0, "retries": 0, "seconds": 395.5, "max_memory_gb": 17.07, "ce": 0.08960537100210786, "rank": 0.0}
+{"step": 218, "steps": 500, "lr": 2e-05, "grad_norm": 1.8229995965957642, "loss_scale": 4.0, "retries": 0, "seconds": 397.3, "max_memory_gb": 17.07, "ce": 0.32240480254404247, "rank": 0.0}
+{"step": 219, "steps": 500, "lr": 2e-05, "grad_norm": 0.6764983534812927, "loss_scale": 4.0, "retries": 0, "seconds": 399.2, "max_memory_gb": 17.07, "ce": 0.12740231992211193, "rank": 0.0}
+{"step": 220, "steps": 500, "lr": 2e-05, "grad_norm": 0.9909998178482056, "loss_scale": 4.0, "retries": 0, "seconds": 401.2, "max_memory_gb": 17.07, "ce": 0.1443963189376518, "rank": 0.0}
+{"step": 221, "steps": 500, "lr": 2e-05, "grad_norm": 1.207172155380249, "loss_scale": 4.0, "retries": 0, "seconds": 403.1, "max_memory_gb": 17.07, "ce": 0.17232810659334064, "rank": 0.0}
+{"step": 222, "steps": 500, "lr": 2e-05, "grad_norm": 2.8533828258514404, "loss_scale": 4.0, "retries": 0, "seconds": 405.0, "max_memory_gb": 17.07, "ce": 0.38471254077740014, "rank": 0.0}
+{"step": 223, "steps": 500, "lr": 2e-05, "grad_norm": 0.7868759632110596, "loss_scale": 4.0, "retries": 0, "seconds": 407.0, "max_memory_gb": 17.07, "ce": 0.14611067704390734, "rank": 0.0}
+{"step": 224, "steps": 500, "lr": 2e-05, "grad_norm": 0.6796360015869141, "loss_scale": 4.0, "retries": 0, "seconds": 408.9, "max_memory_gb": 17.07, "ce": 0.10894985613413155, "rank": 0.09697539359331131}
+{"step": 225, "steps": 500, "lr": 2e-05, "grad_norm": 1.0888655185699463, "loss_scale": 4.0, "retries": 0, "seconds": 410.9, "max_memory_gb": 17.07, "ce": 0.08884500770363957, "rank": 0.0}
+{"step": 226, "steps": 500, "lr": 2e-05, "grad_norm": 0.6971631646156311, "loss_scale": 4.0, "retries": 0, "seconds": 412.7, "max_memory_gb": 17.07, "ce": 0.10466231952887028, "rank": 0.0}
+{"step": 227, "steps": 500, "lr": 2e-05, "grad_norm": 3.7534406185150146, "loss_scale": 4.0, "retries": 0, "seconds": 414.5, "max_memory_gb": 17.07, "ce": 0.10777763871010393, "rank": 0.0}
+{"step": 228, "steps": 500, "lr": 2e-05, "grad_norm": 0.728191077709198, "loss_scale": 4.0, "retries": 0, "seconds": 416.3, "max_memory_gb": 17.07, "ce": 0.07428923319093883, "rank": 0.0}
+{"step": 229, "steps": 500, "lr": 2e-05, "grad_norm": 0.999570369720459, "loss_scale": 4.0, "retries": 0, "seconds": 418.1, "max_memory_gb": 17.07, "ce": 0.1874048626050353, "rank": 0.0}
+{"step": 230, "steps": 500, "lr": 2e-05, "grad_norm": 0.8016409277915955, "loss_scale": 4.0, "retries": 0, "seconds": 419.9, "max_memory_gb": 17.07, "ce": 0.23999245290178806, "rank": 0.0}
+{"step": 231, "steps": 500, "lr": 2e-05, "grad_norm": 0.8695796728134155, "loss_scale": 4.0, "retries": 0, "seconds": 421.7, "max_memory_gb": 17.07, "ce": 0.2163337438250892, "rank": 0.0}
+{"step": 232, "steps": 500, "lr": 2e-05, "grad_norm": 1.7546100616455078, "loss_scale": 4.0, "retries": 0, "seconds": 423.5, "max_memory_gb": 17.07, "ce": 0.12544035795144737, "rank": 0.0}
+{"step": 233, "steps": 500, "lr": 2e-05, "grad_norm": 1.6965603828430176, "loss_scale": 4.0, "retries": 0, "seconds": 425.3, "max_memory_gb": 17.07, "ce": 0.191050790483132, "rank": 0.0}
+{"step": 234, "steps": 500, "lr": 2e-05, "grad_norm": 1.029819130897522, "loss_scale": 4.0, "retries": 0, "seconds": 427.2, "max_memory_gb": 17.07, "ce": 0.18246077001094818, "rank": 0.0}
+{"step": 235, "steps": 500, "lr": 2e-05, "grad_norm": 1.1684067249298096, "loss_scale": 4.0, "retries": 0, "seconds": 429.0, "max_memory_gb": 17.07, "ce": 0.31298447493463755, "rank": 0.0}
+{"step": 236, "steps": 500, "lr": 2e-05, "grad_norm": 0.5531538128852844, "loss_scale": 4.0, "retries": 0, "seconds": 430.8, "max_memory_gb": 17.07, "ce": 0.11267598974518478, "rank": 0.0}
+{"step": 237, "steps": 500, "lr": 2e-05, "grad_norm": 1.7671211957931519, "loss_scale": 4.0, "retries": 0, "seconds": 432.7, "max_memory_gb": 17.07, "ce": 0.21600073820445687, "rank": 0.0}
+{"step": 238, "steps": 500, "lr": 2e-05, "grad_norm": 0.9538184404373169, "loss_scale": 4.0, "retries": 0, "seconds": 434.5, "max_memory_gb": 17.07, "ce": 0.09468177054077387, "rank": 0.0}
+{"step": 239, "steps": 500, "lr": 2e-05, "grad_norm": 0.8235322833061218, "loss_scale": 4.0, "retries": 0, "seconds": 436.3, "max_memory_gb": 17.07, "ce": 0.14585359685588628, "rank": 0.0}
+{"step": 240, "steps": 500, "lr": 2e-05, "grad_norm": 1.2784875631332397, "loss_scale": 4.0, "retries": 0, "seconds": 438.1, "max_memory_gb": 17.07, "ce": 0.10288199700880796, "rank": 0.0}
+{"step": 241, "steps": 500, "lr": 2e-05, "grad_norm": 0.635307788848877, "loss_scale": 4.0, "retries": 0, "seconds": 440.0, "max_memory_gb": 17.07, "ce": 0.11335956468246877, "rank": 0.0}
+{"step": 242, "steps": 500, "lr": 2e-05, "grad_norm": 0.7831205129623413, "loss_scale": 4.0, "retries": 0, "seconds": 441.8, "max_memory_gb": 17.07, "ce": 0.11499843385536224, "rank": 0.0}
+{"step": 243, "steps": 500, "lr": 2e-05, "grad_norm": 3.9932093620300293, "loss_scale": 4.0, "retries": 0, "seconds": 443.6, "max_memory_gb": 17.07, "ce": 0.1941828140988946, "rank": 0.0}
+{"step": 244, "steps": 500, "lr": 2e-05, "grad_norm": 2.5498039722442627, "loss_scale": 4.0, "retries": 0, "seconds": 445.4, "max_memory_gb": 17.07, "ce": 0.36075691028963774, "rank": 0.005601376295089722}
+{"step": 245, "steps": 500, "lr": 2e-05, "grad_norm": 1.0944222211837769, "loss_scale": 4.0, "retries": 0, "seconds": 447.2, "max_memory_gb": 17.07, "ce": 0.33041958790272474, "rank": 0.0}
+{"step": 246, "steps": 500, "lr": 2e-05, "grad_norm": 0.33588993549346924, "loss_scale": 4.0, "retries": 0, "seconds": 449.0, "max_memory_gb": 17.07, "ce": 0.03202961460920051, "rank": 0.0}
+{"step": 247, "steps": 500, "lr": 2e-05, "grad_norm": 0.9235217571258545, "loss_scale": 4.0, "retries": 0, "seconds": 450.9, "max_memory_gb": 17.07, "ce": 0.13512313779210672, "rank": 0.0}
+{"step": 248, "steps": 500, "lr": 2e-05, "grad_norm": 0.8263058662414551, "loss_scale": 4.0, "retries": 0, "seconds": 452.7, "max_memory_gb": 17.07, "ce": 0.09953069721814245, "rank": 0.0}
+{"step": 249, "steps": 500, "lr": 2e-05, "grad_norm": 1.5477266311645508, "loss_scale": 4.0, "retries": 0, "seconds": 454.5, "max_memory_gb": 17.07, "ce": 0.09603034274186939, "rank": 0.0}
+{"step": 250, "steps": 500, "lr": 2e-05, "grad_norm": 0.8833674788475037, "loss_scale": 4.0, "retries": 0, "seconds": 456.3, "max_memory_gb": 17.07, "ce": 0.07726652413839474, "rank": 0.0}
+{"step": 251, "steps": 500, "lr": 2e-05, "grad_norm": 0.5663745403289795, "loss_scale": 4.0, "retries": 0, "seconds": 458.1, "max_memory_gb": 17.07, "ce": 0.10614828870166093, "rank": 0.0}
+{"step": 252, "steps": 500, "lr": 2e-05, "grad_norm": 0.6445903778076172, "loss_scale": 4.0, "retries": 0, "seconds": 459.9, "max_memory_gb": 17.07, "ce": 0.042012745747342706, "rank": 0.0}
+{"step": 253, "steps": 500, "lr": 2e-05, "grad_norm": 0.9675799012184143, "loss_scale": 4.0, "retries": 0, "seconds": 461.7, "max_memory_gb": 17.07, "ce": 0.13749151150113903, "rank": 0.0}
+{"step": 254, "steps": 500, "lr": 2e-05, "grad_norm": 13.477167129516602, "loss_scale": 4.0, "retries": 0, "seconds": 463.6, "max_memory_gb": 17.07, "ce": 0.27132691629230976, "rank": 0.0}
+{"step": 255, "steps": 500, "lr": 2e-05, "grad_norm": 0.6978095769882202, "loss_scale": 4.0, "retries": 0, "seconds": 465.3, "max_memory_gb": 17.07, "ce": 0.1642457713605836, "rank": 0.0}
+{"step": 256, "steps": 500, "lr": 2e-05, "grad_norm": 0.5371504426002502, "loss_scale": 4.0, "retries": 0, "seconds": 467.1, "max_memory_gb": 17.07, "ce": 0.07037992484401911, "rank": 0.0}
+{"step": 257, "steps": 500, "lr": 2e-05, "grad_norm": 1.038596510887146, "loss_scale": 4.0, "retries": 0, "seconds": 468.9, "max_memory_gb": 17.07, "ce": 0.05640987880178727, "rank": 0.0}
+{"step": 258, "steps": 500, "lr": 2e-05, "grad_norm": 1.3217719793319702, "loss_scale": 4.0, "retries": 0, "seconds": 470.7, "max_memory_gb": 17.07, "ce": 0.09912410360993817, "rank": 0.0}
+{"step": 259, "steps": 500, "lr": 2e-05, "grad_norm": 0.3922654986381531, "loss_scale": 4.0, "retries": 0, "seconds": 472.5, "max_memory_gb": 17.07, "ce": 0.02845927531598136, "rank": 0.0}
+{"step": 260, "steps": 500, "lr": 2e-05, "grad_norm": 1.4413464069366455, "loss_scale": 4.0, "retries": 0, "seconds": 474.3, "max_memory_gb": 17.07, "ce": 0.09429941413691267, "rank": 0.0}
+{"step": 261, "steps": 500, "lr": 2e-05, "grad_norm": 0.76523756980896, "loss_scale": 4.0, "retries": 0, "seconds": 476.1, "max_memory_gb": 17.07, "ce": 0.09819268248975277, "rank": 0.0}
+{"step": 262, "steps": 500, "lr": 2e-05, "grad_norm": 1.0661110877990723, "loss_scale": 4.0, "retries": 0, "seconds": 477.9, "max_memory_gb": 17.07, "ce": 0.16669618373271078, "rank": 0.0}
+{"step": 263, "steps": 500, "lr": 2e-05, "grad_norm": 1.2564232349395752, "loss_scale": 4.0, "retries": 0, "seconds": 479.7, "max_memory_gb": 17.07, "ce": 0.1702478100778535, "rank": 0.1250704973936081}
+{"step": 264, "steps": 500, "lr": 2e-05, "grad_norm": 2.61979603767395, "loss_scale": 4.0, "retries": 0, "seconds": 481.5, "max_memory_gb": 17.07, "ce": 0.2474226992635522, "rank": 0.0}
+{"step": 265, "steps": 500, "lr": 2e-05, "grad_norm": 1.2290652990341187, "loss_scale": 4.0, "retries": 0, "seconds": 483.3, "max_memory_gb": 17.07, "ce": 0.07601014006650075, "rank": 0.0}
+{"step": 266, "steps": 500, "lr": 2e-05, "grad_norm": 26.397762298583984, "loss_scale": 4.0, "retries": 0, "seconds": 485.1, "max_memory_gb": 17.07, "ce": 0.5885411353083327, "rank": 0.0}
+{"step": 267, "steps": 500, "lr": 2e-05, "grad_norm": 0.9796267151832581, "loss_scale": 4.0, "retries": 0, "seconds": 487.0, "max_memory_gb": 17.07, "ce": 0.1312100634095259, "rank": 0.0}
+{"step": 268, "steps": 500, "lr": 2e-05, "grad_norm": 1.0585566759109497, "loss_scale": 4.0, "retries": 0, "seconds": 488.8, "max_memory_gb": 17.07, "ce": 0.1944329718244262, "rank": 0.0}
+{"step": 269, "steps": 500, "lr": 2e-05, "grad_norm": 4.675136089324951, "loss_scale": 4.0, "retries": 0, "seconds": 490.6, "max_memory_gb": 17.07, "ce": 0.5364191876142286, "rank": 0.0}
+{"step": 270, "steps": 500, "lr": 2e-05, "grad_norm": 2.2616336345672607, "loss_scale": 4.0, "retries": 0, "seconds": 492.4, "max_memory_gb": 17.07, "ce": 0.18166883720550686, "rank": 0.0}
+{"step": 271, "steps": 500, "lr": 2e-05, "grad_norm": 1.115338921546936, "loss_scale": 4.0, "retries": 0, "seconds": 494.3, "max_memory_gb": 17.07, "ce": 0.04332581668859348, "rank": 0.11697549372911453}
+{"step": 272, "steps": 500, "lr": 2e-05, "grad_norm": 1.693501591682434, "loss_scale": 4.0, "retries": 0, "seconds": 496.1, "max_memory_gb": 17.07, "ce": 0.10867558629252017, "rank": 0.0}
+{"step": 273, "steps": 500, "lr": 2e-05, "grad_norm": 0.8389749526977539, "loss_scale": 4.0, "retries": 0, "seconds": 497.9, "max_memory_gb": 17.07, "ce": 0.04915525941760279, "rank": 0.12502126395702362}
+{"step": 274, "steps": 500, "lr": 2e-05, "grad_norm": 0.6943002939224243, "loss_scale": 4.0, "retries": 0, "seconds": 499.7, "max_memory_gb": 17.07, "ce": 0.0926717147231102, "rank": 0.0}
+{"step": 275, "steps": 500, "lr": 2e-05, "grad_norm": 54.95853042602539, "loss_scale": 4.0, "retries": 0, "seconds": 501.7, "max_memory_gb": 17.07, "ce": 1.0695279681240208, "rank": 0.0}
+{"step": 276, "steps": 500, "lr": 2e-05, "grad_norm": 1.5378692150115967, "loss_scale": 4.0, "retries": 0, "seconds": 503.5, "max_memory_gb": 17.07, "ce": 0.03505704148847144, "rank": 0.0}
+{"step": 277, "steps": 500, "lr": 2e-05, "grad_norm": 0.742716908454895, "loss_scale": 4.0, "retries": 0, "seconds": 505.4, "max_memory_gb": 17.07, "ce": 0.04046284357900731, "rank": 0.0}
+{"step": 278, "steps": 500, "lr": 2e-05, "grad_norm": 2.872694969177246, "loss_scale": 4.0, "retries": 0, "seconds": 507.2, "max_memory_gb": 17.07, "ce": 0.2029049691336695, "rank": 0.02446376532316208}
+{"step": 279, "steps": 500, "lr": 2e-05, "grad_norm": 5.094605922698975, "loss_scale": 4.0, "retries": 0, "seconds": 509.0, "max_memory_gb": 17.07, "ce": 0.05657307500950992, "rank": 0.0}
+{"step": 280, "steps": 500, "lr": 2e-05, "grad_norm": 0.4117088317871094, "loss_scale": 4.0, "retries": 0, "seconds": 510.8, "max_memory_gb": 17.07, "ce": 0.0222015762119554, "rank": 0.0}
+{"step": 281, "steps": 500, "lr": 2e-05, "grad_norm": 2.088632106781006, "loss_scale": 4.0, "retries": 0, "seconds": 512.6, "max_memory_gb": 17.07, "ce": 0.1393734377634246, "rank": 0.0}
+{"step": 282, "steps": 500, "lr": 2e-05, "grad_norm": 0.7824654579162598, "loss_scale": 4.0, "retries": 0, "seconds": 514.4, "max_memory_gb": 17.07, "ce": 0.07722522303811274, "rank": 0.0}
+{"step": 283, "steps": 500, "lr": 2e-05, "grad_norm": 0.9252685308456421, "loss_scale": 4.0, "retries": 0, "seconds": 516.3, "max_memory_gb": 17.07, "ce": 0.08631628754665144, "rank": 0.0}
+{"step": 284, "steps": 500, "lr": 2e-05, "grad_norm": 1.471548080444336, "loss_scale": 4.0, "retries": 0, "seconds": 518.1, "max_memory_gb": 17.07, "ce": 0.027119736914755777, "rank": 0.0}
+{"step": 285, "steps": 500, "lr": 2e-05, "grad_norm": 1.1858681440353394, "loss_scale": 4.0, "retries": 0, "seconds": 520.1, "max_memory_gb": 17.07, "ce": 0.08690816338639706, "rank": 0.0}
+{"step": 286, "steps": 500, "lr": 2e-05, "grad_norm": 1.0755336284637451, "loss_scale": 4.0, "retries": 0, "seconds": 522.1, "max_memory_gb": 17.07, "ce": 0.05342352596926503, "rank": 0.0}
+{"step": 287, "steps": 500, "lr": 2e-05, "grad_norm": 2.1239378452301025, "loss_scale": 4.0, "retries": 0, "seconds": 524.0, "max_memory_gb": 17.07, "ce": 0.12382787914248183, "rank": 0.0}
+{"step": 288, "steps": 500, "lr": 2e-05, "grad_norm": 0.1768985539674759, "loss_scale": 4.0, "retries": 0, "seconds": 525.9, "max_memory_gb": 17.07, "ce": 0.008555743144825101, "rank": 0.0}
+{"step": 289, "steps": 500, "lr": 2e-05, "grad_norm": 2.7011189460754395, "loss_scale": 4.0, "retries": 0, "seconds": 527.7, "max_memory_gb": 17.07, "ce": 0.06649641162948683, "rank": 0.0}
+{"step": 290, "steps": 500, "lr": 2e-05, "grad_norm": 0.7964928150177002, "loss_scale": 4.0, "retries": 0, "seconds": 529.5, "max_memory_gb": 17.07, "ce": 0.07938079886662308, "rank": 0.0}
+{"step": 291, "steps": 500, "lr": 2e-05, "grad_norm": 9.334131240844727, "loss_scale": 4.0, "retries": 0, "seconds": 531.4, "max_memory_gb": 17.07, "ce": 0.03195944341132417, "rank": 0.0}
+{"step": 292, "steps": 500, "lr": 2e-05, "grad_norm": 15.70351791381836, "loss_scale": 4.0, "retries": 0, "seconds": 533.2, "max_memory_gb": 17.07, "ce": 0.2056186793197412, "rank": 0.0}
+{"step": 293, "steps": 500, "lr": 2e-05, "grad_norm": 0.7804127335548401, "loss_scale": 4.0, "retries": 0, "seconds": 535.0, "max_memory_gb": 17.07, "ce": 0.0413195654109586, "rank": 0.0}
+{"step": 294, "steps": 500, "lr": 2e-05, "grad_norm": 42.559505462646484, "loss_scale": 4.0, "retries": 0, "seconds": 536.8, "max_memory_gb": 17.07, "ce": 0.7230882995063439, "rank": 0.0}
+{"step": 295, "steps": 500, "lr": 2e-05, "grad_norm": 68.01610565185547, "loss_scale": 4.0, "retries": 0, "seconds": 538.6, "max_memory_gb": 17.07, "ce": 0.8065586521988735, "rank": 0.0}
+{"step": 296, "steps": 500, "lr": 2e-05, "grad_norm": 15.884777069091797, "loss_scale": 4.0, "retries": 0, "seconds": 540.4, "max_memory_gb": 17.07, "ce": 0.401228605711367, "rank": 0.0}
+{"step": 297, "steps": 500, "lr": 2e-05, "grad_norm": 1.9231709241867065, "loss_scale": 4.0, "retries": 0, "seconds": 542.4, "max_memory_gb": 17.07, "ce": 0.14967313458328135, "rank": 0.0}
+{"step": 298, "steps": 500, "lr": 2e-05, "grad_norm": 3.08164119720459, "loss_scale": 4.0, "retries": 0, "seconds": 544.3, "max_memory_gb": 17.07, "ce": 0.28756556364533026, "rank": 0.0}
+{"step": 299, "steps": 500, "lr": 2e-05, "grad_norm": 0.628914475440979, "loss_scale": 4.0, "retries": 0, "seconds": 546.5, "max_memory_gb": 17.07, "ce": 0.033574091794434935, "rank": 0.0}
+{"step": 300, "steps": 500, "lr": 2e-05, "grad_norm": 1.63041090965271, "loss_scale": 4.0, "retries": 0, "seconds": 548.4, "max_memory_gb": 17.07, "ce": 0.22314166161231697, "rank": 0.0}
+{"step": 301, "steps": 500, "lr": 2e-05, "grad_norm": 1.3514456748962402, "loss_scale": 4.0, "retries": 0, "seconds": 550.3, "max_memory_gb": 17.07, "ce": 0.10273840703302994, "rank": 0.0}
+{"step": 302, "steps": 500, "lr": 2e-05, "grad_norm": 0.28051647543907166, "loss_scale": 4.0, "retries": 0, "seconds": 552.3, "max_memory_gb": 17.07, "ce": 0.012601219990756363, "rank": 0.0}
+{"step": 303, "steps": 500, "lr": 2e-05, "grad_norm": 5.153307914733887, "loss_scale": 4.0, "retries": 0, "seconds": 554.3, "max_memory_gb": 17.07, "ce": 0.055778907335479744, "rank": 0.0}
+{"step": 304, "steps": 500, "lr": 2e-05, "grad_norm": 2.571110725402832, "loss_scale": 4.0, "retries": 0, "seconds": 556.3, "max_memory_gb": 17.07, "ce": 0.10168240263010375, "rank": 0.0}
+{"step": 305, "steps": 500, "lr": 2e-05, "grad_norm": 3.658956289291382, "loss_scale": 4.0, "retries": 0, "seconds": 558.3, "max_memory_gb": 17.07, "ce": 0.13872838273528032, "rank": 0.0}
+{"step": 306, "steps": 500, "lr": 2e-05, "grad_norm": 0.7847083210945129, "loss_scale": 4.0, "retries": 0, "seconds": 560.3, "max_memory_gb": 17.07, "ce": 0.02778893441427499, "rank": 0.0}
+{"step": 307, "steps": 500, "lr": 2e-05, "grad_norm": 2.080176830291748, "loss_scale": 4.0, "retries": 0, "seconds": 562.1, "max_memory_gb": 17.07, "ce": 0.19744208896008786, "rank": 0.0}
+{"step": 308, "steps": 500, "lr": 2e-05, "grad_norm": 3.392178535461426, "loss_scale": 4.0, "retries": 0, "seconds": 564.1, "max_memory_gb": 17.07, "ce": 0.2156998931968701, "rank": 0.0}
+{"step": 309, "steps": 500, "lr": 2e-05, "grad_norm": 2.1392362117767334, "loss_scale": 4.0, "retries": 0, "seconds": 566.1, "max_memory_gb": 17.07, "ce": 0.17782247222203296, "rank": 0.0}
+{"step": 310, "steps": 500, "lr": 2e-05, "grad_norm": 0.6322652697563171, "loss_scale": 4.0, "retries": 0, "seconds": 568.1, "max_memory_gb": 17.07, "ce": 0.05901012473623268, "rank": 0.0}
+{"step": 311, "steps": 500, "lr": 2e-05, "grad_norm": 1.4192384481430054, "loss_scale": 4.0, "retries": 0, "seconds": 570.0, "max_memory_gb": 17.07, "ce": 0.1494857475772733, "rank": 0.0}
+{"step": 312, "steps": 500, "lr": 2e-05, "grad_norm": 0.37757357954978943, "loss_scale": 4.0, "retries": 0, "seconds": 571.8, "max_memory_gb": 17.07, "ce": 0.021352917843614705, "rank": 0.0}
+{"step": 313, "steps": 500, "lr": 2e-05, "grad_norm": 2.7939817905426025, "loss_scale": 4.0, "retries": 0, "seconds": 573.7, "max_memory_gb": 17.07, "ce": 0.19070083606493426, "rank": 0.0}
+{"step": 314, "steps": 500, "lr": 2e-05, "grad_norm": 2.0175364017486572, "loss_scale": 4.0, "retries": 0, "seconds": 575.7, "max_memory_gb": 17.07, "ce": 0.2464748747224803, "rank": 0.0}
+{"step": 315, "steps": 500, "lr": 2e-05, "grad_norm": 0.6453493237495422, "loss_scale": 4.0, "retries": 0, "seconds": 577.5, "max_memory_gb": 17.07, "ce": 0.02980159666913096, "rank": 0.10901421308517456}
+{"step": 316, "steps": 500, "lr": 2e-05, "grad_norm": 1.7508472204208374, "loss_scale": 4.0, "retries": 0, "seconds": 579.4, "max_memory_gb": 17.07, "ce": 0.3177697026039823, "rank": 0.0}
+{"step": 317, "steps": 500, "lr": 2e-05, "grad_norm": 0.9506458044052124, "loss_scale": 4.0, "retries": 0, "seconds": 581.2, "max_memory_gb": 17.07, "ce": 0.08489194946014322, "rank": 0.0}
+{"step": 318, "steps": 500, "lr": 2e-05, "grad_norm": 50.45574951171875, "loss_scale": 4.0, "retries": 0, "seconds": 583.1, "max_memory_gb": 17.07, "ce": 0.8853241297183558, "rank": 0.0}
+{"step": 319, "steps": 500, "lr": 2e-05, "grad_norm": 1.781835913658142, "loss_scale": 4.0, "retries": 0, "seconds": 585.2, "max_memory_gb": 17.07, "ce": 0.11787933336745482, "rank": 0.0}
+{"step": 320, "steps": 500, "lr": 2e-05, "grad_norm": 1.038446307182312, "loss_scale": 4.0, "retries": 0, "seconds": 587.1, "max_memory_gb": 17.07, "ce": 0.09307682531652972, "rank": 0.0}
+{"step": 321, "steps": 500, "lr": 2e-05, "grad_norm": 7.589044570922852, "loss_scale": 4.0, "retries": 0, "seconds": 589.1, "max_memory_gb": 17.07, "ce": 0.11414150684140623, "rank": 0.1251712292432785}
+{"step": 322, "steps": 500, "lr": 2e-05, "grad_norm": 1.2987220287322998, "loss_scale": 4.0, "retries": 0, "seconds": 591.2, "max_memory_gb": 17.07, "ce": 0.020313454602728598, "rank": 0.0}
+{"step": 323, "steps": 500, "lr": 2e-05, "grad_norm": 1.6606528759002686, "loss_scale": 4.0, "retries": 0, "seconds": 593.3, "max_memory_gb": 17.07, "ce": 0.020242636936018243, "rank": 0.0}
+{"step": 324, "steps": 500, "lr": 2e-05, "grad_norm": 0.9401059150695801, "loss_scale": 4.0, "retries": 0, "seconds": 595.3, "max_memory_gb": 17.07, "ce": 0.01790972917660838, "rank": 0.0}
+{"step": 325, "steps": 500, "lr": 2e-05, "grad_norm": 1.1931896209716797, "loss_scale": 4.0, "retries": 0, "seconds": 597.2, "max_memory_gb": 17.07, "ce": 0.19086760250502266, "rank": 0.0}
+{"step": 326, "steps": 500, "lr": 2e-05, "grad_norm": 0.7039098143577576, "loss_scale": 4.0, "retries": 0, "seconds": 599.0, "max_memory_gb": 17.07, "ce": 0.03644234454986872, "rank": 0.10209010541439056}
+{"step": 327, "steps": 500, "lr": 2e-05, "grad_norm": 6.536787509918213, "loss_scale": 4.0, "retries": 0, "seconds": 600.9, "max_memory_gb": 17.07, "ce": 0.1693599794380134, "rank": 0.0}
+{"step": 328, "steps": 500, "lr": 2e-05, "grad_norm": 1.1395412683486938, "loss_scale": 4.0, "retries": 0, "seconds": 602.9, "max_memory_gb": 17.07, "ce": 0.12233418538380647, "rank": 0.0}
+{"step": 329, "steps": 500, "lr": 2e-05, "grad_norm": 1.1820497512817383, "loss_scale": 4.0, "retries": 0, "seconds": 604.7, "max_memory_gb": 17.07, "ce": 0.14693437547248323, "rank": 0.0}
+{"step": 330, "steps": 500, "lr": 2e-05, "grad_norm": 45.76583480834961, "loss_scale": 4.0, "retries": 0, "seconds": 606.5, "max_memory_gb": 17.07, "ce": 0.32283724961598637, "rank": 0.0}
+{"step": 331, "steps": 500, "lr": 2e-05, "grad_norm": 2.909571647644043, "loss_scale": 4.0, "retries": 0, "seconds": 608.3, "max_memory_gb": 17.07, "ce": 0.27106787836237345, "rank": 0.0}
+{"step": 332, "steps": 500, "lr": 2e-05, "grad_norm": 1.7114171981811523, "loss_scale": 4.0, "retries": 0, "seconds": 610.2, "max_memory_gb": 17.07, "ce": 0.10103133914526552, "rank": 0.0}
+{"step": 333, "steps": 500, "lr": 2e-05, "grad_norm": 1.2813973426818848, "loss_scale": 4.0, "retries": 0, "seconds": 611.9, "max_memory_gb": 17.07, "ce": 0.19671695585202542, "rank": 0.0}
+{"step": 334, "steps": 500, "lr": 2e-05, "grad_norm": 1.121322512626648, "loss_scale": 4.0, "retries": 0, "seconds": 613.8, "max_memory_gb": 17.07, "ce": 0.09377096392563544, "rank": 0.0}
+{"step": 335, "steps": 500, "lr": 2e-05, "grad_norm": 4.404477119445801, "loss_scale": 4.0, "retries": 0, "seconds": 615.6, "max_memory_gb": 17.07, "ce": 0.35555448102240916, "rank": 0.0}
+{"step": 336, "steps": 500, "lr": 2e-05, "grad_norm": 2.9169607162475586, "loss_scale": 4.0, "retries": 0, "seconds": 617.4, "max_memory_gb": 17.07, "ce": 0.033682318124192534, "rank": 0.0}
+{"step": 337, "steps": 500, "lr": 2e-05, "grad_norm": 1.8644278049468994, "loss_scale": 4.0, "retries": 0, "seconds": 619.2, "max_memory_gb": 17.07, "ce": 0.13686623666217201, "rank": 0.0}
+{"step": 338, "steps": 500, "lr": 2e-05, "grad_norm": 1.1037485599517822, "loss_scale": 4.0, "retries": 0, "seconds": 621.0, "max_memory_gb": 17.07, "ce": 0.16998048721870873, "rank": 0.0}
+{"step": 339, "steps": 500, "lr": 2e-05, "grad_norm": 0.8952434659004211, "loss_scale": 4.0, "retries": 0, "seconds": 622.9, "max_memory_gb": 17.07, "ce": 0.09929678354455973, "rank": 0.0}
+{"step": 340, "steps": 500, "lr": 2e-05, "grad_norm": 1.492811679840088, "loss_scale": 4.0, "retries": 0, "seconds": 624.7, "max_memory_gb": 17.07, "ce": 0.22432842021225952, "rank": 0.0}
+{"step": 341, "steps": 500, "lr": 2e-05, "grad_norm": 3.5118789672851562, "loss_scale": 4.0, "retries": 0, "seconds": 626.5, "max_memory_gb": 17.07, "ce": 0.18314072635985212, "rank": 0.0}
+{"step": 342, "steps": 500, "lr": 2e-05, "grad_norm": 2.385190963745117, "loss_scale": 4.0, "retries": 0, "seconds": 628.4, "max_memory_gb": 17.07, "ce": 0.5240926992664754, "rank": 0.0}
+{"step": 343, "steps": 500, "lr": 2e-05, "grad_norm": 0.7259657382965088, "loss_scale": 4.0, "retries": 0, "seconds": 630.2, "max_memory_gb": 17.07, "ce": 0.0599660151056014, "rank": 0.0}
+{"step": 344, "steps": 500, "lr": 2e-05, "grad_norm": 0.5817398428916931, "loss_scale": 4.0, "retries": 0, "seconds": 632.0, "max_memory_gb": 17.07, "ce": 0.021431030068924883, "rank": 0.0}
+{"step": 345, "steps": 500, "lr": 2e-05, "grad_norm": 0.7391554713249207, "loss_scale": 4.0, "retries": 0, "seconds": 633.7, "max_memory_gb": 17.07, "ce": 0.15838755373988533, "rank": 0.0}
+{"step": 346, "steps": 500, "lr": 2e-05, "grad_norm": 1.8830976486206055, "loss_scale": 4.0, "retries": 0, "seconds": 635.6, "max_memory_gb": 17.07, "ce": 0.22593518062421936, "rank": 0.0}
+{"step": 347, "steps": 500, "lr": 2e-05, "grad_norm": 1.555491328239441, "loss_scale": 4.0, "retries": 0, "seconds": 637.3, "max_memory_gb": 17.07, "ce": 0.1452798682730645, "rank": 0.0}
+{"step": 348, "steps": 500, "lr": 2e-05, "grad_norm": 0.5640721321105957, "loss_scale": 4.0, "retries": 0, "seconds": 639.1, "max_memory_gb": 17.07, "ce": 0.010761227127659367, "rank": 0.0}
+{"step": 349, "steps": 500, "lr": 2e-05, "grad_norm": 0.3296874761581421, "loss_scale": 4.0, "retries": 0, "seconds": 641.0, "max_memory_gb": 17.07, "ce": 0.07659735067500151, "rank": 0.0}
+{"step": 350, "steps": 500, "lr": 2e-05, "grad_norm": 0.5727271437644958, "loss_scale": 4.0, "retries": 0, "seconds": 642.8, "max_memory_gb": 17.07, "ce": 0.05807900059153326, "rank": 0.0}
+{"step": 351, "steps": 500, "lr": 2e-05, "grad_norm": 2.834155559539795, "loss_scale": 4.0, "retries": 0, "seconds": 644.6, "max_memory_gb": 17.07, "ce": 0.08336102581961313, "rank": 0.0}
+{"step": 352, "steps": 500, "lr": 2e-05, "grad_norm": 2.8592257499694824, "loss_scale": 4.0, "retries": 0, "seconds": 646.4, "max_memory_gb": 17.07, "ce": 0.25858549023541855, "rank": 0.0}
+{"step": 353, "steps": 500, "lr": 2e-05, "grad_norm": 1.5912189483642578, "loss_scale": 4.0, "retries": 0, "seconds": 648.2, "max_memory_gb": 17.07, "ce": 0.1342949871905148, "rank": 0.0}
+{"step": 354, "steps": 500, "lr": 2e-05, "grad_norm": 1.1700838804244995, "loss_scale": 4.0, "retries": 0, "seconds": 650.0, "max_memory_gb": 17.07, "ce": 0.1440481521131005, "rank": 0.0}
+{"step": 355, "steps": 500, "lr": 2e-05, "grad_norm": 0.5560156106948853, "loss_scale": 4.0, "retries": 0, "seconds": 651.8, "max_memory_gb": 17.07, "ce": 0.06329603867561673, "rank": 0.0}
+{"step": 356, "steps": 500, "lr": 2e-05, "grad_norm": 2.3460693359375, "loss_scale": 4.0, "retries": 0, "seconds": 653.6, "max_memory_gb": 17.07, "ce": 0.09214880967192585, "rank": 0.0}
+{"step": 357, "steps": 500, "lr": 2e-05, "grad_norm": 1.0585005283355713, "loss_scale": 4.0, "retries": 0, "seconds": 655.5, "max_memory_gb": 17.07, "ce": 0.06412978527987434, "rank": 0.0}
+{"step": 358, "steps": 500, "lr": 2e-05, "grad_norm": 1.3235357999801636, "loss_scale": 4.0, "retries": 0, "seconds": 657.2, "max_memory_gb": 17.07, "ce": 0.1273672202587477, "rank": 0.0}
+{"step": 359, "steps": 500, "lr": 2e-05, "grad_norm": 1.8152165412902832, "loss_scale": 4.0, "retries": 0, "seconds": 659.0, "max_memory_gb": 17.07, "ce": 0.061949584109243006, "rank": 0.0}
+{"step": 360, "steps": 500, "lr": 2e-05, "grad_norm": 1.4361671209335327, "loss_scale": 4.0, "retries": 0, "seconds": 660.8, "max_memory_gb": 17.07, "ce": 0.0548218012390862, "rank": 0.09106852114200592}
+{"step": 361, "steps": 500, "lr": 2e-05, "grad_norm": 1.7425620555877686, "loss_scale": 4.0, "retries": 0, "seconds": 662.6, "max_memory_gb": 17.07, "ce": 0.19132329845888307, "rank": 0.0}
+{"step": 362, "steps": 500, "lr": 2e-05, "grad_norm": 3.169404983520508, "loss_scale": 4.0, "retries": 0, "seconds": 664.4, "max_memory_gb": 17.07, "ce": 0.1975895536943426, "rank": 0.0}
+{"step": 363, "steps": 500, "lr": 2e-05, "grad_norm": 0.44860103726387024, "loss_scale": 4.0, "retries": 0, "seconds": 666.2, "max_memory_gb": 17.07, "ce": 0.022088059717134456, "rank": 0.0}
+{"step": 364, "steps": 500, "lr": 2e-05, "grad_norm": 0.9096277952194214, "loss_scale": 4.0, "retries": 0, "seconds": 668.1, "max_memory_gb": 17.07, "ce": 0.0352019276797364, "rank": 0.0}
+{"step": 365, "steps": 500, "lr": 2e-05, "grad_norm": 1.1860151290893555, "loss_scale": 4.0, "retries": 0, "seconds": 669.9, "max_memory_gb": 17.07, "ce": 0.0909723480226603, "rank": 0.0}
+{"step": 366, "steps": 500, "lr": 2e-05, "grad_norm": 11.783330917358398, "loss_scale": 4.0, "retries": 0, "seconds": 671.7, "max_memory_gb": 17.07, "ce": 0.18876429425199603, "rank": 0.0}
+{"step": 367, "steps": 500, "lr": 2e-05, "grad_norm": 0.8285585641860962, "loss_scale": 4.0, "retries": 0, "seconds": 673.5, "max_memory_gb": 17.07, "ce": 0.0667541131242615, "rank": 0.0}
+{"step": 368, "steps": 500, "lr": 2e-05, "grad_norm": 0.3539080321788788, "loss_scale": 4.0, "retries": 0, "seconds": 675.3, "max_memory_gb": 17.07, "ce": 0.010033420734544052, "rank": 0.0}
+{"step": 369, "steps": 500, "lr": 2e-05, "grad_norm": 0.8152218461036682, "loss_scale": 4.0, "retries": 0, "seconds": 677.1, "max_memory_gb": 17.07, "ce": 0.028779318614397198, "rank": 0.1134939044713974}
+{"step": 370, "steps": 500, "lr": 2e-05, "grad_norm": 5.043531894683838, "loss_scale": 4.0, "retries": 0, "seconds": 678.9, "max_memory_gb": 17.07, "ce": 0.44636870357135194, "rank": 0.0}
+{"step": 371, "steps": 500, "lr": 2e-05, "grad_norm": 1.9784172773361206, "loss_scale": 4.0, "retries": 0, "seconds": 680.8, "max_memory_gb": 17.07, "ce": 0.08845300573102577, "rank": 0.0}
+{"step": 372, "steps": 500, "lr": 2e-05, "grad_norm": 0.5398197174072266, "loss_scale": 4.0, "retries": 0, "seconds": 682.6, "max_memory_gb": 17.07, "ce": 0.019825744933768874, "rank": 0.0}
+{"step": 373, "steps": 500, "lr": 2e-05, "grad_norm": 1.4779895544052124, "loss_scale": 4.0, "retries": 0, "seconds": 684.4, "max_memory_gb": 17.07, "ce": 0.08947414522299368, "rank": 0.0}
+{"step": 374, "steps": 500, "lr": 2e-05, "grad_norm": 1.094470500946045, "loss_scale": 4.0, "retries": 0, "seconds": 686.3, "max_memory_gb": 17.07, "ce": 0.14714987920160638, "rank": 0.0}
+{"step": 375, "steps": 500, "lr": 2e-05, "grad_norm": 0.9864087104797363, "loss_scale": 4.0, "retries": 0, "seconds": 688.1, "max_memory_gb": 17.07, "ce": 0.14969967745491886, "rank": 0.0}
+{"step": 376, "steps": 500, "lr": 2e-05, "grad_norm": 0.5909481644630432, "loss_scale": 4.0, "retries": 0, "seconds": 689.9, "max_memory_gb": 17.07, "ce": 0.03901681371007726, "rank": 0.0}
+{"step": 377, "steps": 500, "lr": 2e-05, "grad_norm": 0.7060020565986633, "loss_scale": 4.0, "retries": 0, "seconds": 691.6, "max_memory_gb": 17.07, "ce": 0.07379931965806463, "rank": 0.0}
+{"step": 378, "steps": 500, "lr": 2e-05, "grad_norm": 1.529030442237854, "loss_scale": 4.0, "retries": 0, "seconds": 693.4, "max_memory_gb": 17.07, "ce": 0.08481838408624753, "rank": 0.0}
+{"step": 379, "steps": 500, "lr": 2e-05, "grad_norm": 0.7604690790176392, "loss_scale": 4.0, "retries": 0, "seconds": 695.2, "max_memory_gb": 17.07, "ce": 0.10241743221558863, "rank": 0.0}
+{"step": 380, "steps": 500, "lr": 2e-05, "grad_norm": 9.233123779296875, "loss_scale": 4.0, "retries": 0, "seconds": 697.1, "max_memory_gb": 17.07, "ce": 0.22410575494450313, "rank": 0.0}
+{"step": 381, "steps": 500, "lr": 2e-05, "grad_norm": 1.193069338798523, "loss_scale": 4.0, "retries": 0, "seconds": 698.9, "max_memory_gb": 17.07, "ce": 0.21337099706215668, "rank": 0.0}
+{"step": 382, "steps": 500, "lr": 2e-05, "grad_norm": 2.52664852142334, "loss_scale": 4.0, "retries": 0, "seconds": 700.7, "max_memory_gb": 17.07, "ce": 0.3737221070095984, "rank": 0.0}
+{"step": 383, "steps": 500, "lr": 2e-05, "grad_norm": 0.4704552888870239, "loss_scale": 4.0, "retries": 0, "seconds": 702.7, "max_memory_gb": 17.07, "ce": 0.08494193402111705, "rank": 0.0}
+{"step": 384, "steps": 500, "lr": 2e-05, "grad_norm": 2.0477209091186523, "loss_scale": 4.0, "retries": 0, "seconds": 704.7, "max_memory_gb": 17.07, "ce": 0.04511750739402487, "rank": 0.0}
+{"step": 385, "steps": 500, "lr": 2e-05, "grad_norm": 0.7494965195655823, "loss_scale": 4.0, "retries": 0, "seconds": 706.7, "max_memory_gb": 17.07, "ce": 0.10933386861324834, "rank": 0.0}
+{"step": 386, "steps": 500, "lr": 2e-05, "grad_norm": 0.7021027207374573, "loss_scale": 4.0, "retries": 0, "seconds": 708.6, "max_memory_gb": 17.07, "ce": 0.04124580473580863, "rank": 0.0}
+{"step": 387, "steps": 500, "lr": 2e-05, "grad_norm": 0.41642898321151733, "loss_scale": 4.0, "retries": 0, "seconds": 710.7, "max_memory_gb": 17.07, "ce": 0.04252803751114698, "rank": 0.0}
+{"step": 388, "steps": 500, "lr": 2e-05, "grad_norm": 1.2018300294876099, "loss_scale": 4.0, "retries": 0, "seconds": 712.6, "max_memory_gb": 17.07, "ce": 0.037917776564427186, "rank": 0.0}
+{"step": 389, "steps": 500, "lr": 2e-05, "grad_norm": 0.4230819344520569, "loss_scale": 4.0, "retries": 0, "seconds": 714.6, "max_memory_gb": 17.07, "ce": 0.00920708458579611, "rank": 0.0}
+{"step": 390, "steps": 500, "lr": 2e-05, "grad_norm": 26.255252838134766, "loss_scale": 4.0, "retries": 0, "seconds": 716.4, "max_memory_gb": 17.07, "ce": 0.25619838587135746, "rank": 0.0}
+{"step": 391, "steps": 500, "lr": 2e-05, "grad_norm": 1.6634701490402222, "loss_scale": 4.0, "retries": 0, "seconds": 718.3, "max_memory_gb": 17.07, "ce": 0.23548005097836722, "rank": 0.0}
+{"step": 392, "steps": 500, "lr": 2e-05, "grad_norm": 0.038284510374069214, "loss_scale": 4.0, "retries": 0, "seconds": 720.1, "max_memory_gb": 17.07, "ce": 0.0013799599973935983, "rank": 0.12357497960329056}
+{"step": 393, "steps": 500, "lr": 2e-05, "grad_norm": 1.2120660543441772, "loss_scale": 4.0, "retries": 0, "seconds": 721.9, "max_memory_gb": 17.07, "ce": 0.10641674950602464, "rank": 0.0}
+{"step": 394, "steps": 500, "lr": 2e-05, "grad_norm": 1.4086698293685913, "loss_scale": 4.0, "retries": 0, "seconds": 723.7, "max_memory_gb": 17.07, "ce": 0.1023281363713977, "rank": 0.0}
+{"step": 395, "steps": 500, "lr": 2e-05, "grad_norm": 0.21180853247642517, "loss_scale": 4.0, "retries": 0, "seconds": 725.5, "max_memory_gb": 17.07, "ce": 0.0059830971749761375, "rank": 0.0}
+{"step": 396, "steps": 500, "lr": 2e-05, "grad_norm": 1.0071762800216675, "loss_scale": 4.0, "retries": 0, "seconds": 727.3, "max_memory_gb": 17.07, "ce": 0.043668870738201804, "rank": 0.0}
+{"step": 397, "steps": 500, "lr": 2e-05, "grad_norm": 0.5501851439476013, "loss_scale": 4.0, "retries": 0, "seconds": 729.1, "max_memory_gb": 17.07, "ce": 0.08653694298118353, "rank": 0.0}
+{"step": 398, "steps": 500, "lr": 2e-05, "grad_norm": 0.5763351321220398, "loss_scale": 4.0, "retries": 0, "seconds": 730.9, "max_memory_gb": 17.07, "ce": 0.058667441756369954, "rank": 0.0}
+{"step": 399, "steps": 500, "lr": 2e-05, "grad_norm": 2.583360433578491, "loss_scale": 4.0, "retries": 0, "seconds": 732.7, "max_memory_gb": 17.07, "ce": 0.19861409895929683, "rank": 0.0}
+{"step": 400, "steps": 500, "lr": 2e-05, "grad_norm": 13.158416748046875, "loss_scale": 4.0, "retries": 0, "seconds": 734.6, "max_memory_gb": 17.07, "ce": 0.0894062619986471, "rank": 0.0}
+{"step": 401, "steps": 500, "lr": 2e-05, "grad_norm": 2.815885305404663, "loss_scale": 4.0, "retries": 0, "seconds": 736.4, "max_memory_gb": 17.07, "ce": 0.1703111567421729, "rank": 0.0}
+{"step": 402, "steps": 500, "lr": 2e-05, "grad_norm": 1.0705885887145996, "loss_scale": 4.0, "retries": 0, "seconds": 738.2, "max_memory_gb": 17.07, "ce": 0.09370390299045539, "rank": 0.11385632306337357}
+{"step": 403, "steps": 500, "lr": 2e-05, "grad_norm": 0.18010492622852325, "loss_scale": 4.0, "retries": 0, "seconds": 740.0, "max_memory_gb": 17.07, "ce": 0.004503507983372401, "rank": 0.0}
+{"step": 404, "steps": 500, "lr": 2e-05, "grad_norm": 1.9644784927368164, "loss_scale": 4.0, "retries": 0, "seconds": 741.9, "max_memory_gb": 17.07, "ce": 0.15923804651811224, "rank": 0.0}
+{"step": 405, "steps": 500, "lr": 2e-05, "grad_norm": 0.2645432651042938, "loss_scale": 4.0, "retries": 0, "seconds": 743.6, "max_memory_gb": 17.07, "ce": 0.00860914092118037, "rank": 0.0}
+{"step": 406, "steps": 500, "lr": 2e-05, "grad_norm": 0.9108064770698547, "loss_scale": 4.0, "retries": 0, "seconds": 745.4, "max_memory_gb": 17.07, "ce": 0.0961341850647841, "rank": 0.0}
+{"step": 407, "steps": 500, "lr": 2e-05, "grad_norm": 3.9914767742156982, "loss_scale": 4.0, "retries": 0, "seconds": 747.2, "max_memory_gb": 17.07, "ce": 0.7934043400018709, "rank": 0.26582443714141846}
+{"step": 408, "steps": 500, "lr": 2e-05, "grad_norm": 2.100152015686035, "loss_scale": 4.0, "retries": 0, "seconds": 749.1, "max_memory_gb": 17.07, "ce": 0.3458219975600514, "rank": 0.0}
+{"step": 409, "steps": 500, "lr": 2e-05, "grad_norm": 0.5140936970710754, "loss_scale": 4.0, "retries": 0, "seconds": 750.9, "max_memory_gb": 17.07, "ce": 0.12971851189467998, "rank": 0.0}
+{"step": 410, "steps": 500, "lr": 2e-05, "grad_norm": 0.6014256477355957, "loss_scale": 4.0, "retries": 0, "seconds": 752.6, "max_memory_gb": 17.07, "ce": 0.015713756347395247, "rank": 0.0}
+{"step": 411, "steps": 500, "lr": 2e-05, "grad_norm": 4.582629203796387, "loss_scale": 4.0, "retries": 0, "seconds": 754.4, "max_memory_gb": 17.07, "ce": 0.18660980821368867, "rank": 0.0}
+{"step": 412, "steps": 500, "lr": 2e-05, "grad_norm": 0.41923099756240845, "loss_scale": 4.0, "retries": 0, "seconds": 756.2, "max_memory_gb": 17.07, "ce": 0.13411518688917567, "rank": 0.12494200468063354}
+{"step": 413, "steps": 500, "lr": 2e-05, "grad_norm": 37.57792663574219, "loss_scale": 4.0, "retries": 0, "seconds": 758.0, "max_memory_gb": 17.07, "ce": 0.12838644294834012, "rank": 0.0}
+{"step": 414, "steps": 500, "lr": 2e-05, "grad_norm": 46.84855651855469, "loss_scale": 4.0, "retries": 0, "seconds": 759.9, "max_memory_gb": 17.07, "ce": 1.0953660190152732, "rank": 0.0}
+{"step": 415, "steps": 500, "lr": 2e-05, "grad_norm": 3.1212525367736816, "loss_scale": 4.0, "retries": 0, "seconds": 761.7, "max_memory_gb": 17.07, "ce": 0.3671240887360909, "rank": 0.0}
+{"step": 416, "steps": 500, "lr": 2e-05, "grad_norm": 0.2585749626159668, "loss_scale": 4.0, "retries": 0, "seconds": 763.4, "max_memory_gb": 17.07, "ce": 0.01106073034588917, "rank": 0.0}
+{"step": 417, "steps": 500, "lr": 2e-05, "grad_norm": 2.6182937622070312, "loss_scale": 4.0, "retries": 0, "seconds": 765.2, "max_memory_gb": 17.07, "ce": 0.4173480944700714, "rank": 0.12703584134578705}
+{"step": 418, "steps": 500, "lr": 2e-05, "grad_norm": 0.5765938758850098, "loss_scale": 4.0, "retries": 0, "seconds": 767.1, "max_memory_gb": 17.07, "ce": 0.04368669150608184, "rank": 0.0}
+{"step": 419, "steps": 500, "lr": 2e-05, "grad_norm": 0.3604014813899994, "loss_scale": 4.0, "retries": 0, "seconds": 768.8, "max_memory_gb": 17.07, "ce": 0.04917530338298093, "rank": 0.0}
+{"step": 420, "steps": 500, "lr": 2e-05, "grad_norm": 2.34415864944458, "loss_scale": 4.0, "retries": 0, "seconds": 770.7, "max_memory_gb": 17.07, "ce": 0.2574216496814188, "rank": 0.0}
+{"step": 421, "steps": 500, "lr": 2e-05, "grad_norm": 0.3689505159854889, "loss_scale": 4.0, "retries": 0, "seconds": 772.7, "max_memory_gb": 17.07, "ce": 0.022059370305214543, "rank": 0.12283601611852646}
+{"step": 422, "steps": 500, "lr": 2e-05, "grad_norm": 3.319200277328491, "loss_scale": 4.0, "retries": 0, "seconds": 774.7, "max_memory_gb": 17.07, "ce": 0.24989836822032885, "rank": 0.0}
+{"step": 423, "steps": 500, "lr": 2e-05, "grad_norm": 2.886554002761841, "loss_scale": 4.0, "retries": 0, "seconds": 776.7, "max_memory_gb": 17.07, "ce": 0.34270689026016043, "rank": 0.0}
+{"step": 424, "steps": 500, "lr": 2e-05, "grad_norm": 0.8400687575340271, "loss_scale": 4.0, "retries": 0, "seconds": 778.7, "max_memory_gb": 17.07, "ce": 0.03323235437255789, "rank": 0.0}
+{"step": 425, "steps": 500, "lr": 2e-05, "grad_norm": 0.7354440689086914, "loss_scale": 4.0, "retries": 0, "seconds": 780.7, "max_memory_gb": 17.07, "ce": 0.031023762370750774, "rank": 0.0}
+{"step": 426, "steps": 500, "lr": 2e-05, "grad_norm": 1.697222113609314, "loss_scale": 4.0, "retries": 0, "seconds": 782.8, "max_memory_gb": 17.07, "ce": 0.1520396564665134, "rank": 0.0}
+{"step": 427, "steps": 500, "lr": 2e-05, "grad_norm": 0.5404331088066101, "loss_scale": 4.0, "retries": 0, "seconds": 784.8, "max_memory_gb": 17.07, "ce": 0.03423290415594238, "rank": 0.0}
+{"step": 428, "steps": 500, "lr": 2e-05, "grad_norm": 0.6819256544113159, "loss_scale": 4.0, "retries": 0, "seconds": 786.9, "max_memory_gb": 17.07, "ce": 0.0231154969878844, "rank": 0.0}
+{"step": 429, "steps": 500, "lr": 2e-05, "grad_norm": 0.23510095477104187, "loss_scale": 4.0, "retries": 0, "seconds": 788.8, "max_memory_gb": 17.07, "ce": 0.009597711679816712, "rank": 0.0}
+{"step": 430, "steps": 500, "lr": 2e-05, "grad_norm": 0.7846890091896057, "loss_scale": 4.0, "retries": 0, "seconds": 790.8, "max_memory_gb": 17.07, "ce": 0.08374249450662319, "rank": 0.0}
+{"step": 431, "steps": 500, "lr": 2e-05, "grad_norm": 6.106893062591553, "loss_scale": 4.0, "retries": 0, "seconds": 792.8, "max_memory_gb": 17.07, "ce": 0.3378004596552273, "rank": 0.0}
+{"step": 432, "steps": 500, "lr": 2e-05, "grad_norm": 0.8292356729507446, "loss_scale": 4.0, "retries": 0, "seconds": 794.8, "max_memory_gb": 17.07, "ce": 0.05237989705710788, "rank": 0.0}
+{"step": 433, "steps": 500, "lr": 2e-05, "grad_norm": 5.145674705505371, "loss_scale": 4.0, "retries": 0, "seconds": 796.7, "max_memory_gb": 17.07, "ce": 0.09289197684029205, "rank": 0.0}
+{"step": 434, "steps": 500, "lr": 2e-05, "grad_norm": 0.3372342586517334, "loss_scale": 4.0, "retries": 0, "seconds": 798.5, "max_memory_gb": 17.07, "ce": 0.005705777704861248, "rank": 0.0}
+{"step": 435, "steps": 500, "lr": 2e-05, "grad_norm": 1.9027626514434814, "loss_scale": 4.0, "retries": 0, "seconds": 800.2, "max_memory_gb": 17.07, "ce": 0.16114760867094446, "rank": 0.0}
+{"step": 436, "steps": 500, "lr": 2e-05, "grad_norm": 1.7734777927398682, "loss_scale": 4.0, "retries": 0, "seconds": 802.0, "max_memory_gb": 17.07, "ce": 0.1483695980741686, "rank": 0.0}
+{"step": 437, "steps": 500, "lr": 2e-05, "grad_norm": 3.6587629318237305, "loss_scale": 4.0, "retries": 0, "seconds": 803.8, "max_memory_gb": 17.07, "ce": 0.20620235965498068, "rank": 0.11472561210393906}
+{"step": 438, "steps": 500, "lr": 2e-05, "grad_norm": 0.9274665117263794, "loss_scale": 4.0, "retries": 0, "seconds": 805.8, "max_memory_gb": 17.07, "ce": 0.04244358319192543, "rank": 0.0}
+{"step": 439, "steps": 500, "lr": 2e-05, "grad_norm": 1.0503536462783813, "loss_scale": 4.0, "retries": 0, "seconds": 807.7, "max_memory_gb": 17.07, "ce": 0.10445101263758261, "rank": 0.0}
+{"step": 440, "steps": 500, "lr": 2e-05, "grad_norm": 0.6089009046554565, "loss_scale": 4.0, "retries": 0, "seconds": 809.7, "max_memory_gb": 17.07, "ce": 0.03384561899656546, "rank": 0.0}
+{"step": 441, "steps": 500, "lr": 2e-05, "grad_norm": 5.193615913391113, "loss_scale": 4.0, "retries": 0, "seconds": 811.6, "max_memory_gb": 17.07, "ce": 0.4015672987425205, "rank": 0.0}
+{"step": 442, "steps": 500, "lr": 2e-05, "grad_norm": 0.4811243712902069, "loss_scale": 4.0, "retries": 0, "seconds": 813.5, "max_memory_gb": 17.07, "ce": 0.043591915845809126, "rank": 0.0}
+{"step": 443, "steps": 500, "lr": 2e-05, "grad_norm": 0.6791467666625977, "loss_scale": 4.0, "retries": 0, "seconds": 815.6, "max_memory_gb": 17.07, "ce": 0.04577762757708115, "rank": 0.0}
+{"step": 444, "steps": 500, "lr": 2e-05, "grad_norm": 0.46420904994010925, "loss_scale": 4.0, "retries": 0, "seconds": 817.5, "max_memory_gb": 17.07, "ce": 0.043870072889149014, "rank": 0.0}
+{"step": 445, "steps": 500, "lr": 2e-05, "grad_norm": 1.9517804384231567, "loss_scale": 4.0, "retries": 0, "seconds": 819.4, "max_memory_gb": 17.07, "ce": 0.35235829099337934, "rank": 0.08150660991668701}
+{"step": 446, "steps": 500, "lr": 2e-05, "grad_norm": 9.789992332458496, "loss_scale": 4.0, "retries": 0, "seconds": 821.2, "max_memory_gb": 17.07, "ce": 0.09337835057112898, "rank": 0.0}
+{"step": 447, "steps": 500, "lr": 2e-05, "grad_norm": 0.5306103825569153, "loss_scale": 4.0, "retries": 0, "seconds": 823.0, "max_memory_gb": 17.07, "ce": 0.12151554825823041, "rank": 0.0}
+{"step": 448, "steps": 500, "lr": 2e-05, "grad_norm": 3.0144312381744385, "loss_scale": 4.0, "retries": 0, "seconds": 824.8, "max_memory_gb": 17.07, "ce": 0.1495944716843951, "rank": 0.0}
+{"step": 449, "steps": 500, "lr": 2e-05, "grad_norm": 0.36113861203193665, "loss_scale": 4.0, "retries": 0, "seconds": 826.5, "max_memory_gb": 17.07, "ce": 0.007375790482456068, "rank": 0.0}
+{"step": 450, "steps": 500, "lr": 2e-05, "grad_norm": 1.9390101432800293, "loss_scale": 4.0, "retries": 0, "seconds": 828.4, "max_memory_gb": 17.07, "ce": 0.07605762491311907, "rank": 0.0}
+{"step": 451, "steps": 500, "lr": 2e-05, "grad_norm": 2.60428524017334, "loss_scale": 4.0, "retries": 0, "seconds": 830.2, "max_memory_gb": 17.07, "ce": 0.06502909108166932, "rank": 0.0}
+{"step": 452, "steps": 500, "lr": 2e-05, "grad_norm": 1.1738529205322266, "loss_scale": 4.0, "retries": 0, "seconds": 832.0, "max_memory_gb": 17.07, "ce": 0.15241625948237925, "rank": 0.0}
+{"step": 453, "steps": 500, "lr": 2e-05, "grad_norm": 1.9409841299057007, "loss_scale": 4.0, "retries": 0, "seconds": 833.8, "max_memory_gb": 17.07, "ce": 0.0578440132585456, "rank": 0.0}
+{"step": 454, "steps": 500, "lr": 2e-05, "grad_norm": 18.207061767578125, "loss_scale": 4.0, "retries": 0, "seconds": 835.6, "max_memory_gb": 17.07, "ce": 0.2971640246714742, "rank": 0.0}
+{"step": 455, "steps": 500, "lr": 2e-05, "grad_norm": 3.6654105186462402, "loss_scale": 4.0, "retries": 0, "seconds": 837.5, "max_memory_gb": 17.07, "ce": 0.1011205058653104, "rank": 0.0}
+{"step": 456, "steps": 500, "lr": 2e-05, "grad_norm": 0.6985408067703247, "loss_scale": 4.0, "retries": 0, "seconds": 839.3, "max_memory_gb": 17.07, "ce": 0.016042737724546896, "rank": 0.0}
+{"step": 457, "steps": 500, "lr": 2e-05, "grad_norm": 0.45658233761787415, "loss_scale": 4.0, "retries": 0, "seconds": 841.1, "max_memory_gb": 17.07, "ce": 0.02735866485909355, "rank": 0.0}
+{"step": 458, "steps": 500, "lr": 2e-05, "grad_norm": 1.9658384323120117, "loss_scale": 4.0, "retries": 0, "seconds": 842.9, "max_memory_gb": 17.07, "ce": 0.20057556172741897, "rank": 0.0}
+{"step": 459, "steps": 500, "lr": 2e-05, "grad_norm": 2.697204351425171, "loss_scale": 4.0, "retries": 0, "seconds": 844.7, "max_memory_gb": 17.07, "ce": 0.11730930613975943, "rank": 0.0}
+{"step": 460, "steps": 500, "lr": 2e-05, "grad_norm": 0.14157050848007202, "loss_scale": 4.0, "retries": 0, "seconds": 846.5, "max_memory_gb": 17.07, "ce": 0.005691257028047403, "rank": 0.0}
+{"step": 461, "steps": 500, "lr": 2e-05, "grad_norm": 3.169820547103882, "loss_scale": 4.0, "retries": 0, "seconds": 848.4, "max_memory_gb": 17.07, "ce": 0.32909538338344646, "rank": 0.0}
+{"step": 462, "steps": 500, "lr": 2e-05, "grad_norm": 0.6036567091941833, "loss_scale": 4.0, "retries": 0, "seconds": 850.2, "max_memory_gb": 17.07, "ce": 0.021676837135146343, "rank": 0.0}
+{"step": 463, "steps": 500, "lr": 2e-05, "grad_norm": 0.171522319316864, "loss_scale": 4.0, "retries": 0, "seconds": 852.0, "max_memory_gb": 17.07, "ce": 0.0050659929634093714, "rank": 0.0}
+{"step": 464, "steps": 500, "lr": 2e-05, "grad_norm": 0.05187675729393959, "loss_scale": 4.0, "retries": 0, "seconds": 853.8, "max_memory_gb": 17.07, "ce": 0.00224459679066058, "rank": 0.0}
+{"step": 465, "steps": 500, "lr": 2e-05, "grad_norm": 2.495830774307251, "loss_scale": 4.0, "retries": 0, "seconds": 855.5, "max_memory_gb": 17.07, "ce": 0.13217426855226222, "rank": 0.0}
+{"step": 466, "steps": 500, "lr": 2e-05, "grad_norm": 1.4948444366455078, "loss_scale": 4.0, "retries": 0, "seconds": 857.4, "max_memory_gb": 17.07, "ce": 0.08219582773926959, "rank": 0.0}
+{"step": 467, "steps": 500, "lr": 2e-05, "grad_norm": 22.735654830932617, "loss_scale": 4.0, "retries": 0, "seconds": 859.2, "max_memory_gb": 17.07, "ce": 0.17132001294612564, "rank": 0.0}
+{"step": 468, "steps": 500, "lr": 2e-05, "grad_norm": 12.477293014526367, "loss_scale": 4.0, "retries": 0, "seconds": 861.0, "max_memory_gb": 17.07, "ce": 0.08297931605648046, "rank": 0.0}
+{"step": 469, "steps": 500, "lr": 2e-05, "grad_norm": 1.590257167816162, "loss_scale": 4.0, "retries": 0, "seconds": 862.8, "max_memory_gb": 17.07, "ce": 0.10943547349552318, "rank": 0.0}
+{"step": 470, "steps": 500, "lr": 2e-05, "grad_norm": 1.8161860704421997, "loss_scale": 4.0, "retries": 0, "seconds": 864.7, "max_memory_gb": 17.07, "ce": 0.07286103317255765, "rank": 0.0}
+{"step": 471, "steps": 500, "lr": 2e-05, "grad_norm": 2.3037586212158203, "loss_scale": 4.0, "retries": 0, "seconds": 866.5, "max_memory_gb": 17.07, "ce": 0.21536048196230695, "rank": 0.0}
+{"step": 472, "steps": 500, "lr": 2e-05, "grad_norm": 2.3302812576293945, "loss_scale": 4.0, "retries": 0, "seconds": 868.3, "max_memory_gb": 17.07, "ce": 0.04709954548798123, "rank": 0.0}
+{"step": 473, "steps": 500, "lr": 2e-05, "grad_norm": 0.3465645909309387, "loss_scale": 4.0, "retries": 0, "seconds": 870.0, "max_memory_gb": 17.07, "ce": 0.01326375893313525, "rank": 0.12434838712215424}
+{"step": 474, "steps": 500, "lr": 2e-05, "grad_norm": 1.0027607679367065, "loss_scale": 4.0, "retries": 0, "seconds": 871.9, "max_memory_gb": 17.07, "ce": 0.11332754966315406, "rank": 0.0}
+{"step": 475, "steps": 500, "lr": 2e-05, "grad_norm": 1.9046868085861206, "loss_scale": 4.0, "retries": 0, "seconds": 873.6, "max_memory_gb": 17.07, "ce": 0.18260013845429057, "rank": 0.0}
+{"step": 476, "steps": 500, "lr": 2e-05, "grad_norm": 0.21993429958820343, "loss_scale": 4.0, "retries": 0, "seconds": 875.5, "max_memory_gb": 17.07, "ce": 0.007899392839135544, "rank": 0.0}
+{"step": 477, "steps": 500, "lr": 2e-05, "grad_norm": 0.8609859943389893, "loss_scale": 4.0, "retries": 0, "seconds": 877.3, "max_memory_gb": 17.07, "ce": 0.03201362054414858, "rank": 0.0}
+{"step": 478, "steps": 500, "lr": 2e-05, "grad_norm": 1.2932738065719604, "loss_scale": 4.0, "retries": 0, "seconds": 879.1, "max_memory_gb": 17.07, "ce": 0.08712349830511812, "rank": 0.0}
+{"step": 479, "steps": 500, "lr": 2e-05, "grad_norm": 3.5237410068511963, "loss_scale": 4.0, "retries": 0, "seconds": 880.9, "max_memory_gb": 17.07, "ce": 0.3628507033849928, "rank": 0.0}
+{"step": 480, "steps": 500, "lr": 2e-05, "grad_norm": 0.5804839134216309, "loss_scale": 4.0, "retries": 0, "seconds": 882.6, "max_memory_gb": 17.07, "ce": 0.09859475244593341, "rank": 0.0}
+{"step": 481, "steps": 500, "lr": 2e-05, "grad_norm": 1.7507072687149048, "loss_scale": 4.0, "retries": 0, "seconds": 884.5, "max_memory_gb": 17.07, "ce": 0.04988055549938508, "rank": 0.0}
+{"step": 482, "steps": 500, "lr": 2e-05, "grad_norm": 3.2354159355163574, "loss_scale": 4.0, "retries": 0, "seconds": 886.2, "max_memory_gb": 17.07, "ce": 0.05846782234675629, "rank": 0.0}
+{"step": 483, "steps": 500, "lr": 2e-05, "grad_norm": 8.160881996154785, "loss_scale": 4.0, "retries": 0, "seconds": 888.0, "max_memory_gb": 17.07, "ce": 0.4901666967998608, "rank": 0.0}
+{"step": 484, "steps": 500, "lr": 2e-05, "grad_norm": 2.2569077014923096, "loss_scale": 4.0, "retries": 0, "seconds": 889.8, "max_memory_gb": 17.07, "ce": 0.4564782838760948, "rank": 0.0}
+{"step": 485, "steps": 500, "lr": 2e-05, "grad_norm": 0.9625498652458191, "loss_scale": 4.0, "retries": 0, "seconds": 891.6, "max_memory_gb": 17.07, "ce": 0.04622445131553832, "rank": 0.0}
+{"step": 486, "steps": 500, "lr": 2e-05, "grad_norm": 2.2782251834869385, "loss_scale": 4.0, "retries": 0, "seconds": 893.4, "max_memory_gb": 17.07, "ce": 0.17797823516571043, "rank": 0.0}
+{"step": 487, "steps": 500, "lr": 2e-05, "grad_norm": 17.91301918029785, "loss_scale": 4.0, "retries": 0, "seconds": 895.2, "max_memory_gb": 17.07, "ce": 0.2873985742030527, "rank": 0.0}
+{"step": 488, "steps": 500, "lr": 2e-05, "grad_norm": 1.0695672035217285, "loss_scale": 4.0, "retries": 0, "seconds": 897.0, "max_memory_gb": 17.07, "ce": 0.06901232023528792, "rank": 0.0}
+{"step": 489, "steps": 500, "lr": 2e-05, "grad_norm": 2.850689172744751, "loss_scale": 4.0, "retries": 0, "seconds": 898.8, "max_memory_gb": 17.07, "ce": 0.19351065369664866, "rank": 0.0}
+{"step": 490, "steps": 500, "lr": 2e-05, "grad_norm": 0.3209730088710785, "loss_scale": 4.0, "retries": 0, "seconds": 900.6, "max_memory_gb": 17.07, "ce": 0.01581470083874592, "rank": 0.0}
+{"step": 491, "steps": 500, "lr": 2e-05, "grad_norm": 0.8218419551849365, "loss_scale": 4.0, "retries": 0, "seconds": 902.4, "max_memory_gb": 17.07, "ce": 0.12537065009746584, "rank": 0.0}
+{"step": 492, "steps": 500, "lr": 2e-05, "grad_norm": 0.3429209291934967, "loss_scale": 4.0, "retries": 0, "seconds": 904.2, "max_memory_gb": 17.07, "ce": 0.009044051716045942, "rank": 0.0}
+{"step": 493, "steps": 500, "lr": 2e-05, "grad_norm": 0.86928391456604, "loss_scale": 4.0, "retries": 0, "seconds": 906.1, "max_memory_gb": 17.07, "ce": 0.04164593260247784, "rank": 0.0}
+{"step": 494, "steps": 500, "lr": 2e-05, "grad_norm": 1.4742517471313477, "loss_scale": 4.0, "retries": 0, "seconds": 907.9, "max_memory_gb": 17.07, "ce": 0.10186970197764822, "rank": 0.0}
+{"step": 495, "steps": 500, "lr": 2e-05, "grad_norm": 0.9923144578933716, "loss_scale": 4.0, "retries": 0, "seconds": 909.7, "max_memory_gb": 17.07, "ce": 0.04583333553364355, "rank": 0.0}
+{"step": 496, "steps": 500, "lr": 2e-05, "grad_norm": 0.29576727747917175, "loss_scale": 4.0, "retries": 0, "seconds": 911.5, "max_memory_gb": 17.07, "ce": 0.010443472214774374, "rank": 0.0}
+{"step": 497, "steps": 500, "lr": 2e-05, "grad_norm": 2.109375, "loss_scale": 4.0, "retries": 0, "seconds": 913.3, "max_memory_gb": 17.07, "ce": 0.15028985638355152, "rank": 0.0}
+{"step": 498, "steps": 500, "lr": 2e-05, "grad_norm": 0.9160761833190918, "loss_scale": 4.0, "retries": 0, "seconds": 915.0, "max_memory_gb": 17.07, "ce": 0.10262638111089473, "rank": 0.0}
+{"step": 499, "steps": 500, "lr": 2e-05, "grad_norm": 2.727335214614868, "loss_scale": 4.0, "retries": 0, "seconds": 916.8, "max_memory_gb": 17.07, "ce": 0.18307064038845056, "rank": 0.0}
+{"step": 500, "steps": 500, "lr": 2e-05, "grad_norm": 2.383162260055542, "loss_scale": 4.0, "retries": 0, "seconds": 918.6, "max_memory_gb": 17.07, "ce": 0.2101661121523648, "rank": 0.0}

runs/single_llava/replacement/L01_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/replacement/L01_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.1.mlp.down_proj",
+    "model.language_model.layers.1.mlp.up_proj",
+    "model.language_model.layers.1.self_attn.k_proj",
+    "model.language_model.layers.1.self_attn.o_proj",
+    "model.language_model.layers.1.mlp.gate_proj",
+    "model.language_model.layers.1.self_attn.v_proj",
+    "model.language_model.layers.1.self_attn.q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/replacement/L01_s13/config.json

@@ -0,0 +1,24 @@
+{
+  "behavior": "replacement",
+  "seed": 13,
+  "model": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "backbone": "llava",
+  "train": "/data/OPSD/assets/v3/replacement.jsonl",
+  "items": 1000,
+  "steps": 500,
+  "epochs": 2,
+  "steps_per_epoch": 250,
+  "groups_per_step": 4,
+  "lr": 2e-05,
+  "scheduler": "constant",
+  "warmup_steps": 0,
+  "final_fraction": 1.0,
+  "weight_decay": 0.0,
+  "clip": 1.0,
+  "initial_loss_scale": 4.0,
+  "adapter": {
+    "layer": 1,
+    "rank": 16,
+    "trainable_parameters": 1249280
+  }
+}

runs/single_llava/replacement/L02_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/replacement/L02_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.2.mlp.gate_proj",
+    "model.language_model.layers.2.self_attn.k_proj",
+    "model.language_model.layers.2.self_attn.o_proj",
+    "model.language_model.layers.2.self_attn.v_proj",
+    "model.language_model.layers.2.self_attn.q_proj",
+    "model.language_model.layers.2.mlp.up_proj",
+    "model.language_model.layers.2.mlp.down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/replacement/L02_s13/config.json

@@ -0,0 +1,24 @@
+{
+  "behavior": "replacement",
+  "seed": 13,
+  "model": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "backbone": "llava",
+  "train": "/data/OPSD/assets/v3/replacement.jsonl",
+  "items": 1000,
+  "steps": 500,
+  "epochs": 2,
+  "steps_per_epoch": 250,
+  "groups_per_step": 4,
+  "lr": 2e-05,
+  "scheduler": "constant",
+  "warmup_steps": 0,
+  "final_fraction": 1.0,
+  "weight_decay": 0.0,
+  "clip": 1.0,
+  "initial_loss_scale": 4.0,
+  "adapter": {
+    "layer": 2,
+    "rank": 16,
+    "trainable_parameters": 1249280
+  }
+}

runs/single_llava/replacement/L06_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/replacement/L06_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.6.mlp.gate_proj",
+    "model.language_model.layers.6.self_attn.k_proj",
+    "model.language_model.layers.6.self_attn.v_proj",
+    "model.language_model.layers.6.self_attn.q_proj",
+    "model.language_model.layers.6.self_attn.o_proj",
+    "model.language_model.layers.6.mlp.up_proj",
+    "model.language_model.layers.6.mlp.down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/replacement/L06_s13/config.json

@@ -0,0 +1,24 @@
+{
+  "behavior": "replacement",
+  "seed": 13,
+  "model": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "backbone": "llava",
+  "train": "/data/OPSD/assets/v3/replacement.jsonl",
+  "items": 1000,
+  "steps": 500,
+  "epochs": 2,
+  "steps_per_epoch": 250,
+  "groups_per_step": 4,
+  "lr": 2e-05,
+  "scheduler": "constant",
+  "warmup_steps": 0,
+  "final_fraction": 1.0,
+  "weight_decay": 0.0,
+  "clip": 1.0,
+  "initial_loss_scale": 4.0,
+  "adapter": {
+    "layer": 6,
+    "rank": 16,
+    "trainable_parameters": 1249280
+  }
+}

runs/single_llava/replacement/L08_s13/adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/OPSD/assets/models/llava-1.5-7b-hf
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/OPSD/assets/models/llava-1.5-7b-hf
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

runs/single_llava/replacement/L08_s13/adapter/adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.language_model.layers.8.mlp.gate_proj",
+    "model.language_model.layers.8.self_attn.v_proj",
+    "model.language_model.layers.8.self_attn.k_proj",
+    "model.language_model.layers.8.self_attn.o_proj",
+    "model.language_model.layers.8.mlp.up_proj",
+    "model.language_model.layers.8.self_attn.q_proj",
+    "model.language_model.layers.8.mlp.down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

runs/single_llava/replacement/L08_s13/config.json

@@ -0,0 +1,24 @@
+{
+  "behavior": "replacement",
+  "seed": 13,
+  "model": "/data/OPSD/assets/models/llava-1.5-7b-hf",
+  "backbone": "llava",
+  "train": "/data/OPSD/assets/v3/replacement.jsonl",
+  "items": 1000,
+  "steps": 500,
+  "epochs": 2,
+  "steps_per_epoch": 250,
+  "groups_per_step": 4,
+  "lr": 2e-05,
+  "scheduler": "constant",
+  "warmup_steps": 0,
+  "final_fraction": 1.0,
+  "weight_decay": 0.0,
+  "clip": 1.0,
+  "initial_loss_scale": 4.0,
+  "adapter": {
+    "layer": 8,
+    "rank": 16,
+    "trainable_parameters": 1249280
+  }
+}