LuxAI Nemo resume checkpoint, group 1

BUDGET.json

@@ -0,0 +1,20 @@
+{
+  "sessions": {
+    "prior-initialization-and-storage": {
+      "estimated_usd": 0.25,
+      "note": "Conservative reconciled allowance before resumed start"
+    },
+    "1gkxd3pkvhhi8l:1788633966.05": {
+      "seconds": 2051.5786509513855,
+      "hourly_usd": 2.1199999999999997,
+      "estimated_usd": 1.2081518722269269
+    },
+    "1gkxd3pkvhhi8l:1788636535.725": {
+      "seconds": 272.67283630371094,
+      "hourly_usd": 2.1199999999999997,
+      "estimated_usd": 0.16057400360107418
+    }
+  },
+  "estimated_usd": 1.618725875828001,
+  "updated_unix": 1788636808.3978362
+}

CHECKPOINT_MANIFEST.json

@@ -0,0 +1,41 @@
+{
+  "version": 1,
+  "repo": "luxopes/LuxAI-1-Nemo-Checkpoints",
+  "config_hash": "13c210aee2e0cb667cb6864f09e474afe08e6926623856867b5ece65652f8072",
+  "data_fingerprint": "afb8ab111fcc6f5f3235c1fab31095f0085ac05923a90ce9b61589d31601a91c:ef609ecddc52ad88347338b7ac9b392539eee2305dd40379ac4aeb54349e1c7d",
+  "next_group": 1,
+  "checkpoint_name": "checkpoint-000001",
+  "files": {
+    "BUDGET.json": {
+      "bytes": 578,
+      "sha256": "2ad608d24d2498233ae087c10e2b5ee378942bf097bf1bed36567c61f0e338d3"
+    },
+    "checkpoint/COMPLETE.json": {
+      "bytes": 507,
+      "sha256": "16eb32347a6b9e0b88029a108dc1d5c6a6c506d2ac8def12b1071931702fa965"
+    },
+    "checkpoint/adapter/README.md": {
+      "bytes": 5402,
+      "sha256": "33a698ab30cfa10ef43e73ca8df6704250fe3da8101bfc69b839c03168e14d0a"
+    },
+    "checkpoint/adapter/adapter_config.json": {
+      "bytes": 1378,
+      "sha256": "255ee463e9f8bcb64d7a15067ec3ce899df66afce921e89068b1fa472b776f54"
+    },
+    "checkpoint/adapter/adapter_model.safetensors": {
+      "bytes": 975259816,
+      "sha256": "29df85d8c7c9c80f0b3a71e0ee70afa0e12a94963905f8496207b8f0cc198a17"
+    },
+    "checkpoint/optimizer-rng.pt": {
+      "bytes": 1950842693,
+      "sha256": "cb0d3e2b7f899ad65872f0708308e0d6bc275c282c1f312e7e679fae9d484440"
+    },
+    "training_config.json": {
+      "bytes": 1242,
+      "sha256": "798a090fb53a3b3a9168efead0d11033fcc614fae475a4c33277d6247770e174"
+    }
+  },
+  "active_max_seq_length": 65536,
+  "complete_resume_state": true,
+  "includes_base_weights": false
+}

checkpoint/COMPLETE.json

@@ -0,0 +1,14 @@
+{
+  "next_group": 1,
+  "input_tokens": 71345,
+  "loss_tokens": 39305,
+  "data_fingerprint": "afb8ab111fcc6f5f3235c1fab31095f0085ac05923a90ce9b61589d31601a91c:ef609ecddc52ad88347338b7ac9b392539eee2305dd40379ac4aeb54349e1c7d",
+  "config_hash": "13c210aee2e0cb667cb6864f09e474afe08e6926623856867b5ece65652f8072",
+  "active_max_seq_length": 65536,
+  "optimizer_updates": 1,
+  "skipped_rows": 0,
+  "skipped_input_tokens": 0,
+  "skipped_loss_tokens": 0,
+  "longest_trained_sequence": 7878,
+  "milestone": false
+}

checkpoint/adapter/README.md

@@ -0,0 +1,208 @@
+---
+base_model: /workspace/hf-cache/hub/models--mistralai--Ministral-3-14B-Base-2512/snapshots/5b0ceedbb42dff466ae60b258ba296f32da51384
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/workspace/hf-cache/hub/models--mistralai--Ministral-3-14B-Base-2512/snapshots/5b0ceedbb42dff466ae60b258ba296f32da51384
+- lora
+- transformers
+- unsloth
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.20.0

checkpoint/adapter/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": {
+    "base_model_class": "Mistral3ForConditionalGeneration",
+    "parent_library": "transformers.models.mistral3.modeling_mistral3",
+    "unsloth_fixed": true
+  },
+  "base_model_name_or_path": "mistralai/Ministral-3-14B-Base-2512",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": "5b0ceedbb42dff466ae60b258ba296f32da51384",
+  "target_modules": "model\\.language_model\\.layers\\.\\d+\\.(self_attn\\.(q_proj|k_proj|v_proj|o_proj)|mlp\\.(gate_proj|up_proj|down_proj))",
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

checkpoint/adapter/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29df85d8c7c9c80f0b3a71e0ee70afa0e12a94963905f8496207b8f0cc198a17
+size 975259816

checkpoint/optimizer-rng.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb0d3e2b7f899ad65872f0708308e0d6bc275c282c1f312e7e679fae9d484440
+size 1950842693

training_config.json

@@ -0,0 +1,40 @@
+{
+  "model_id": "mistralai/Ministral-3-14B-Base-2512",
+  "model_revision": "5b0ceedbb42dff466ae60b258ba296f32da51384",
+  "prepared_dir": "training/prepared",
+  "run_dir": "training/runs/nemo-unsloth-bf16-64k",
+  "backend": "unsloth",
+  "freeze_vision": true,
+  "auto_stop_pod": true,
+  "budget_usd": 150.0,
+  "finalization_reserve_usd": 10.0,
+  "other_cost_reserve_usd": 5.0,
+  "storage_hourly_allowance_usd": 0.03,
+  "max_gpu_hourly_usd": 2.2,
+  "budget_action": "save_and_stop",
+  "required_gpu": "RTX PRO 6000",
+  "pod_name_prefix": "luxai-nemo",
+  "watchdog_poll_seconds": 15,
+  "stall_timeout_minutes": 60,
+  "max_seq_length": 65536,
+  "oom_fallback_seq_length": 51200,
+  "seed": 3407,
+  "lora_rank": 64,
+  "lora_alpha": 128,
+  "lora_dropout": 0.0,
+  "learning_rate": 0.0001,
+  "min_lr_ratio": 0.1,
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.01,
+  "max_grad_norm": 1.0,
+  "loss_chunk_size": 256,
+  "checkpoint_minutes": 15,
+  "checkpoint_loss_tokens": 50000000,
+  "hf_checkpoint_repo": "luxopes/LuxAI-1-Nemo-Checkpoints",
+  "hf_checkpoint_hours": 6,
+  "hf_checkpoint_retry_minutes": 15,
+  "hf_model_repo": "luxopes/LuxAI-1-Nemo",
+  "hf_adapter_repo": "luxopes/LuxAI-1-Nemo-LoRA",
+  "hf_private": true,
+  "upload_after_training": true
+}